From 01d532c84e8fa162617b180be8b65fc88493428c Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 4 Sep 2026 13:18:02 +0800 Subject: [PATCH 01/28] Add migration overview documentation for RavenDB to SQL transition --- docs/ravendb-to-sql-migration-overview.md | 246 ++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 docs/ravendb-to-sql-migration-overview.md diff --git a/docs/ravendb-to-sql-migration-overview.md b/docs/ravendb-to-sql-migration-overview.md new file mode 100644 index 0000000000..8c5cecb922 --- /dev/null +++ b/docs/ravendb-to-sql-migration-overview.md @@ -0,0 +1,246 @@ +# Moving data from RavenDB to SQL + +## Problem + +A customer can already point ServiceControl at SQL Server or PostgreSQL. They cannot bring their existing data with them. + +## Strategy + +- Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention is shorter, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. +- The required set is small because unresolved failures are the only category a customer can act on, and its assumed that customers keep that number low by resolving and archiving. +- Anything not selected simply ages out of RavenDB, and the customer deletes the old database when they are ready. + +## Goals + +- **Minimal downtime**. Only the required data copies with ServiceControl closed. Optional data copies in the background while it serves traffic. +- **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. +- **No writes through the client**. Nothing the copier does changes the source, so the old database stays a valid fallback. +- **Abandonable up to a known point**. Free while ServiceControl is closed. After it opens, abandoning loses whatever SQL has ingested since. +- **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. +- **Every identifier anything depends on is carried across**. The event log and historic retry operations are renumbered, because nothing references their keys. +- **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. +- **No silent loss**. A migration cannot end with a selected category incomplete, and skipped rows are counted and reported. +- **Bounded impact on a live instance**. Throttled behind normal ingestion and streamed, so memory does not track the size of the database. +- **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. +- **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. + +## Nice to have, but not planned in this initial design (phase 1) + +- **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. +- **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. +- **Steerable while running.** No pause, resume, or abort. Changing anything means editing configuration and restarting. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always a ServiceControl EF Core persister, both at versions this build can read. +- **Custom migration UI via ServicePulse.** Custom checks and the event log will be used for progress reporting, but migration configuration and migration engine control will not be available via the UI. + +## Supported migration scenarios + +The copier runs inside the ServiceControl host, so every row and every message body travels from the source, through ServiceControl, to the target. There is no database-to-database transfer, no backup and restore, and no replication. Whether a combination works therefore comes down to whether the ServiceControl host can reach both ends at once, and how long it takes comes down to how far the data has to travel. + +**Supported locations:** + +- **The RavenDB source**: embedded on the ServiceControl host, self-hosted on the same network in a container, VM or bare metal, or RavenDB Cloud +- **The SQL target**: SQL Server or PostgreSQL on the ServiceControl host, elsewhere on the same network, in a container, or as a managed cloud service such as Azure SQL, Amazon RDS or Google Cloud SQL +- **Any combination of the two**, subject to the requirements below + +**By where the data has to travel:** + +- **On-prem to on-prem.** The common case and the fastest. Embedded or self-hosted RavenDB to SQL on the same host or the same network. +- **On-prem to cloud.** RavenDB on the network, managed SQL in a cloud. Works, but each batch is one round trip, so write latency multiplies by the number of batches rather than being amortised away. +- **Cloud to on-prem.** RavenDB Cloud down to local SQL. Works, and the customer pays egress on everything copied, most of which is archived messages and their bodies. +- **Cloud to cloud.** Works, and is only sensible when ServiceControl runs alongside one of them. A host sitting on-prem between two clouds pulls every byte down and pushes it straight back up. +- **The message body store is a third location.** Bodies come out of RavenDB attachments and go wherever the target is configured to put them: a filesystem, Azure Blob or S3, with small text bodies kept inline in the database. That decision is made at the same time as the database move. + +**Infrastructure requirements:** + +- The ServiceControl host needs network access to the RavenDB source, the SQL target and the body store simultaneously +- Both RavenDB databases, primary and throughput, on one server or cluster +- A SQL Server target must have Full-Text Search installed. `--setup` checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent, because message search is not optional. A stock SQL Server container image does not include it. PostgreSQL needs nothing extra, since its index is a GIN over `to_tsvector` +- A managed target's transient failures are already survivable: retry on failure is on by default and there is no setting to turn it off + +**Not supported:** + +- Any server-to-server copy: no backup and restore, no RavenDB ETL or replication into SQL, no external data pipeline +- A host that can reach only one of the two databases at a time, so no staged move by way of an offline copy +- Primary and throughput RavenDB databases in different locations +- 🍅 Merging two ServiceControl instances into one SQL database, or splitting one instance across several +- Anything but RavenDB as the source, or anything but a ServiceControl EF Core persister as the target + +🍅 Unverified: a source whose RavenDB licence has expired. The licence check inside `DatabaseSetup.Execute` is RavenDB's own, polling the server's `/license/status` and refusing an expired one, and the read-only lifecycle never calls it. So the one thing that would have blocked this is removed by design. Whether the RavenDB server itself still serves reads on an expired licence is a question about RavenDB's behaviour and licence terms that reading our code cannot answer, and one test would settle it. A customer whose licence has expired is exactly the one most likely to be leaving RavenDB. + +## Migration workflow + +1. Upgrade ServiceControl as normal, still on RavenDB. +2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated) they want copied. +3. Run `--setup` to creates the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. +4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, 🍅 and how long ServiceControl will be closed, so nobody starts a copy without knowing what it will do. +5. Start ServiceControl (`MigrationMode=true`). +6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. +7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. +8. ServiceControl opens, and whatever [optional data](#optional) they asked for is copied in the background while the instance runs normally. They can watch it from ServicePulse custom checks and events, but not steer it. +9. They run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. +10. RavenDB data can be removed. + +- If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. +- A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. +- An explicit override exists for a customer who has changed their mind and accepts leaving data behind. + +## Architecture + +```mermaid +flowchart TB + cfg["Configuration + restart
the only way to change anything"] + checks["Custom checks + activity feed
progress, with no new client needed"] + + subgraph host["One ServiceControl host process, started with MigrationMode = true"] + direction LR + raven["RavenDB persister
own AssemblyLoadContext
new read-only lifecycle"] + engine["MigrationEngine
categories, throttle,
dry run, verification"] + target["EF Core persister
SQL Server or PostgreSQL
own AssemblyLoadContext"] + raven -->|"IMigrationSource"| engine + engine -->|"IMigrationTarget"| target + end + + old[("Old RavenDB
read only, never written to")] + sql[("SQL Server or PostgreSQL
plus a new checkpoint table")] + bodies[("Message body store
filesystem, Azure Blob or S3")] + + cfg --> host + host --> checks + old --> raven + target --> sql + target --> bodies +``` + +- **No provider-specific code in the engine or the source.** The source is always RavenDB and the target is always an `IPersistence`. +- **Both persisters load into the same process**, each into its own `AssemblyLoadContext`. +- **The engine references neither assembly.** It knows only `IMigrationSource` and `IMigrationTarget`, and treats the resume cursor as an opaque value it passes from one to the other, so it can be tested against fakes on either side. + +## Startup sequence + +```mermaid +flowchart TB + A["Restart with MigrationMode = true"] --> B["Open the SQL target, exactly as today"] + B --> C["Open the old RavenDB, read only"] + C --> D{"All checks pass?"} + D -->|"No"| E["Host does not start.
Says which check failed.
Nothing has been copied."] + D -->|"Yes"| F["Copy what cannot be recreated.
Minutes. ServiceControl still closed."] + F --> G["ServiceControl opens.
New failed messages go straight to SQL."] + G --> H["Copy the selected history in the background,
throttled behind normal ingestion"] + H --> I["Verify row counts on both sides,
category by category"] + I --> J{"MigrationMode = false,
everything complete?"} + J -->|"No"| K["Host does not start.
Names what is outstanding.
An override exists."] + J -->|"Yes"| L["RavenDB is never opened again"] +``` + +**Checked before a single row moves:** + +- The SQL schema is current +- Message body storage is writable +- Both RavenDB databases are reachable +- The source is at a version this build can read +- The selected categories are valid + +## Data to be migrated (Categories) + +### Required + +- Unresolved failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table 🍅 A prerequisite to migration could be for a customer to clean this up first. i.e. archive. This will move the failed message to optional and can be migrated in the background. +- Message redirects +- Endpoint settings +- Known endpoints, including the monitored flag. One category, because the flag is a property of the endpoint row and cannot be copied without it +- Notification settings +- The licence trial end date +- Throughput history +- Retry operations, unacknowledged and historic. One category, because RavenDB holds both lists in a single document +- Licensing report masks +- The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again +- Subscriptions + +### Optional + +- Archived and resolved failed messages: the biggest category by far, and most of the copying time +- The event log, without which the ServicePulse activity feed starts empty +- Custom checks, which cost almost nothing to skip because every check re-reports on its next interval +- Group comments and failed error imports +- Failed message edits + +### Not migrated + +- The fifteen RavenDB index definitions, which map to a much smaller set of ordinary SQL indexes, and two of which are dead already +- The transient in-flight collections, which are empty when nothing is running +- `ArchiveBatches`, which exists only because of how RavenDB works +- Integration events still waiting to be sent when you switch over are never sent +- Broker and audit service version details, which refill on the throughput collector's next run + +## Reading from RavenDB + +- A third RavenDB lifecycle opens the source: connect, check the version, stop. It never calls `DatabaseSetup.Execute`. +- Both source databases must be on the same server or cluster (`LicensingDataStore.cs:35`). +- 🍅 The only version check that exists today compares the RavenDB server version to the RavenDB client version, and runs only for an external source. Nothing stamps a ServiceControl data version in the database, so refusing an unsupported source and naming the version to upgrade through is new work. +- Duration scales with distance to the source. Each body costs two round trips, a document load and an attachment fetch, and egress out of RavenDB Cloud is billed to the customer. + +## Writing to SQL + +- A whole `FailedMessage` is written with its stored status intact. No existing caller does that, though the dialect upsert already accepts a status, so the gap is smaller than it looks. +- `UniqueMessageId` keeps its value, but converts type: the source holds a string and the target column is a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. +- `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages have no `@expires`, so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. +- Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. +- Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. +- 🍅 Three identifiers narrow on the way across, and the dry run counts all three: throughput endpoint names are lower-cased into the key; subscriptions lose the message-type version, so two subscriptions differing only in major version merge onto one row; and a subscription key longer than 200 characters cannot be stored at all. + +## Batching and throttling + +- Batch size comes from the provider: SQL Server computes it from the 2,100-parameter ceiling, PostgreSQL uses a flat 50. +- The throttle is a configurable pause between batches. Lowering it, or turning `MigrationMode` off, is the only remedy for a copy competing with production. + +## Checkpointing and resume + +- 🍅 The checkpoint table is a new migration on the target. +- It holds one row per category: the selection that row ran with, the state, the resume cursor, copied and skipped counts, timings and the last error. +- Only the copier writes to it. The status command reads it. +- Rows and the resume cursor commit in one transaction. +- If a message is in both databases the SQL row wins and the copier skips it, so every category is safe to run twice. +- Each category has its own cursor, so a half-copied category picks up where it stopped. + +## Error handling + +- A `UniqueMessageId` that will not parse as a GUID is skipped and counted. +- A message whose body cannot be read is skipped whole, after three attempts. Exhausted attempts count toward the halt threshold. +- A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. + +## Dry run + +Runnable before anything starts, and again later against whatever is still outstanding. It never writes to RavenDB. + +What it resolves and reports: + +- Whether the source is embedded or external, and which server +- Which two RavenDB databases, and the setting each name came from +- What it found in each of them +- Rows per category, and message-body volume per category +- A duration for the window while ServiceControl is closed, as a range + +It runs the same six checks that gate startup, so a missing setting surfaces before a customer books an outage. + +It counts, before anything moves, the rows that cannot cross as they stand: + +- Documents whose `UniqueMessageId` will not parse as a GUID +- Throughput endpoint names that differ only in case, and so merge onto one row +- Subscriptions that differ only in message-type version, and so merge onto one row +- Subscriptions whose message type or transport address exceeds the 200-character key limit +- Integration event dispatches still queued, which are not copied and will never be sent + +It reports no duration for the optional categories, and nothing about load on the source. + +## Configuration and control + +- A customer sets `MigrationMode` and the list of categories next to it. A status command and a custom check report back. +- Categories are read fresh at every startup. Adding one copies it on the next restart, removing one deletes nothing. +- There is no HTTP API, no pause, no resume, no abort, and no way to add a category to a running instance. All of those mean editing configuration and restarting. +- The checkpoint table is a record of what happened, not a control channel. +- Stopping a copy takes a restart, so it cannot be stopped in ten seconds. + +## Out of scope + +- The audit instance, which has no EF Core persister at all, so a customer who finishes this migration is still running RavenDB for audit. The documentation needs to say so plainly and early +- The monitoring instance, which keeps its data in memory, so there is nothing to move From e0676d85a3ad241b05d7d1519dd39dc7a2ec0fa6 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 10 Sep 2026 14:35:06 +0800 Subject: [PATCH 02/28] Refine migration overview documentation for RavenDB to SQL transition --- docs/ravendb-to-sql-migration-overview.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/ravendb-to-sql-migration-overview.md b/docs/ravendb-to-sql-migration-overview.md index 8c5cecb922..60188b7acd 100644 --- a/docs/ravendb-to-sql-migration-overview.md +++ b/docs/ravendb-to-sql-migration-overview.md @@ -62,16 +62,13 @@ The copier runs inside the ServiceControl host, so every row and every message b - Any server-to-server copy: no backup and restore, no RavenDB ETL or replication into SQL, no external data pipeline - A host that can reach only one of the two databases at a time, so no staged move by way of an offline copy - Primary and throughput RavenDB databases in different locations -- 🍅 Merging two ServiceControl instances into one SQL database, or splitting one instance across several - Anything but RavenDB as the source, or anything but a ServiceControl EF Core persister as the target -🍅 Unverified: a source whose RavenDB licence has expired. The licence check inside `DatabaseSetup.Execute` is RavenDB's own, polling the server's `/license/status` and refusing an expired one, and the read-only lifecycle never calls it. So the one thing that would have blocked this is removed by design. Whether the RavenDB server itself still serves reads on an expired licence is a question about RavenDB's behaviour and licence terms that reading our code cannot answer, and one test would settle it. A customer whose licence has expired is exactly the one most likely to be leaving RavenDB. - ## Migration workflow 1. Upgrade ServiceControl as normal, still on RavenDB. 2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated) they want copied. -3. Run `--setup` to creates the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. +3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. 4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, 🍅 and how long ServiceControl will be closed, so nobody starts a copy without knowing what it will do. 5. Start ServiceControl (`MigrationMode=true`). 6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. @@ -137,6 +134,7 @@ flowchart TB - The SQL schema is current - Message body storage is writable - Both RavenDB databases are reachable +- The client certificate is valid, where the source is an external server - The source is at a version this build can read - The selected categories are valid @@ -144,7 +142,7 @@ flowchart TB ### Required -- Unresolved failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table 🍅 A prerequisite to migration could be for a customer to clean this up first. i.e. archive. This will move the failed message to optional and can be migrated in the background. +- Unresolved failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table. - Message redirects - Endpoint settings - Known endpoints, including the monitored flag. One category, because the flag is a property of the endpoint row and cannot be copied without it @@ -171,6 +169,7 @@ flowchart TB - `ArchiveBatches`, which exists only because of how RavenDB works - Integration events still waiting to be sent when you switch over are never sent - Broker and audit service version details, which refill on the throughput collector's next run +- The last computed licensed-endpoint count, which is recomputed from the throughput data that is being copied ## Reading from RavenDB @@ -207,6 +206,10 @@ flowchart TB - A `UniqueMessageId` that will not parse as a GUID is skipped and counted. - A message whose body cannot be read is skipped whole, after three attempts. Exhausted attempts count toward the halt threshold. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. +- A row already past the target's retention cutoff is skipped and counted. The sweeper would delete it within the hour, so copying it would write a body to blob storage for nothing. Two retention periods are in play: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. +- A group comment whose failure group has no messages left in the target is skipped and counted. RavenDB never expires comments but the SQL sweeper reclaims orphans, so copying one writes a row the sweeper deletes within the hour. +- The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone lets ten thousand failures pass on a five-million-row table as "only 0.2%". +- Verification therefore cannot treat any count difference as a fault. It accounts for all four skip rules, or it reports every successful migration as broken. ## Dry run From 4db14b80421bb28bbf0b45ad1a82b55029e7441b Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 10 Sep 2026 14:46:20 +0800 Subject: [PATCH 03/28] Clarify documentation on RavenDB source requirements for containerized ServiceControl migration --- docs/ravendb-to-sql-migration-overview.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/ravendb-to-sql-migration-overview.md b/docs/ravendb-to-sql-migration-overview.md index 60188b7acd..bef9fa6af7 100644 --- a/docs/ravendb-to-sql-migration-overview.md +++ b/docs/ravendb-to-sql-migration-overview.md @@ -38,7 +38,7 @@ The copier runs inside the ServiceControl host, so every row and every message b **Supported locations:** -- **The RavenDB source**: embedded on the ServiceControl host, self-hosted on the same network in a container, VM or bare metal, or RavenDB Cloud +- **The RavenDB source**: embedded on the ServiceControl host (Windows installations only, see below), self-hosted on the same network in a container, VM or bare metal, or RavenDB Cloud - **The SQL target**: SQL Server or PostgreSQL on the ServiceControl host, elsewhere on the same network, in a container, or as a managed cloud service such as Azure SQL, Amazon RDS or Google Cloud SQL - **Any combination of the two**, subject to the requirements below @@ -61,6 +61,7 @@ The copier runs inside the ServiceControl host, so every row and every message b - Any server-to-server copy: no backup and restore, no RavenDB ETL or replication into SQL, no external data pipeline - A host that can reach only one of the two databases at a time, so no staged move by way of an offline copy +- **An embedded RavenDB source when ServiceControl runs in a container.** Reading an embedded database means starting a RavenDB server process, and the container image does not carry one: `ServiceControl.Persistence.RavenDB.csproj:36` excludes the `RavenDBServer` directory from the artifact, and the copy that would restore it at `:44` is conditional on `CI` not being set, which the Dockerfile sets. A containerised instance migrating away from embedded RavenDB has to point at an external RavenDB server rather than at a data directory. Windows installations are unaffected: the installer unzips the server unconditionally - Primary and throughput RavenDB databases in different locations - Anything but RavenDB as the source, or anything but a ServiceControl EF Core persister as the target @@ -142,7 +143,7 @@ flowchart TB ### Required -- Unresolved failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table. +- Unresolved **and retry-issued** failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table. Retry-issued messages are required for the same reason unresolved ones are: issuing a retry deletes the expiry, so they never age out. Leaving one behind means the retry confirmation arrives with no row to mark resolved, and the message stays missing from the customer's list while the retry actually succeeded - Message redirects - Endpoint settings - Known endpoints, including the monitored flag. One category, because the flag is a property of the endpoint row and cannot be copied without it From 5891c621ff4b1efccd26ee27ddeff04bb64d7e6b Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 11 Sep 2026 14:04:53 +0800 Subject: [PATCH 04/28] Enhance migration overview documentation for RavenDB to SQL transition, clarifying strategies, goals, and migration limitations. --- docs/ravendb-to-sql-migration-overview.md | 83 +++++++++++++++++------ 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/docs/ravendb-to-sql-migration-overview.md b/docs/ravendb-to-sql-migration-overview.md index bef9fa6af7..93306b9566 100644 --- a/docs/ravendb-to-sql-migration-overview.md +++ b/docs/ravendb-to-sql-migration-overview.md @@ -7,15 +7,15 @@ A customer can already point ServiceControl at SQL Server or PostgreSQL. They ca ## Strategy - Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention is shorter, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. -- The required set is small because unresolved failures are the only category a customer can act on, and its assumed that customers keep that number low by resolving and archiving. +- The required set is small because unresolved failures are the only category a customer can act on, and the strategy assumes customers keep that number low by resolving and archiving. **A neglected instance breaks that assumption**: unresolved failures can legitimately be months old, and a large backlog of them makes the closed window long rather than short. The dry run is what tells a customer which case they are in. - Anything not selected simply ages out of RavenDB, and the customer deletes the old database when they are ready. ## Goals - **Minimal downtime**. Only the required data copies with ServiceControl closed. Optional data copies in the background while it serves traffic. - **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. -- **No writes through the client**. Nothing the copier does changes the source, so the old database stays a valid fallback. -- **Abandonable up to a known point**. Free while ServiceControl is closed. After it opens, abandoning loses whatever SQL has ingested since. +- **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. +- **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and RavenDB is untouched: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. - **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. - **Every identifier anything depends on is carried across**. The event log and historic retry operations are renumbered, because nothing references their keys. - **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. @@ -24,10 +24,12 @@ A customer can already point ServiceControl at SQL Server or PostgreSQL. They ca - **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. - **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. -## Nice to have, but not planned in this initial design (phase 1) +## Deliberately not built, and not currently planned + +*Each of these was considered and left out. None of them is scheduled: if one becomes a requirement it is new design work, not a later increment of this one.* - **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. -- **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. +- **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB, so once the host has served traffic there is no rollback of any kind. - **Steerable while running.** No pause, resume, or abort. Changing anything means editing configuration and restarting. - **A general-purpose migration tool.** The source is always RavenDB and the target is always a ServiceControl EF Core persister, both at versions this build can read. - **Custom migration UI via ServicePulse.** Custom checks and the event log will be used for progress reporting, but migration configuration and migration engine control will not be available via the UI. @@ -70,7 +72,7 @@ The copier runs inside the ServiceControl host, so every row and every message b 1. Upgrade ServiceControl as normal, still on RavenDB. 2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated) they want copied. 3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. -4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, 🍅 and how long ServiceControl will be closed, so nobody starts a copy without knowing what it will do. +4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the estimate is worth](#what-the-estimate-is-worth) before booking an outage around it. 5. Start ServiceControl (`MigrationMode=true`). 6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. 7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. @@ -80,7 +82,9 @@ The copier runs inside the ServiceControl host, so every row and every message b - If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. - A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. -- An explicit override exists for a customer who has changed their mind and accepts leaving data behind. +- An explicit override exists for a customer who has changed their mind and accepts leaving data behind. It marks the outstanding categories as abandoned, which is a deliberate end state rather than a failure, so the progress check settles and the guard stays armed for any later migration. +- **Steps 5 to 7 are the abort window**, which is not the override above: see [the one point you can go back](#the-one-point-you-can-go-back). +- A category that stops because too many rows failed is *halted*, and restarting will halt it again. Fix the cause and restart to resume it, or abandon it deliberately if you accept the loss. ## Architecture @@ -138,6 +142,7 @@ flowchart TB - The client certificate is valid, where the source is an external server - The source is at a version this build can read - The selected categories are valid +- `RetryHistoryDepth` is greater than zero. At zero or less, the first completed retry after the migration deletes the entire copied retry history, and no row count would ever show it ## Data to be migrated (Categories) @@ -160,7 +165,8 @@ flowchart TB - Archived and resolved failed messages: the biggest category by far, and most of the copying time - The event log, without which the ServicePulse activity feed starts empty - Custom checks, which cost almost nothing to skip because every check re-reports on its next interval -- Group comments and failed error imports +- Failed error imports, the record of errors that could not be ingested +- Group comments, **copied last of everything**, after archived and resolved messages. A comment survives only once the failed messages its group is built from have arrived, so on a large archive the comments are the last thing to appear. An empty comment field partway through a migration is the copy still running, not data loss - Failed message edits ### Not migrated @@ -172,12 +178,40 @@ flowchart TB - Broker and audit service version details, which refill on the throughput collector's next run - The last computed licensed-endpoint count, which is recomputed from the throughput data that is being copied +## What does not come across + +**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. + +**Rows skipped one at a time, and counted.** Each of these shows up in the skipped count for its category, broken out by reason, so you can see how much went and why: + +- A failed message whose `UniqueMessageId` is not a GUID. The target column is a `uniqueidentifier` and the value is never regenerated, because it is simultaneously the primary key, the ServicePulse URL, the retry correlation key and the body lookup key. +- A failed message with no processing attempts recorded against it. The SQL model keeps the newest attempt and derives the failure time, the failing endpoint and the exception from it, all of which are required columns, so a message with nothing to derive them from cannot be written at all rather than being written blank. +- A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. +- A row already past the target's retention cutoff. The sweeper would delete it within the hour, so copying it would write a body to blob storage for nothing. +- A group comment whose failure group has no messages left in the target. RavenDB never expires comments, but the SQL sweeper reclaims orphans. + +**Things that change shape, and are not counted as skips at all.** The dry run counts these before anything moves, so they are a number you see in advance rather than a discovery afterwards. They are also the ones to read twice: + +- **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, in the one category every customer copies. A message that failed five times arrives showing one attempt, and the other four are gone. +- **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. +- **A subscription whose message type or transport address exceeds 200 characters cannot be stored at all.** +- **Throughput endpoint names that differ only in case merge onto one row**, because the target key is lower-cased. +- **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. + +**A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. + +## The one point you can go back + +While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL and RavenDB is untouched and still authoritative. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. + +That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. + ## Reading from RavenDB - A third RavenDB lifecycle opens the source: connect, check the version, stop. It never calls `DatabaseSetup.Execute`. - Both source databases must be on the same server or cluster (`LicensingDataStore.cs:35`). -- 🍅 The only version check that exists today compares the RavenDB server version to the RavenDB client version, and runs only for an external source. Nothing stamps a ServiceControl data version in the database, so refusing an unsupported source and naming the version to upgrade through is new work. -- Duration scales with distance to the source. Each body costs two round trips, a document load and an attachment fetch, and egress out of RavenDB Cloud is billed to the customer. +- The source has to be at a ServiceControl version this build can read, and nothing in RavenDB records one today. The only version check that exists compares the RavenDB server version to the RavenDB client version, and runs only for an external source. So a marker is stamped into the database on upgrade, and a source without one, or one from a newer major version, is refused by name rather than misread. +- Duration scales with distance to the source. The copier already holds the document from the stream, so each body costs **one** round trip rather than two, but it is one per message and they are not batched. Egress out of RavenDB Cloud is billed to the customer. See [how long the background copy actually takes](#how-long-the-background-copy-actually-takes). ## Writing to SQL @@ -186,16 +220,16 @@ flowchart TB - `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages have no `@expires`, so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. - Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. - Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. -- 🍅 Three identifiers narrow on the way across, and the dry run counts all three: throughput endpoint names are lower-cased into the key; subscriptions lose the message-type version, so two subscriptions differing only in major version merge onto one row; and a subscription key longer than 200 characters cannot be stored at all. +- Identifiers narrow on the way across, and the dry run counts every kind. What narrows, merges or cannot be stored at all is in [what does not come across](#what-does-not-come-across). ## Batching and throttling -- Batch size comes from the provider: SQL Server computes it from the 2,100-parameter ceiling, PostgreSQL uses a flat 50. -- The throttle is a configurable pause between batches. Lowering it, or turning `MigrationMode` off, is the only remedy for a copy competing with production. +- Batch size comes from the provider: SQL Server divides its own parameter budget by the column count, PostgreSQL uses a flat 50 rows. +- The throttle is a configurable pause between batches, defaulting to 100 ms. Lowering it, or turning `MigrationMode` off, is the only remedy for a copy competing with production. ## Checkpointing and resume -- 🍅 The checkpoint table is a new migration on the target. +- The checkpoint table is a new table on the target, created by `--setup` along with the rest of the schema. - It holds one row per category: the selection that row ran with, the state, the resume cursor, copied and skipped counts, timings and the last error. - Only the copier writes to it. The status command reads it. - Rows and the resume cursor commit in one transaction. @@ -204,13 +238,12 @@ flowchart TB ## Error handling -- A `UniqueMessageId` that will not parse as a GUID is skipped and counted. -- A message whose body cannot be read is skipped whole, after three attempts. Exhausted attempts count toward the halt threshold. +- Which rows are skipped, and why, is in [what does not come across](#what-does-not-come-across). What follows is the mechanics around those rules. +- A body is read up to three times before the message is skipped whole, and the exhausted attempts count toward the halt threshold. +- Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. -- A row already past the target's retention cutoff is skipped and counted. The sweeper would delete it within the hour, so copying it would write a body to blob storage for nothing. Two retention periods are in play: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. -- A group comment whose failure group has no messages left in the target is skipped and counted. RavenDB never expires comments but the SQL sweeper reclaims orphans, so copying one writes a row the sweeper deletes within the hour. - The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone lets ten thousand failures pass on a five-million-row table as "only 0.2%". -- Verification therefore cannot treat any count difference as a fault. It accounts for all four skip rules, or it reports every successful migration as broken. +- Verification therefore cannot treat any count difference as a fault. It accounts for all five skip rules, or it reports every successful migration as broken. ## Dry run @@ -224,7 +257,7 @@ What it resolves and reports: - Rows per category, and message-body volume per category - A duration for the window while ServiceControl is closed, as a range -It runs the same six checks that gate startup, so a missing setting surfaces before a customer books an outage. +It runs the same seven checks that gate startup, so a missing setting surfaces before a customer books an outage. It counts, before anything moves, the rows that cannot cross as they stand: @@ -236,6 +269,14 @@ It counts, before anything moves, the rows that cannot cross as they stand: It reports no duration for the optional categories, and nothing about load on the source. +### When you can run the read-only commands + +`--migration-source-report`, `--migration-verify` and `--migration-dry-run` all open the RavenDB source. **On an embedded source that means stopping the ServiceControl service first**, because a second RavenDB process cannot attach to a data directory the first one holds. Plan the dry run as part of the outage rather than as something you run the day before while the instance keeps serving traffic. On an external source, a container or RavenDB Cloud, all three run against a live instance with no interruption. + +One deployment shape they cannot help at all: all three need shell access to the host, so a containerised instance has no easy way to run them, and a containerised instance cannot use an embedded source either. + +`--migration-status` is the exception and is deliberately so: it reads only the checkpoint table in SQL and never opens the source, so it works on every source shape at any time, including during the background copy. It is the command to use for watching progress. + ## Configuration and control - A customer sets `MigrationMode` and the list of categories next to it. A status command and a custom check report back. @@ -246,5 +287,5 @@ It reports no duration for the optional categories, and nothing about load on th ## Out of scope -- The audit instance, which has no EF Core persister at all, so a customer who finishes this migration is still running RavenDB for audit. The documentation needs to say so plainly and early +- The audit instance, which has no EF Core persister at all, so a customer who finishes this migration is still running RavenDB for audit. This is stated up front under [Problem](#problem), because it changes whether the migration is worth doing at all - The monitoring instance, which keeps its data in memory, so there is nothing to move From 94d13ba535fa9bc0e49da120073b04ec811f490e Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 11 Sep 2026 15:59:44 +0800 Subject: [PATCH 05/28] Implement the foundation for migration engine --- .github/workflows/ci.yml | 8 +- .../.editorconfig | 5 + .../MigrationSourceReportCommandTests.cs | 62 ++++ .../MigrationSourceServer.cs | 94 +++++ .../ServiceControl.Migration.Tests.csproj | 27 ++ .../TwoPersistersInOneProcessTests.cs | 128 +++++++ .../DataMigration/RavenMigrationSource.cs | 41 +++ .../RavenReadOnlySourceLifecycle.cs | 189 ++++++++++ .../RavenPersistenceConfiguration.cs | 9 +- .../ReadOnlySourceLifecycleTests.cs | 346 ++++++++++++++++++ .../DataMigration/IMigrationSource.cs | 16 + .../DataMigration/IMigrationSourceFactory.cs | 6 + .../DataMigration/MigrationSourceDatabase.cs | 7 + .../MigrationSourceDescription.cs | 8 + ...rovals.PlatformSampleSettings.approved.txt | 1 + .../MigrationSourceReportArgumentTests.cs | 21 ++ src/ServiceControl.slnx | 1 + .../Commands/MigrationSourceReportCommand.cs | 56 +++ src/ServiceControl/Hosting/Help.txt | 14 + src/ServiceControl/Hosting/HostArguments.cs | 16 + .../Infrastructure/Settings/Settings.cs | 1 + .../Persistence/PersistenceFactory.cs | 42 ++- src/ServiceControl/ServiceControl.csproj | 1 + 23 files changed, 1089 insertions(+), 10 deletions(-) create mode 100644 src/ServiceControl.Migration.Tests/.editorconfig create mode 100644 src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs create mode 100644 src/ServiceControl.Migration.Tests/MigrationSourceServer.cs create mode 100644 src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj create mode 100644 src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs create mode 100644 src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs create mode 100644 src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b276a4134..1d01930ff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: # Categories are declared by the property in each test project. A category can # span several projects that share infrastructure, so that the container is provisioned once # and the union of their build closures is compiled once. - test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] + test-category: [ DefaultCore, DefaultMonitoring, Migration, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] include: - os: windows-latest os-name: Windows @@ -96,7 +96,7 @@ jobs: id: select run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }} - name: Download RavenDB Server - if: matrix.test-category == 'DefaultCore' + if: contains(fromJSON('["DefaultCore", "Migration"]'), matrix.test-category) run: ./tools/download-ravendb-server.ps1 - name: Build id: build @@ -120,7 +120,7 @@ jobs: # prevent and cost the critical path about two minutes; the step timeouts turn a hang into a # five minute failure instead of a thirty minute one. - name: Setup WSL - if: contains(fromJSON('["RabbitMQ", "SqlServer", "PostgreSql", "IBMMQ"]'), matrix.test-category) + if: contains(fromJSON('["RabbitMQ", "SqlServer", "PostgreSql", "IBMMQ", "Migration"]'), matrix.test-category) uses: Particular/setup-wsl-action@v1.3.0 timeout-minutes: 5 with: @@ -129,7 +129,7 @@ jobs: memory: 8GB - name: Setup SQL Server uses: Particular/install-sql-server-action@v3.0.0 - if: matrix.test-category == 'SqlServer' + if: contains(fromJSON('["SqlServer", "Migration"]'), matrix.test-category) with: connection-string-env-var: ServiceControl_Persistence_SqlServer_ConnectionString catalog: ServiceControl diff --git a/src/ServiceControl.Migration.Tests/.editorconfig b/src/ServiceControl.Migration.Tests/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Migration.Tests/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs new file mode 100644 index 0000000000..8184bc21de --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Hosting.Commands; + +[TestFixture] +[NonParallelizable] +class MigrationSourceReportCommandTests +{ + [SetUp] + public void SetUp() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", null); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", null); + } + + [Test] + public async Task The_report_names_the_databases_the_settings_and_the_collections() + { + var settings = new Settings(persisterType: "RavenDB", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + + var writer = new StringWriter(); + var original = Console.Out; + Console.SetOut(writer); + + try + { + await new MigrationSourceReportCommand().Execute(new HostArguments([]), settings); + } + finally + { + Console.SetOut(original); + } + + var report = writer.ToString(); + + Assert.Multiple(() => + { + Assert.That(report, Does.Contain("(external)")); + Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); + Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); + Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); + }); + } +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs new file mode 100644 index 0000000000..9b6e933c03 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence; +using ServiceControl.RavenDB; +using TestHelper; + +/// +/// One embedded RavenDB for the whole assembly, holding a seeded database that stands in for a +/// customer's old instance. EmbeddedServer.Instance is process wide, so there can only be one. +/// +static class MigrationSourceServer +{ + public static string ServerUrl { get; private set; } + + public static string PrimaryDatabase { get; private set; } + + public static string ThroughputDatabase { get; private set; } + + public static async Task Start(CancellationToken cancellationToken = default) + { + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Data"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Logs"); + var port = PortUtility.GetAssignedOrAvailablePort(33377); + + ServerUrl = $"http://localhost:{port}"; + PrimaryDatabase = "primary"; + ThroughputDatabase = "throughput"; + + var configuration = new EmbeddedDatabaseConfiguration(ServerUrl, PrimaryDatabase, dbPath, logPath, "Operations") { RunInMemory = true }; + database = EmbeddedDatabase.Start(configuration, lifetime); + + using var store = await database.Connect(cancellationToken); + + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(PrimaryDatabase)), cancellationToken); + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(ThroughputDatabase)), cancellationToken); + + using var seeder = new DocumentStore + { + Urls = [ServerUrl], + Database = PrimaryDatabase, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + using var session = seeder.OpenAsyncSession(); + await session.StoreAsync(new EndpointSettings { Name = "Sales", TrackInstances = true }, "EndpointSettings/1", cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + + public static async Task Stop() + { + if (database is null) + { + return; + } + + await database.Stop(CancellationToken.None); + database.Dispose(); + database = null; + lifetime.StopApplication(); + } + + static EmbeddedDatabase database; + static readonly TestLifetime lifetime = new(); + + sealed class TestLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => stopping.Cancel(); + + readonly CancellationTokenSource stopping = new(); + } +} + +[SetUpFixture] +public class MigrationSourceServerFixture +{ + [OneTimeSetUp] + public Task StartSource() => MigrationSourceServer.Start(); + + [OneTimeTearDown] + public Task StopSource() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj new file mode 100644 index 0000000000..8b9f287883 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj @@ -0,0 +1,27 @@ + + + + net10.0 + Migration + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs new file mode 100644 index 0000000000..af0c18df37 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -0,0 +1,128 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.Loader; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class TwoPersistersInOneProcessTests +{ + Settings settings; + string bodyStoragePath; + + [SetUp] + public void SetUp() + { + bodyStoragePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n")); + + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // CI's install-sql-server-action publishes only this variable and creates the catalog + // ServiceControl; a developer machine has neither, so the local default is the fallback. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", + Environment.GetEnvironmentVariable("ServiceControl_Persistence_SqlServer_ConnectionString") + ?? "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", bodyStoragePath); + SetVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + SetVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + SetVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + [Test] + public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_identity() + { + Assert.That(settings.AssemblyLoadContextResolver(typeof(Settings).Assembly.Location), Is.InstanceOf(), + "This test is meaningless if the resolver is pinned to AssemblyLoadContext.Default the way every acceptance test pins it."); + + var services = new ServiceCollection(); + services.AddPersistence(settings); + var targetSettings = settings.PersisterSpecificSettings; + + await using var source = await PersistenceFactory.OpenMigrationSource(settings); + + var sourceContext = AssemblyLoadContext.GetLoadContext(source.GetType().Assembly); + var targetContext = AssemblyLoadContext.GetLoadContext(settings.PersisterSpecificSettings.GetType().Assembly); + + Assert.Multiple(() => + { + Assert.That(sourceContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The RavenDB persister must load into its own plugin context, or this proves nothing."); + Assert.That(targetContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The target persister must be isolated too, otherwise only one half of the pairing is being tested."); + Assert.That(sourceContext, Is.Not.SameAs(targetContext), + "Source and target must land in separate contexts. That separation is the whole feature, and nothing else here asserts it."); + Assert.That(source, Is.InstanceOf(), + "A plugin-context type must still cast to the host's copy of the interface."); + Assert.That(settings.PersisterSpecificSettings, Is.SameAs(targetSettings), + "Opening a source must leave the target's settings object exactly where it was."); + }); + + var description = await source.Describe(); + + Assert.Multiple(() => + { + Assert.That(AssemblyLoadContext.GetLoadContext(description.GetType().Assembly), Is.SameAs(AssemblyLoadContext.Default), + "A shared type produced inside the plugin context must arrive as the host's own type."); + Assert.That(description.PrimaryDatabase, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + "The source read its own RavenDB settings rather than the target's."); + }); + } + + [Test] + public void Asking_a_SQL_persister_for_a_source_names_the_setting_to_change() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", "SQLServer"); + + try + { + var refusal = Assert.ThrowsAsync(async () => await PersistenceFactory.OpenMigrationSource(new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)))); + + Assert.That(refusal.Message, Does.Contain("cannot be read as a migration source").And.Contain("ServiceControl/Migration/SourcePersistenceType")); + } + finally + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", null); + } + } + + [Test] + public void Asking_a_SQL_persister_for_maintenance_mode_is_still_refused() + { + var refusal = Assert.Throws(() => PersistenceFactory.Create(settings, maintenanceMode: true)); + + Assert.That(refusal.Message, Does.Contain("Maintenance mode is not supported").And.Contain("SQLServer"), + "PersistenceFactory's SupportsMaintenanceMode guard is what turns a persister mismatch into a sentence instead of a cast exception. A restructure that drops it compiles and passes every other test."); + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List variables = []; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs new file mode 100644 index 0000000000..edf971e7c2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -0,0 +1,41 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client.Documents.Operations; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence.DataMigration; + +sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMigrationSource +{ + public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); + + public async Task Describe(CancellationToken cancellationToken = default) + { + var settings = lifecycle.Settings; + var build = await lifecycle.DocumentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + return new MigrationSourceDescription( + settings.UseEmbeddedServer, + settings.UseEmbeddedServer ? settings.ServerUrl : settings.ConnectionString, + settings.DatabaseName, + settings.ThroughputDatabaseName, + build.ProductVersion); + } + + public async Task> CountCollections(MigrationSourceDatabase database, CancellationToken cancellationToken = default) + { + var databaseName = database == MigrationSourceDatabase.Primary + ? lifecycle.Settings.DatabaseName + : lifecycle.Settings.ThroughputDatabaseName; + + var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections; + } + + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs new file mode 100644 index 0000000000..d5ecc8694f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -0,0 +1,189 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions.Database; +using Raven.Client.ServerWide.Operations; +using ServiceControl.RavenDB; + +sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings) : IAsyncDisposable +{ + public RavenPersisterSettings Settings => settings; + + public IDocumentStore DocumentStore => documentStore ?? throw new InvalidOperationException($"The migration source is not open. Call {nameof(Open)} first."); + + public async Task Open(CancellationToken cancellationToken = default) + { + if (documentStore is not null) + { + throw new InvalidOperationException("The migration source is already open. Opening it twice would abandon the first server without stopping it."); + } + + try + { + var serverUrl = settings.UseEmbeddedServer ? StartEmbedded() : settings.ConnectionString; + documentStore = Connect(serverUrl); + + if (!settings.UseEmbeddedServer) + { + // Only an external server can be older than the client; an embedded one ships beside it. + await StartupChecks.EnsureServerVersion(documentStore, cancellationToken); + } + + // The persister cannot reach the host's Settings class for the root namespace, so it is spelled + // out here: a customer reads these two keys back out of app.config, not out of code. + await EnsureReadable(settings.DatabaseName, $"ServiceControl/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); + await EnsureReadable(settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}", cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await DisposeAsync(); + throw; + } + catch (Exception) + { + await DisposeAsync(); + throw; + } + } + + public IAsyncDocumentSession OpenSession(string databaseName) => + DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName, NoTracking = true }); + + async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) + { + var record = await DocumentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken); + + if (record is null) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is."); + } + + await LoadDatabase(databaseName, settingKey, cancellationToken); + } + + async Task LoadDatabase(string databaseName, string settingKey, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + return; + } + catch (DatabaseLoadTimeoutException) when (settings.UseEmbeddedServer) + { + // A large embedded database routinely exceeds the load timeout on first open, which + // RavenEmbeddedPersistenceLifecycle already allows for the same way. + await Task.Delay(EmbeddedLoadRetryDelay, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) when (e is not DatabaseLoadTimeoutException) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it.", e); + } + } + } + + string Located() => settings.UseEmbeddedServer + ? $"{settings.ServerUrl} (embedded, data directory '{settings.DatabasePath}', from 'ServiceControl/DBPath')" + : settings.ConnectionString; + + string StartEmbedded() + { + var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode); + + embedded = EmbeddedDatabase.Start(configuration, lifetime); + + return embedded.ServerUrl; + } + + IDocumentStore Connect(string serverUrl) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = [serverUrl], + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }; + + if (!settings.UseEmbeddedServer) + { + store.Certificate = RavenClientCertificate.FindClientCertificate(settings); + } + + store.OnBeforeRequest += RefuseWrite; + + return store.Initialize(); + } + + static void RefuseWrite(object? sender, BeforeRequestEventArgs e) + { + if (IsRead(e.Request.Method, new Uri(e.Url).AbsolutePath)) + { + return; + } + + throw new InvalidOperationException($"The RavenDB migration source is open read-only and refused a {e.Request.Method} to '{e.Url}'. Nothing in a migration may write to the database it is reading."); + } + + static bool IsRead(HttpMethod method, string path) + { + if (method == HttpMethod.Get || method == HttpMethod.Head) + { + // HiLo persists the id range it hands out, so it writes despite being a GET. + return !path.Contains("/hilo/", StringComparison.OrdinalIgnoreCase); + } + + return method == HttpMethod.Post && Array.Exists(ReadOnlyPostPaths, suffix => path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); + } + + public async ValueTask DisposeAsync() + { + documentStore?.Dispose(); + documentStore = null; + + if (embedded is not null) + { + // Stop force-kills only once this token cancels; EmbeddedDatabase sets the graceful wait to an hour. + using var shutdown = new CancellationTokenSource(EmbeddedShutdownTimeout); + await embedded.Stop(shutdown.Token); + embedded.Dispose(); + embedded = null; + } + } + + static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; + static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); + static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); + + IDocumentStore? documentStore; + EmbeddedDatabase? embedded; + readonly SourceLifetime lifetime = new(); + + sealed class SourceLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + + public void StopApplication() + { + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 7dfaeb715c..b15f70cc04 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -1,15 +1,17 @@ -namespace ServiceControl.Persistence.RavenDB +namespace ServiceControl.Persistence.RavenDB { using System; using System.IO; using System.Reflection; using Configuration; using CustomChecks; + using DataMigration; using Microsoft.Extensions.Logging; using Particular.LicensingComponent.Contracts; using ServiceControl.Infrastructure; + using ServiceControl.Persistence.DataMigration; - class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration + class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration, IMigrationSourceFactory { public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; @@ -89,5 +91,8 @@ public IPersistence Create(PersistenceSettings settings) var specificSettings = (RavenPersisterSettings)settings; return new RavenPersistence(specificSettings); } + + public IMigrationSource CreateSource(PersistenceSettings settings) => + new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)settings)); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs new file mode 100644 index 0000000000..e7ce5edc04 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -0,0 +1,346 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Session; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +class ReadOnlySourceLifecycleTests +{ + string databaseName; + IDocumentStore bootstrapStore; + RavenPersisterSettings sourceSettings; + + [SetUp] + public async Task SetUp() + { + var embeddedServer = await SharedEmbeddedServer.GetInstance(); + databaseName = Guid.NewGuid().ToString("n"); + + bootstrapStore = new DocumentStore + { + Urls = [embeddedServer.ServerUrl], + Database = databaseName, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(databaseName))); + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord($"{databaseName}-throughput"))); + + using (var session = bootstrapStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessage { UniqueMessageId = "abc", Status = FailedMessageStatus.Archived }, "FailedMessages/abc"); + await session.SaveChangesAsync(); + } + + sourceSettings = new RavenPersisterSettings + { + DatabaseName = databaseName, + ThroughputDatabaseName = $"{databaseName}-throughput", + ConnectionString = embeddedServer.ServerUrl, + ErrorRetentionPeriod = TimeSpan.FromDays(10) + }; + } + + [TearDown] + public void TearDown() => bootstrapStore?.Dispose(); + + [Test] + public async Task Opening_the_source_creates_no_index() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + + Assert.That(statistics.CountOfIndexes, Is.Zero, "Opening a migration source must not create the fifteen ServiceControl indexes on it."); + } + [Test] + public async Task Opening_the_source_creates_no_database() + { + var absentThroughput = $"{databaseName}-absent"; + sourceSettings.ThroughputDatabaseName = absentThroughput; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentThroughput).And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentThroughput)), Is.Null, "Opening a migration source must not create a database that was missing."); + } + + [Test] + public async Task Opening_the_source_writes_no_database_settings() + { + var before = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings)) + { + await lifecycle.Open(); + } + + var after = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + Assert.That(after, Is.EquivalentTo(before), "Opening a migration source must not rewrite the settings of the database it reads."); + } + + [Test] + public async Task Opening_the_source_configures_no_expiration() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Opening a migration source must not enable RavenDB document expiry against it, or the customer's fallback data is deleted while they are migrating."); + } + + [Test] + public async Task Writing_through_the_source_store_throws() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def"); + + var exception = Assert.ThrowsAsync(async () => await session.SaveChangesAsync()); + + Assert.That(exception.Message, Does.Contain("open read-only")); + await AssertAbsent("FailedMessages/def"); + } + + [Test] + public async Task A_source_session_cannot_even_stage_a_write() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var exception = Assert.ThrowsAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def")); + + Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the OnBeforeStore refusal. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); + } + + [Test] + public async Task Failed_message_status_reads_back_as_stored() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + var loaded = await session.LoadAsync("FailedMessages/abc"); + + Assert.That(loaded.Status, Is.EqualTo(FailedMessageStatus.Archived), "Without SaveEnumsAsIntegers every message status is misread, and it looks like data corruption rather than a missing convention."); + } + + [Test] + public async Task A_failed_open_leaves_no_store_behind() + { + sourceSettings.ThroughputDatabaseName = $"{databaseName}-absent"; + + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + + Assert.ThrowsAsync(async () => await lifecycle.Open()); + + var afterFailure = Assert.Throws(() => _ = lifecycle.DocumentStore); + + Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed Open must dispose what it created. The caller never received a source, so nothing else can, and on the embedded path what leaks is a live RavenDB server process holding the customer's data directory."); + + await lifecycle.DisposeAsync(); + } + [Test] + public async Task The_RavenDB_configuration_opens_a_source_that_describes_itself() + { + var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); + + await using var source = factory.CreateSource(sourceSettings); + await source.Open(); + var description = await source.Describe(); + + Assert.Multiple(() => + { + Assert.That(description.Embedded, Is.False); + Assert.That(description.PrimaryDatabase, Is.EqualTo(databaseName)); + Assert.That(description.ThroughputDatabase, Is.EqualTo($"{databaseName}-throughput")); + Assert.That(description.ServerVersion, Does.StartWith("6.")); + }); + } + + [Test] + public async Task An_unopened_source_refuses_to_describe_itself() + { + var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); + + await using var source = factory.CreateSource(sourceSettings); + + var exception = Assert.ThrowsAsync(async () => await source.Describe()); + + Assert.That(exception.Message, Does.Contain("is not open"), "Separating construction from Open is what lets a host register a source in its container, and it buys a state where nothing is connected yet. That state has to fail loudly rather than return an empty description."); + } + + [Test] + public async Task The_source_counts_the_collections_it_finds() + { + var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); + + await using var source = factory.CreateSource(sourceSettings); + await source.Open(); + var collections = await source.CountCollections(MigrationSourceDatabase.Primary); + + Assert.That(collections["FailedMessages"], Is.EqualTo(1)); + } + + [Test] + public async Task Opening_the_source_twice_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain("already open"), "A second Open would abandon the first store, and on the embedded path a running server process with it."); + } + + [Test] + public async Task An_embedded_open_that_cannot_start_leaves_nothing_behind() + { + sourceSettings.ConnectionString = null; + sourceSettings.DatabasePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("n")); + + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + + Assert.CatchAsync(async () => await lifecycle.Open()); + + var afterFailure = Assert.Throws(() => _ = lifecycle.DocumentStore); + + Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed embedded open must leave no store, or the caller cannot tell an unopened source from a half-open one."); + + await lifecycle.DisposeAsync(); + } + + [Test] + public async Task Generating_an_id_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + + Assert.CatchAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "hilo", Status = FailedMessageStatus.Unresolved })); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo writes an id range to the source before SaveChanges is reached, so a guard that only sees SaveChanges lets it through."); + } + + [Test] + public async Task A_patch_against_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( + new PatchOperation("FailedMessages/abc", null, new PatchRequest { Script = "this.Status = 1;" }))); + + Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch never goes through a session, so it bypasses OnBeforeStore entirely. This is the write style the RavenDB persister itself uses."); + } + + [Test] + public async Task A_bulk_insert_into_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + { + await using var bulk = lifecycle.DocumentStore.BulkInsert(databaseName); + await bulk.StoreAsync(new FailedMessage { UniqueMessageId = "bulk", Status = FailedMessageStatus.Unresolved }, "FailedMessages/bulk"); + }); + + await AssertAbsent("FailedMessages/bulk"); + } + + [Test] + public async Task Reconfiguring_expiry_on_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync( + new ConfigureExpirationOperation(new ExpirationConfiguration { Disabled = false, DeleteFrequencyInSec = 60 }))); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Enabling expiry on the source would start deleting the customer fallback data."); + } + + [Test] + public async Task Deleting_an_untracked_document_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + session.Delete("FailedMessages/abc"); + + Assert.CatchAsync(async () => await session.SaveChangesAsync()); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "Delete by id on an untracked document is deferred, so it never raises OnBeforeDelete."); + } + + [Test] + public async Task Reading_the_source_still_works() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var loaded = await session.LoadAsync("FailedMessages/abc"); + var queried = await session.Query().ToListAsync(); + + Assert.Multiple(() => + { + Assert.That(loaded, Is.Not.Null, "A guard that fails closed still has to let every read the migration needs through."); + Assert.That(queried, Has.Count.EqualTo(1)); + }); + } + + async Task AssertAbsent(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + + Assert.That(found, Is.Null, $"The source refused the write, so '{documentId}' must not be in the database. Asserting the exception alone tests the guard, not the guarantee."); + } + + async Task CountDocuments() + { + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + return statistics.CountOfDocuments; + } + + async Task LoadStatus(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + return found.Status; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs new file mode 100644 index 0000000000..9ba0554964 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +public interface IMigrationSource : IAsyncDisposable +{ + Task Open(CancellationToken cancellationToken = default); + + Task Describe(CancellationToken cancellationToken = default); + + // Collection names as the source reports them, not migration categories + Task> CountCollections(MigrationSourceDatabase database, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs new file mode 100644 index 0000000000..f015402f2c --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +public interface IMigrationSourceFactory +{ + IMigrationSource CreateSource(PersistenceSettings settings); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs new file mode 100644 index 0000000000..de1d54c3a7 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs @@ -0,0 +1,7 @@ +namespace ServiceControl.Persistence.DataMigration; + +public enum MigrationSourceDatabase +{ + Primary, + Throughput +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs new file mode 100644 index 0000000000..c8dcd27f02 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +public sealed record MigrationSourceDescription( + bool Embedded, + string ServerUrl, + string PrimaryDatabase, + string ThroughputDatabase, + string ServerVersion); diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index b7303ff3a8..b851fb956d 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,6 +63,7 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", + "MigrationSourcePersistenceType": "RavenDB", "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs new file mode 100644 index 0000000000..d3c8851ed0 --- /dev/null +++ b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.UnitTests.Hosting; + +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceControl.Hosting.Commands; + +[TestFixture] +class MigrationSourceReportArgumentTests +{ + [Test] + public void The_flag_selects_the_report_command() => + Assert.That(new HostArguments(["--migration-source-report"]).Command, Is.EqualTo(typeof(MigrationSourceReportCommand))); + + [Test] + public void No_flag_still_selects_the_run_command() => + Assert.That(new HostArguments([]).Command, Is.EqualTo(typeof(RunCommand))); + + [Test] + public void The_setup_flag_is_undisturbed() => + Assert.That(new HostArguments(["--setup"]).Command, Is.EqualTo(typeof(SetupCommand))); +} diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index e9a507b944..669218a7af 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -55,6 +55,7 @@ + diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs new file mode 100644 index 0000000000..ee2743bb8b --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -0,0 +1,56 @@ +namespace ServiceControl.Hosting.Commands +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; + + class MigrationSourceReportCommand : AbstractCommand + { + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + await using var source = await PersistenceFactory.OpenMigrationSource(settings, cancellationToken); + + var description = await source.Describe(cancellationToken); + + Console.Out.WriteLine("ServiceControl migration source report"); + Console.Out.WriteLine(); + Console.Out.WriteLine($"Source persistence : {settings.MigrationSourcePersistenceType} ({(description.Embedded ? "embedded" : "external")})"); + Console.Out.WriteLine($"Server URL : {description.ServerUrl}"); + Console.Out.WriteLine($"Server version : {description.ServerVersion}"); + Console.Out.WriteLine($"Primary database : {description.PrimaryDatabase} (from ServiceControl/RavenDB/DatabaseName)"); + Console.Out.WriteLine($"Throughput database : {description.ThroughputDatabase} (from LicensingComponent/RavenDB/ThroughputDatabaseName)"); + + await PrintCollections(source, MigrationSourceDatabase.Primary, description.PrimaryDatabase, cancellationToken); + await PrintCollections(source, MigrationSourceDatabase.Throughput, description.ThroughputDatabase, cancellationToken); + } + + static async Task PrintCollections(IMigrationSource source, MigrationSourceDatabase database, string databaseName, CancellationToken cancellationToken) + { + var counted = new SortedDictionary(StringComparer.Ordinal); + + foreach (var collection in await source.CountCollections(database, cancellationToken)) + { + counted[collection.Key] = collection.Value; + } + + Console.Out.WriteLine(); + Console.Out.WriteLine($"Collections in {databaseName}:"); + + if (counted.Count == 0) + { + Console.Out.WriteLine(" (none)"); + return; + } + + foreach (var collection in counted) + { + Console.Out.WriteLine($" {collection.Key,-42}{collection.Value,12:N0}"); + } + } + } +} diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index d616180539..8bceaf1eb6 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -23,6 +23,20 @@ This mode runs no setup, so combining it with --setup or --setup-and-run is refu Message bodies must be stored somewhere every host can read, so this mode should not be combined with file system body storage unless the path is a shared mount. +MIGRATION SOURCE REPORT + + ServiceControl.exe --migration-source-report + +Reports what a RavenDB to SQL migration would read: whether the old database is embedded or on its +own server, which server, both database names with the setting each came from, and a row count for +every collection. + +An EXTERNAL RavenDB source can be reported on while ServiceControl is running. An EMBEDDED source +cannot: ServiceControl starts its own RavenDB process against that data directory, and a second one +cannot attach to it. Stop the ServiceControl service first, run the report, and start it again. An +instance that does not ship the RavenDB server, such as the container image, cannot report on an +embedded source at all. + SERVICE INSTALL AND UNINSTALL AND CONFIGURATION OPTIONS As of Service Control 1.7 the command line uninstall and install switches have been removed. diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index 88fe15a162..c8c0b694e0 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -62,6 +62,15 @@ public HostArguments(string[] args) } }; + var migrationSourceReportOptions = new OptionSet + { + { + "migration-source-report", + "Report what a migration would read from the old RavenDB database, without changing it", + s => Command = typeof(MigrationSourceReportCommand) + } + }; + try { // Parsed before setup returns, so setup can refuse to provision an ingestion-only worker. @@ -94,6 +103,13 @@ public HostArguments(string[] args) return; } + migrationSourceReportOptions.Parse(args); + + if (Command == typeof(MigrationSourceReportCommand)) + { + return; + } + defaultOptions.Parse(args); } catch (Exception e) diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index c601c07b57..eaa846e5b8 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -185,6 +185,7 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } + public string MigrationSourcePersistenceType => SettingsReader.Read(SettingsRootNamespace, "Migration/SourcePersistenceType", "RavenDB"); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 892920cf18..9d4f255e7f 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -2,13 +2,17 @@ namespace ServiceControl.Persistence { using System; using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence.DataMigration; static class PersistenceFactory { public static IPersistence Create(Settings settings, bool maintenanceMode = false) { - var persistenceConfiguration = CreatePersistenceConfiguration(settings); + var persistenceConfiguration = CreatePersistenceConfiguration(settings.PersistenceType, settings); if (maintenanceMode && !persistenceConfiguration.SupportsMaintenanceMode) { @@ -24,11 +28,41 @@ public static IPersistence Create(Settings settings, bool maintenanceMode = fals return persistence; } - static IPersistenceConfiguration CreatePersistenceConfiguration(Settings settings) + public static IMigrationSource CreateMigrationSource(Settings settings) { + var persistenceType = settings.MigrationSourcePersistenceType; + var persistenceConfiguration = CreatePersistenceConfiguration(persistenceType, settings); + + if (persistenceConfiguration is not IMigrationSourceFactory sourceFactory) + { + throw new Exception($"The '{persistenceType}' persistence cannot be read as a migration source. Set ServiceControl/Migration/SourcePersistenceType to the persistence that holds the data being migrated away from."); + } + + // Not PersisterSpecificSettings: it is null here, and a host that has populated it put the + // target's connection string in it. + return sourceFactory.CreateSource(persistenceConfiguration.CreateSettings(Settings.SettingsRootNamespace)); + } + + public static async Task OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) + { + var source = CreateMigrationSource(settings); + await source.Open(cancellationToken); + + return source; + } + + static IPersistenceConfiguration CreatePersistenceConfiguration(string persistenceType, Settings settings) + { + var persistenceManifest = PersistenceManifestLibrary.Find(persistenceType) + ?? throw new Exception($"There is no persistence named '{persistenceType}'. Available: {string.Join(", ", PersistenceManifestLibrary.PersistenceManifests.Where(m => m.IsSupported).Select(m => m.Name))}."); + + if (persistenceManifest.TypeName is null) + { + throw new Exception($"The '{persistenceManifest.DisplayName}' persistence no longer ships an assembly and cannot be loaded."); + } + try { - var persistenceManifest = PersistenceManifestLibrary.Find(settings.PersistenceType); var assemblyPath = Path.Combine(persistenceManifest.Location, $"{persistenceManifest.AssemblyName}.dll"); var loadContext = settings.AssemblyLoadContextResolver(assemblyPath); var customizationType = Type.GetType(persistenceManifest.TypeName, loadContext.LoadFromAssemblyName, null, true); @@ -37,7 +71,7 @@ static IPersistenceConfiguration CreatePersistenceConfiguration(Settings setting } catch (Exception e) { - throw new Exception($"Could not load persistence customization type {settings.PersistenceType}.", e); + throw new Exception($"Could not load persistence customization type {persistenceType}.", e); } } } diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj index 7afd8f88cf..d9b9958c43 100644 --- a/src/ServiceControl/ServiceControl.csproj +++ b/src/ServiceControl/ServiceControl.csproj @@ -68,6 +68,7 @@ + From 42e91ab83a2ea1f3a77abef0c9efceef0e8b2a6f Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 15 Sep 2026 13:59:10 +0800 Subject: [PATCH 06/28] Add unit tests for migration engine functionality and improve migration source reporting - Implemented MigrationEngineFailurePathTests to validate behavior during write failures and halts. - Created MigrationEngineHaltTests to ensure categories halt correctly on systemic failures. - Added MigrationEngineOptionsTests to verify default and environment variable configurations. - Developed MigrationEngineOrderingTests to check execution order of migration categories. - Introduced MigrationEngineResumeTests to confirm no duplicates or gaps after restarts. - Added MigrationEngineRunCategoriesTests to ensure all categories run in specified order. - Implemented MigrationEngineSkipReasonTests to validate skip reasons and their aggregation. - Enhanced MigrationEngineThrottleTests to verify pause behavior for optional categories. - Updated MigrationSourceReportCommand for improved output and clarity. - Modified Help.txt and HostArguments.cs for better command descriptions. - Refined Settings.cs and PersistenceFactory.cs for clearer migration source configuration. --- README.md | 4 + .../migration-system-design-diagram.png | Bin 0 -> 319535 bytes .../ravendb-to-sql-migration-instructions.md | 54 ++++ .../ravendb-to-sql-migration-overview.md | 34 +-- .../MigrationSourceReportCommandTests.cs | 25 +- .../TwoPersistersInOneProcessTests.cs | 3 +- .../DataMigration/RavenMigrationSource.cs | 47 +++- .../RavenReadOnlySourceLifecycle.cs | 42 +-- .../RavenPersistenceConfiguration.cs | 4 +- .../ReadOnlySourceLifecycleTests.cs | 105 ++++--- .../DataMigration/HaltThreshold.cs | 17 ++ .../IMigrationCheckpointStore.cs | 40 +++ .../DataMigration/IMigrationSource.cs | 19 +- .../DataMigration/IMigrationSourceFactory.cs | 6 +- .../DataMigration/IMigrationTarget.cs | 25 ++ .../DataMigration/MigrationBatch.cs | 27 ++ .../DataMigration/MigrationCategoryIds.cs | 24 ++ .../MigrationCategoryRegistry.cs | 38 +++ .../DataMigration/MigrationEngine.cs | 266 ++++++++++++++++++ .../DataMigration/MigrationEngineOptions.cs | 48 ++++ .../DataMigration/MigrationSettings.cs | 19 ++ .../DataMigration/MigrationSkipReason.cs | 6 + .../DataMigration/MigrationSourceDatabase.cs | 7 - .../MigrationSourceDescription.cs | 16 +- .../Fakes/InMemoryMigrationCheckpointStore.cs | 26 ++ .../Fakes/InMemoryMigrationSource.cs | 85 ++++++ .../Fakes/InMemoryMigrationSourceTests.cs | 78 +++++ .../Fakes/InMemoryMigrationTarget.cs | 85 ++++++ .../Fakes/InMemoryMigrationTargetTests.cs | 80 ++++++ .../Migration/HaltThresholdTests.cs | 50 ++++ .../MigrationCategoryRegistryTests.cs | 110 ++++++++ .../Migration/MigrationContractShapeTests.cs | 99 +++++++ .../MigrationEngineBodyRetryTests.cs | 124 ++++++++ .../MigrationEngineCategorySelectionTests.cs | 64 +++++ .../MigrationEngineCollisionTests.cs | 45 +++ .../Migration/MigrationEngineCopyTests.cs | 64 +++++ .../MigrationEngineFailurePathTests.cs | 163 +++++++++++ .../Migration/MigrationEngineHaltTests.cs | 84 ++++++ .../Migration/MigrationEngineOptionsTests.cs | 80 ++++++ .../Migration/MigrationEngineOrderingTests.cs | 137 +++++++++ .../Migration/MigrationEngineResumeTests.cs | 60 ++++ .../MigrationEngineRunCategoriesTests.cs | 67 +++++ .../MigrationEngineSkipReasonTests.cs | 96 +++++++ .../Migration/MigrationEngineThrottleTests.cs | 62 ++++ .../Commands/MigrationSourceReportCommand.cs | 40 +-- src/ServiceControl/Hosting/Help.txt | 9 +- src/ServiceControl/Hosting/HostArguments.cs | 2 +- .../Infrastructure/Settings/Settings.cs | 3 +- .../Persistence/PersistenceFactory.cs | 6 +- 49 files changed, 2443 insertions(+), 152 deletions(-) create mode 100644 docs/migration/migration-system-design-diagram.png create mode 100644 docs/migration/ravendb-to-sql-migration-instructions.md rename docs/{ => migration}/ravendb-to-sql-migration-overview.md (89%) create mode 100644 src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs delete mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs diff --git a/README.md b/README.md index d9a8edeb65..86493536c0 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ It's also possible to [locally test containers built from PRs in GitHub Containe If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile. +## Migrating from RavenDB to SQL Server or PostgreSQL + +See [Migrating from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md). + ## Secrets Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**. diff --git a/docs/migration/migration-system-design-diagram.png b/docs/migration/migration-system-design-diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..15845976ae747d176e438dfb06641e47bc39df37 GIT binary patch literal 319535 zcmdqIXH*o;w>L^sBrAwyP!SP{N{)kqL;=Y;2@)jdGy@`8a)u!(ISm%Ps*|+gtUZMSXdZhWZz?z!Rfc}ZtyJMDZRtOs*EPOF}-(pPv9u0>w<+v z+WDUYyVs%691H6f7QP$9)^Q6PbXA}~%1GVLUP-;hLN&i)H3mq}1WLpy>D?&a=r&}ZX@!U;= zjMSW}*?36jjf(j}(^KUA)1N{a$!@%Vy(TT4`;d}dizCM+cWsD9O;2(UG^sH!Qqw4<4;-FPOIYibR}wB_%>I|0r(-{N|1iDa3@OiIisH2ty0*mG zf@juRu=52mQlBG_e>poo*xn|MVr5`KF9_Q_`CHtN8k9W~Il_!3KfqwNH#B~syw=uJ z96=q_zo9V+9GX7{+MlJ1r}d0=dEWc*;qPIWxomI2o%r5lR$ERK($l~1SIaBaCH(N~ zz6`q->8ioCZd^3RT8aGczG8QEPR0Pj$-YTH5(7%*gR)0)ayHQHCa1_1X|*_h`M=3a z2U5t(&%i*h4vpKv8q0(%JbEx`ryHEtG0v+(+gtQg=6_Ru^k{rUSWei9zV#d7V{X+~ zzXpkePPSXVpf7v3U!1Qy*~EqZUF3p)$M;_KZg)XTv-pc7DHTj=BFed~{R&oVUjE%K zG_>9c=dN0poG1plw`zDTRK9DlX3uU^=>{p6B$-WGu+1kbE| z8TK!N=4(GV8$U+|{c6L_F=A0%brliz!@&LzQg_pD%~j<1weC1r!{k5Rmg>AoS8g`c z_!o=Bwp>Nnv`SOl-2K^(+6w>GQQzeMCr$rxrCGRF1BX3s*%(+eu6#=nLZ%b{B0N*< zBr6{;%`a~+b)+QAwq`x{{iG!iw?K3I4Ra8nC9SW(hQsUcv8`75oq)H7`BH&UqH+rwWw4(u7Cai!ODhZpE@ZZZB|;lUT}1EmP3?D)xN03RnAfb7bi8 zxPl72vAq_QSA{ImSA3M(K)MgI>zz9!Vg6g%IK34Ovz;ya~U);Gg<(? zl6}3)iWnMAO%nTCL!5OVyPMQl5$Da@)B?aAT`73dm0aHhK2)rdA7nW zt?UdByftchb1Z;ew~V{;)fHGk-l#;9rDZVRuITsZR-X)?5#+t23bpc zd(8>(>wonn|3lKGhBZEOSJBRRaPO?tZ?41pq@ycS*V0JU|#df2~+U$0kbE1w&gdMU4D1DiB;B?PF4e-hTX?r`2MzM4!2{{<$h}wW}31Tu$NR| zPt+Jf+uoIc3#j-i`*8T6DO|t9w#(rr)1h_ofztd~MV!q8V{M6PV}9$!9K9X^5zHK1 zrFGUNC3#s7>)YJ^^?0S7)OST}HvCKL*^Ai|^CNS>{np-{U+4WZ3cc^1G{247+S(6p z1bLwoAGr>N2EVM_3DWo4Yl_Hg@3bR?3q2u?t1B}>uRI029Vqm=|GLblHyP{LPVLSA zY9=C3*aPX58EuSv!~SJ38TQtPDrUg{aLV!Z{MO`pk}PVhv?>4%7~1NjY+BSNW3(?Q zklMC5h~9-w<(}g#Nx8(;W8+kRs6HcuRP}(x+_vEmXhV^gXYM;I zM({iKqjxi<;*=BWv;3Cy9<41sm-C$8-1nxrXVq%C8A=-Oz4Ob0x78}8tUYlZI)RUJ z%>OYt#m~5Gz1=vD8Y{+4IfHlBkirc@1}m3Z*8ybB!^IK8nTNg1Oz6G<5?;V*A6fLo z77f(Hixd<+8NhpCdJC&NyNy1+{Vz%P-$^ZSjmx&m-3@iE;qQ$v`p|+4fWIo`>2#}x>sg4I51x2Zcm34sXk@B zeAaJ!DUJZEeY+__a0Mn%rU`vjiVOdgW;B9kTJc#JP;Bv3t<+@)e%5yUlgqz9_i4s| z%*OB4A_XbD@z${DUXH|7P*y7{BKl?IFXT205ZNJ=>1RqmVu|-5p#H+otR7mV-`2O% zw(`{tw)dOyh@6RpWZ+UKnGkZ{oZtt}rNGWWD->e7WexQb6xeujdb^lrfh68=bxJV!-Plm{y>cx3MhCods7spV>zRE{Sk=2pQ`kWZ=eC8g;|8~Jyl5uKAOAG% zFG1n7NDD+iz3KVN=fD^g&3{E(fov|IVL z5d(kb=!p^_RSyeZ3_hCqHaC@?Jj7Zh3au+N%P_1MsLh@qh7{Rna8qqR&`-MUJK>Zq zC15O8K{4)I{n_DpHEV$KA3NNvIQiz2FtDs&L@QXkGsF8o(KTTPo_`q> zgk5*IPBr0QG=HzJ7Y@dT;d~$7KkF62VyL$>t7sS&?z@cOK&=F+`~(=L}Cc4BEaIl^ICj_n90xuOl(`;@pNViFI7Rql(beINCzujD#O=|5iyt9fp z8vBFc<$$4K(^UVT>A9s|X~@o&+*x+x<5qZT^SkI9o+uA(>f7=~zZVw*uArzl3A>jZ zQe`iXCMrZdzbYBO9Hz6lUmT}%3uh49Ogm@Dl=byYmsmoYA?Ek%v9a*eZl}X;QCLq4 zez|&_Uc;aZh*Rq_rU=CM!Qio>KypcEfA{zbn& z@7}7$o;6RCwTm=Q{DM9}sk;%MwrtyG>Mysw2cP-n#ngF7+;Y`aIgzc&0=t6!`FT&) zpzVJ+GGIt0G4LH-QW^3{Sa+3g#)q z=3hqYt|3Rwfr2pYh^^)iWd~0%e14bm-kL!*XD**gUr5lKwy%5W*pzib(K*D2z<^4U zgj{p^nTt+3;I)f$k0l>)`H_9OD!9gF>y_?!hzkL`r{4fhRQ`1Sw2Q|hUq)av=tv=Y zD<+4$y~W5gW3i`~+qXkv`cng@pPpOyi34lG8FJMi$!z5_u~N9(tHmS#2+yAqvRln< z1tr5j)%y0ukmc9(VBuf~==;8JTVAhhnGRd|;-&1vEl!^2F2#%euHmq@H0bpaKU7KY z{^iPxPGYSu_`sr&DZge=8F!f=(u-_xI0~}y{9HbFMtkhXccH|}$)J#*@w|J4NdVN< z#f`3a0U8${X7&$e*$x=kW%!GMjNROUH_wSab8FZSp$py9b>>MlxUoHbR@{(3z2Fhf zlVSyq@;muMATq1|3@MaaxjC&hJnqu7^B1MMr{CB+o{_6idqGF5emdZd%iF*ucql$? z?8yWjtd(zeCOEWeiXW>$XLl}|h=8dt6#u($%TjWTiBJfcgi*gPGEoG2QjQHNl-{hI zIoj7~L30Y{a8Qpd#)kp)k`%tQmn2Yaha~YFi`&fh>z?{G%V-=22^-j>WNDU$Da)})zc6Wu zjCflaB9t~OCcL1m2U};CW35NXQ$d_oqXHdNKYQv{ZjALSLlt^V@YW$L6(ME3(s?LQ z#V*0`qN5_%qSSPZ_o)du_3Dx&qxWdsFM^HNB!LdUUna49nV9!+MG_iDFv)H|7Z zdf5^KUR-}FmoJHkF7%x?#k3yVX z@N)6n#hf6u;DK&W^Evx*Uh!BXh{S}7$jBQ!HpWybFiBkTA+7VFC0-!NTD*e$Tuqzj z_`_Oy3v$93589Lg-(*v~x}tcp7qYW|nC)d^=e;;Zyd(uU=~iQ=_Wd(R^`DK-p@Yah zdUJmp1{%~#k-}xhO}dj=@6pMOwoc5aq|IiO+M^+}F~X$)-RKYTMo=}XZ%kK0*VZzA zu&}kuPWav6kh(pbA`hMoTV;-o{pkSWDq67nKl6v9$!ckywP{(EbTa>tnOSLdR5yF1>sh4oZfbTABaD9B> zzT>6cf$yfG7DL~a>^;xf`pOYr7=k&=>mN+0Oxul?+-UQ0rOlL?n2h4qXc8zlfiPbS z7YaPqAI?nOreC5QacbkgZz;F<5O^!m7v%{s3msG_2o(nsmM?>8-t`lm{CFWoO}cMH zmRsR8Vyqi7p``FEQGu`3<*6#?&Kgxr&q4XmAs3NYERK9$RFFhA8ZH>o2fbbzs+BCkd6;peqXbx%!5 zz@t{f_JT2$Q$&Zki;kx8CH^ZWlgwz^-_7kI35%`dHWH#rx{dtFh@yt0ngz!|PK%hh zE<<|ddoYDY%f1Y4_MQ7uHyFsLerl0P8x-Sek@RGQq#Gff1bD=~cntIh#CIXlw^}Z? z>z@iJn1Oa{PQ~fcp{Oo3-+KE`S<88*@*xk!K9i>Ehb4fH|HI)%A4!I9>huU3`sg!p z{aM&jsbc29uRTp-g423n6TP@O5)T25XFx?e?m2wpi^@{&Ezn~1f~SMcTatfEF_lFQ?|t~A8K3sG0pnQV zvVRr<6do+mwtRSTGe5b@4aT3}^aDIObn;_;xiDiz3|Yk3qqBNaJE3t3H)VJx0LJ7C z0L9QRlk!0JiVRL5Z?Mw+Wmm%8#ap<#FBIxH*uxnewbBj8PM@m`Rx!38Tx9M!UJzY6 znmClVZ~ly1U#W^Y`SIV*z5(<#YK15_%OlN$as0SFvr|;Ug?!!kiNd`Jn6|NZsyfBt zTGv7JshH$szQE*Fcqc02XVe8EfIg3Wg>%SWI;3S~f?M0MuRsLigXj%q?S11r6z^KL zwv*`@)15W69>R?HZ8w#M9}MVBq3S&zns*OQ7nj?M1sLkDr217M zyZVZZWsJss4HRv?d6Yyn1D-${iav_z`<`FQ(e#S4A|x*Tg&%Lni=kF+#01YI?Dr>= z({2`dlLE=Fmh8XJJMDH}jJ>Eg85q8&N!CVnM6{de)Z%?!+>?sN7v7VQcDeSR+7}Pf zX5$_wEgK}Je^SAL@;)%wyk$qND9RTL4032U(_Y0?+&&}_Mi%pRmoLv3WmP;wfMY@o z1`*v!hjuqS=M26xgfZ|W6~Ov@1{A9ypFcz^Ypu|*dyr_0_@ay8LpCsET3(tHW# zap{T>*xj?4;%A~pyUKpo=jllmFY?L1!W(V)(NO8agk$Du4>s z8aXmC{Aq9evbH4b4v9;NBfskB%#!8k+_tD+=4r$t7(y>nx6Xfxyz}mI1wJ*6bn>}& z11BhE_<3(pmA(T%N~L~k?GNQrF42nHx1wU@#osQ;%a02@8OFPe`=Gz5C%WX=82#rX z55L!upQ^pT27c3&;j5!vnx*An*K)$zG=ZMe%!6bb(ULI>HRG@9@%Z@saQc@tLRDKn z&{nP&t@HZzU)FlGQpJM~n;nR2l^raPd$Ud_JC=mOPWR+CW-vD!e{BZ3nYM&jj@i}( z%JDp%eyn+Qt1rBGk@fw|Gf4#$qB3mVUIu$lt!2blu1*3+QCzh>#GH>l#JFjLZs1aK zp6q6Qx`$OU_37LQ&)8&H!0?}BiO*LAe%ap0w*meVmrtTC7YoXd=4LEu#J$B1%N3v1 zcSANSN_>R8afB+vXYJ=3UXN9=WxhI)&?*&_Fp_N4|GD>mT)W!KPlU4bdDz!aBuL1Y zNqL+61(p|CINS~}2tiS=JXRqh8N&&jz1Xjg@Y$x6$P@{x?Oou_3^-jj9N4FT!3-F_ z`Qy@y6Zlec9DXM*>QoZEPuJUoH4xc8lSIF-`23zw%V~VX>G}5WjUI`{5*K{v6zA|S zw4dr*7#-9CZ;&@{OO!A@Lz@5!P@ru#n!v`0DHCo9!QNIIfErC~GT*_%lkPy}jJk;t z%1>9wjshTUrOT4(YP}^5gLts~DN@qYVI<^7i-|ZNdJX9puIQlhBiKfQXCSu-2xYZ_ ze$u~sYF}#JyJ-}~fPaN-U5j7@+4LN*e1?AedetL#($8&sX^sbBZ?MR(!lV^l{lWh8 znMQefeVeCo_x4m^iLuWB6MQhjvj#OzIeu?x&vUWWN2cx-A3hV^>2Uh{R=|0v`PXX( zle0R=`qagkQ;dVL_m76nEEnV3^mCeK%KhJrLcv$9pe|-5-X+cLnd$@&Rtp;e@q{+E z3lnEYQKRckDhrseUkl=%-sQY?;pkU9UXHB6u*e~B)tLDFy$uf^qH0@Ut+`oGch{4- zR4U8`C7mzo$E|qQ)ohuGm+#-}*JnZUv>mC%CyPg?I7bCusOw7%D3ntalIr4a)%#2h z)@ila4J_Q!3=59jQC~!*TXqI|e^3>~Th;DPIeg5QFTWel8I2)lK0%#Y{7_yU7S@F~piGeq?zVYtd4-08~(8$lTbg;b`(HA^<#GW+r zQ!&xv>}`NU%f7z)v}h!#(g)ITQu77PrFH`zYq%D6zp0x86MLGG2?R&GDYaP8pN<}) zgbsXRxFTak%DQT#c5i~&Ye_ZfW-@v-7cBR_^v3c5i=p{mSl_687isHB)Ekp)MdN=d zupihlMNlnMK5U2V+bk#6gbyFzQ@1Dfk(axVQ>;2o$1J3s7x#!*6 zUHh*~{eOhFH6srM?@8S-x?}Avyx&_o=8z}n|sFof1H1LTm3+VuO;MtRpQO^;Gx8gxHWS2QIk$M zVzXni$?#%R@bXzx?@`_dXApf#Pf(iL zBDCevy6Gb7vXog`b)bT?tDUP>33k}LQkv^2;3hRIZm>f@6)ER3ddYQm81^1xIgj-U7~*dRFoCx$~dK zGXUH43kuZ2XCceu7Hl)+5JA8-QG3tV#z~zE;i!2itJ#ml6Bar(|6D>;ka~}4RsklU!p2#Qg z8?!x2$&Tm*zUk4(ZaCsa@yV&;!xm$Qm%n$CUx=iUfKp# zMs41DNB%-Lj#j!*`+hUlp%QUFc;9>E_cofT&6xGbj$FPE;CHmBomPzp9GG;FjgO(T z8in2CT+A=N;w01_$xd{%^z7Z|5v`7k-*=zj8!_Rx z_mnM{00NdBt8}57gMO9Ft8Pj;UD?`G=?ri-JCI$b`Rr%l5JmsTH-fjR;$=FQ)P8WJ z)npM7gQ!Qg-Ef?(er>>GdU`~f!p2Lx?YRaD&Uv?N7efam$JIA4eawB>gVrt8M(t2b z1b(%!t3V_(3iiN;LcV0DsC14xdG^h*93jh&K)$Xy>l(0zXe%p&eRfSgm-8n!Ctc#2 zxdMyG#r8WCDdz*XgU?gF)xvT2kEoiWlAejZ{c*q`I{LPlXy5)aP18lfMDw#qAenhr z{Ka^V>Qjk?Yz5}(kFE#r|6JwSikKLh!i)KK-(LdcZzc+pRgpW1to@L^4Z{}EW67b` zKbD-y8BL>w9j2#1de)VXA9m{g$Q&v&SkX*|l_|Y;dWjTm(R*y^>iG5c)mq4caxXlx z@H6f}P-+$he4A<7r7OVuLEEu3hSXXvT!tZHV;LM9J^X9Z;Q@>2iy5P3QY3kBi<{kT{_OGf5kyEwHB=%^(?tvHd$ciq?nnun)j9<^{ zU2bYKvV5By58IaW_pPjuDIbN8a7)~ETnyO$6YB#XD~A0J;4rG32R{qHIlPMvicB-N z-T$xn+p>4J)4@Vrf9y~alR#XSY2)c;ojBv31A@?A^bo!DJhSglw^xiq8luxvTexX` z-?je``NU|}am44Bb8OaMur{X;%Z);wqZr#Tc=Z_*FaZ5jI5VLmjPhGDw{Id)e#DKz z=n_w>1GR9xaM3wP)v6$oeznUzE=I7h*+-Nm=_lkKo3%pOzCS|z!fIFaBX}3%U3J5Q zoZOY>>U}tn&VkbEK zhmK;tLB}!jC!_N(yJq73ewKgY+`2*xJmE$r8oA##sHtn${~LRK^eC3SNCJW%xT;@9 z17T-apU{?gE}$MbO_9=f92t^hf^Hr%H7R4z1AD;qJofoPWtr|=82eaTqY`<)Saf|ujA<-N1 zzk=Y_giXDLI641B8UAzO+)e3jq9L{YSD&Gwafycf8siDhiZ)r}}rtu|N^G6FjF7GF6!8oA*)*_7Hjjb*Rod^8eCq`3*&VEpm|gOeEkkQxW2kt)34)@y}gM) zy-kO{4F7MJ{6A+oiS*koG1=Q|ilwSA$ET;zm&Vm)S8Z0^KPZL0PVg+@>HjN-rtgDL zynApwtIDU!&mU~Y`FFvj5py6=W#a4ZehOBI{J%KwOrV@86b8{?+iPi=%C#AZwUj|_Gn;=KKho% z@nzd>Rs`j7xmhYu_`jx0EKZECL|K=W7o_4U#C!7YGai550=D1PkIu17E4^vmrDyN0CtVOEva`A$a#_q%S zHh~5WW}u*Zi}OiI{O=z?o>=`W4K5b=$%Ic<%Cp|*fs7DOBBYv?*BwY~R&dkh|Cl)h zqD&V|GiZT~S={4DbiRa>&FxL#?5|_le zHGF^U>R`y_fj?ZL`1E3$tP&3E@mw(!e$WJ+r@wtVfPY`LrAoJjOH@=-&-M9p7|)c$ zp2Jn!(}DTs(lG7X(KM#dZBM6vpNf?GeE4W$<86^XAM$5*#~WSrKj~`*hi6e^=?ljq zij3iHq%m~?+lGvm#i_qHwxQG-BldV7EhXUWiV;`u41;-`KY4&P`u9x1-mYY=zHg(P zmR{hD9!{#zOwBN@`mSyan~Kc}68jJ=uQP7K#Bi5nv#pH@o~BMZEDTm}T6)mjvW?Uo z;j7IaVX!hwuNem$>9DJD!+lQL@R*r2uX!wn7HbGE1SP2ClxDhaiYPpNiYiGw>mef_ zjTH`QW%O>w3+h1&`4Jn3;4_lWp`X@!+J>Gcuoix8WpI3)$Uumoga$D@!sO!*L z4mktvt8hViSCImNNw{!Pxk$qyy0w?pe55k4))>S}UBtdiuJAW=fsYNztX^TdQqK$9 zo+ZKR0`yeQp25;9TV@MrbWxB8@p31Z{pxw#TR3c@s@V2rQ{?y_| z>@y9#SFwrWU`-^6l%90C>m{NEExc*&J3a|n5f?*{Ji%P>A`>EWmvEa4?NqjgXpT;h z9qjx`4ggHVe*e@xqKU{Fzcb~pd$-@oTIU^u7<%yqqP=oDUEy0&v#o00}%r` zXjqDBW1G@`fkQ{D+`wqkc0ohnAwllkDrk50pk-$9Ofb1S;0khdvNhi85Aln&W z$Db#5ck~oy2o6zL%*9v`*%KLM6wTvyLooz58!m=^(Vq|;$S&7LKD)7o1T(I?6n($<{9&u*u*IyCbA>0XVe8I-IFuFy6pd`Um3?wM;803mi%4`600K5xqv{aJ6>aC+^c~p@xv?7Nh;ToL~X; zK~544L~c8fSf0y;^Je4QF4z)^jdzH>(eBORY1o`Mt#b zr4;*%C8^uAJJ`bw?^#R1OvBczTb*fKPbiIK>rWkvVFcXzd*Nao&9kk~x^>>%q~FeR z=#`xvUv58@#qc5f5QPAb#l@@xL(#E(YtWvhd$r(k#O&ay@!9T7k{0KCwhgLoPLH!} zQ22F%SY0fxJT@QGjS$*KSf^=Uu16pcfpj>C=4cNd4!L5jVI?EoR_PV6v!2Z7bQ@$? zxmeVlXPdtEaod+J3FVQS?O(~+o*l2YGoGDt-Fe&*R;#Yl(kn*Gdhw~$;&CKM{p!nv zX%4IF76qc``gEvAKwlQoHG^$CYt2@iZkwf{HhrJb{4tXBndYDzYXe6# z&CM?P>}Uw{pdfh`|HgRAm#r!L#qF6gPLfWvkeefwytQPw54D$vir!?&`)%@#2X3^-Wt7jJQ*&^PP9S0=k3tJtuZ^T8awZ5sFAB5^UeQk1x|mPcD>o#0EVT+K6&Y<4 z`*^&h-X-1d50Gk9Kj;%7P5DrUSR7lJ9_`BFj~L+$8Y(;!IU+gP%}&D zi#JVwhh?bDZMQn5bEb2N?Ysf!NK5$b8AqVMGT+HFjD29mNnVy9Qz7!03 zL@fDMKY0j#LX#>o-jd&Xbt_#5vL$9wT;RaWSe_xy&dWW<=DpuRCs`KwHTDl`f%Z|@ z4c;~uSlmJ|bs6{MOS?B5kpVN-A#Z*DB%KzBKWy4zRnG7jzQcL?RX7Z*;z*{BS>h?K z#JFPfb_FWk4xf?kiA&$~>sx(V{HQY9xs$qC(NIzsZ_>h?^xD+b2^8F`c4f9f{8`?7 zo5{XiK21E@g(ni%FSQ!<-pUz2W_7t#n7#fq;cCQICx7n)TX&+x_t+iV?za%>^!dJ!^47Myq<{ZV46#+bRJ2@qu&aZvKOl+w z&4?XN>%QDKLWtMdQfEW0hfPQEj4(yhk6Qg9al)oV6zVq%Xae<-nyND-4gPSfG}7-|b1)k&8CB(VOR%@1WMwo}OA^ahzSbL7Kb(u( z#c2sYv)+`rUf#EJ_i%U8zb#q&eCbG6v+YBsWSEsajnq8prjr9C(?|!ol9uYj$>=Vy zmlt=i8Ev)wafx1@s15mWNIa2-`d_Gx04a--p@PVx&EJX!9SIizi}q*SUzdOd4FEdb zw;)<3Wn)G9hft@x3&DJrw1siOj5g7VH|Od}JR9g)TXse`1f=IvgX#EEaJ3Ir6Wx$S zc-BL-V4a>(YP$_&{ zBWz;N&X%inGmuP^c5-7q?!wFRfb_xchY_u$4c6O`WW;q6E=E<%Q^jV%$S)0_8`d0P zMY}TWa3`V+T{<}uE9F*Sx8bv)#y6l2BG`4m%(%_s02@_WAh~ z)8Eb395cx{MJS!+x!c)6`Lja%x_ry5o|y~3fj}p2o|X8;+-djyrhiO)X#Fc5aEWx1 zvUgOIkNdB4sgXCw6InBkqmZmj&cK(=%-@DCwAG%++`n_JYV~Cs-NT>G@NZf>IWu5f zw{12@VbRJaY-uUggXdhMlErR>Ig0hd73SlNzD9pqaz$^4p1CQzlwZ6w-^AzZbKb>oA8&=dd7b%!&II~8=BsE$c)xvMP5 zRvE9UIXiZgC+<3_vGOMR@xXN9?(Aq=k*siRO52hpHv9-`b32^*o2oR;)_f(Q|KqKF ztExoQ-cv5h*LYxm-T%R%u;3 zeR%^vTsajmv|7w3g3K=rn&p^Li%JAnmJcs)l?uDmnqs0NR@kK+e&oyZuieTw`I-9r z^l;Fx3nj6QvR`vre7udJ;aP%gD6b}?qAS|cuFJHKKS}&z2H4BC^{o{Wz`wcd z`g#y@N~rWYDgu$;AiwdOa2KW+8?p91r2XB%n+V2!=KG_!l>{HNpm8LkWRLvDYbzp0 z1)y84(`B+#mFrEMxmrI~4}PhfE~HR-o*le;Kd5uspx>RYeLEP(AjtwK1xb+s(Z-1D zg*}Gd)upgZ!S*?pLC(YCk6T?+EI?H2?a9t1HR9Tt(y5z$yzv5}fjy9n^dEPl;du>? z&B>+_E@nD!9v50e5KMd>w*2U6BY=-JE?C9Ml|jg#rj9K-Ot;vVdY3SoIelV}g9dWC zxL2%0s9OvjZrt9?YyUE31q%}6e6LLQr|4b0M&%d8lO~j}?;s40$XnBf0<8hgn}iU6 zIu2k&AMUS4LgRSbO2fYii10qr+VOhK?|R;+w4#$e-!73%n!x(>w|I`|nqd`hQl7=x zAISKW+d($XY$J{(Jl6?s7Z%!byE^u4^S2<1>~R(HfxG$?z5_7qPx93j?pJT8o)ygY zy@IL@aI#zuK?3LxbJ&LS!T&Y1Pl0)cxS}z1wA}Pz(Vr! zhOy3vTE)a0gXsy-e5&hNV+^IoRPdRzJE?uLqPo$E7RSt!d!t&$gwarqFz7kUy zW%NX&qGCNe8mAU=t3e=+qK(~AnC9b0{=Ks;UqkrS%aqgWkjxN;32d@4#6q(e^sxLf z919_FF!qZ3J#&uLQpM|-4qf~i#Wd~`HZ(XcHBUgxE zeDHLMp+EhuvVO_*B19;@^oSimiVxbZVX1m}Hxx`~z^q0T!R>E4YHor&GnP+3Qv| z8kpXiT^GO>CWwEEAdk>#UiaKxlE7oQyfH~5puPy2!(->$QZMJaFehWn1Iyo@E`VM) z)*8HT0W(ajTVuEvs~@v6Cxkt}y|j?+%H?-SrN$=`U|Bt?G%ca-ulSHfEADLx>NIqV za4ckYik|J9fo$1)dSPY(?2Cx*H2^w{6RHwZzfG}UG3u$PYSOCQFXwx;x$>!4^t15t z5liW>-R7_rJ_oJjlkOfof$#i`On?9mf$h8xeR;9q*l*&5 zdC}i_gtD_ulXjs8M9$|Yt{J$Z2*wY!L|@T@>;}1o4g|%) zppR;vyJ5#b;t){#DDM&^ia(AnoF3qJg)o`AbrNubq9{!6!$AaYBu(^)jL-r5!L;!S z?86^;;H|#F15>C9z<4ixWnGoI-oaraU5_!5=pW-2yQ@f$HIA&DN=?pAfC@@{*~`ro zF;#l)h{48A7xjp$;j>XGSF8>Az^<%TYot_sjr!Cf4g^Xh<^r@%T3L;k*-!m}J5zak z=k{{w3dy_QEbjiXCS6jK)O${6(cF`xj=}obKI}nGV~*FVyoX6B*KP!rr3Kc^EiCNI z7~uN!l!M~6lN#`w0&ub75PDm1bgaRN!y3 zMKkT$=}k8pr4_WxsSu){`N22xjG zx=L)eI3ibeDptdb)f{+uF<(%MI234c((z+t@MSfe*GE=bW1+MUmo;t#NG<5I>T0Y% zWwuTM+iy56K8(1a=;>$u;;PQU6P0%ObG6agni*Kiw&}ykgP3)u<+B5TT-nqQBSDXs zRhp0O-iK2(dD4atm4UwO*1YUE#7n!Q`&ed~zv>rLzy9I{w}1SX9rCe47KV9`pN36S z8rMEF^UcGc5+l-AvMF8%bG2m5c2_d1;ySHP7}3!((=V_^3Fh^iw{?upkXG{AIsRua z&`(=$d`I~waAk#nb*~{J$ZXK=lNzh}SOLMdGNxAZ$w$ztxEO^ny~t2oQ+vN~lPJhc zm#VUp$;c93-JWuJlwFwZGY!+zM#^`78=ht9*UiN0LK7ku{)x7(oD{_y{lRgKG_Z2%^2RU9k*D_UxxQPr zPude{nOc^;EJL8E4}rXO+iU^Py??UcCD8yN7#4a zIieC8_;%|p*BQklL?iR$-m*|*zy=*>voPm(TIes|4{5SqMBxSV%hnPnZp1j&pcn-dK(G$oSt<9EP>MU<08Y>2a1W(kSyf&iMqJXwzw@ zAPrJ3-H_TW7A(!@p~YL#`#4z}!rU-+?XF#OoFb~(GS4gg70_H2uo|Cwk}HUY!Xkdl z`lL#|<-NQbMRfcr9UZ6w-`nNo^jXK9uWI~saX$KF1MajsVsQQ3dzS70;qJYon)Av90? zz2iOK`Mz_`IOmUh#~tJT!5HkFwfCB<%{kYa&wMuL)1BWO&7G3#8*gVg7xUFk9_)=a z=)vzK5MOKQ%eL{{k;7yMt4^2zNGIlWCzx+DJc8}FrNf4c1w*gMz~f?{%WVwjN#Tbn z-9p_)dC=}0f= zdg@4tS7G*3WfT%!-Q_w3T;}HHL?V+%?_ZZD)APl9y=&;e=bYh;8!aux!ljE<&&uamxM54hzxQ>pI0cgyeW#4uU4i1s}A)NPffzt*~6lA2T2UQ3Wx z??;3V*gW{hyYiRB>02?#_4dYSQ867SCycWdRPPJpaPkZ*V`I{2KfP6M+{gm44X*rK z<}aU3=GOtv&~2NUQFi$HaMzO3hY8!E8h~$wM0w{my;~0moB2fO3(-HJy%}r%bY@3k zGr&u`?Pytb{fme6Xr0YCR~o`Z-Sy^wMJ zz}l#M8%QL^C6@as|@Ub~8)KE12|`|Ba%v~lF@7(;rt zuVM(zI@}I^BGK(+l>B^MbRGm0!=ZzR6T2fTUobv6iD2CX4!eju`y~?@SOg2(1FejN zcTIfy?`qN(w&g2<58LDH5(O$=aCtKKvMcuZQle^YalB4%3*NpLr%`<%>{vg?biUV7 zX3}wm~hN&GokPWy-_X8=d#Ir0g}aL0FR6MlZQtZU2ONz^Gw8 zdia8ds>H}^jo0;}rO|n=Et*?>m3V~d3VqCVt-`R@>fMib(XvNdlPy42B4pTWb4*z= z=YZqIi~fo^T=5@v-wB8{fJD;mG*_#`ZGEpGT$&ACVMH9;tMAI5Vr792<8=%hL#f=4 zelu~l|52jmXlmHKCWYVDf-QncYP|ZXC0)hFEQy}W+8Fx&1e5wN8u#~(_e+c$ zbyy@EULsqLNDLd?6)DlMMmqsV|Q>ru~j(xdd zEf?AKMU`AxU^KEZ<^>US0$r^=`{O1IAQo){ zS*pZ&tM4)|y0;TZqyl0h)linlQw& zx+PSUsF2}RQZ`!=A=O91#$BZi!4*JMF_FS3PMcY-y!0j@97!O*>b|oeGjY>CN-#aI z-L1g(r{wl*Z7z7}v^u2rvFGPp#o#QU(+n+305@<8-aY*DYUuX7E!0QZrEq?-F_4C+ zUV5_Wmzp*s8@S#Iv!U6}As*aUt}vA8f(41wv$f3fsjK5u#wB=ZaDQ!3^K|z)H3$DW zCm>p5C+9UE0_0louKev(m2SezR5GIu*Ek8pHN9%{aAg|Bk!CM+#yLE6uGYF$A|}ez zj#zPY_sX}|xh=CfiGUaTrG)G z`!w?d`h!6!IfTfRRY*Fw2OWvuDC`2>`MNi+Q^5lTV7W^ksclvm=(aO>o6aI0& z^dGV_R0a*F@?j5`q(|?LYP{*K6sz(jE}-USfZJ%UT>l76v6eSxfI)n1Qd`>VnBS}w zUlc?*fUi?-EfzA8Sd5&ILsN<6ss}ym`>luTc z+=y=xG;7su^Yd71*D#SZNmgCIX;gn|pwDliHdkl+d$zb$g78t+_jiPazmooC-A4<9 z3oD}8d17-07Z}M0P&Lk#J6&RYUf}{y)1cXg-_4h~4F2jFv9n>JUu+I94%EZm#Pn%p zU#YPoz}U23gZ?(x`(T|ypZ|>FL!VY`%T{@mn$|;uDtXLI=O-$PIF*BFvkzt$t z{n9^*S~2=A58r(MBq5_+)pF52u}cIDdVaoW*SuP+RWF!(!8}+U;!du6ZN5p%NUFMtU zBDz&6gCV$Nh!)!+=XXwQdqeNHNL!-JP2^oWFCnf^eZK;nCL_PV4 z>+|l({@V5;dkhhFk@{E<#ETwE4P9DP;CaAK^b`hCQQmddXOb`P2<3U?W209eywnkE zyZfR)X|@#r{@ZL)1laW&cz0}-`I?%*V|{ThjQT3I3DHJAruCgI%VroM4#Uk9{i9huMTHb)oj;86EfhIG=YyB2~EEMMJY6e!V3aL8HB-Znj z-Q5hQy1wNlf?xmsUjWah_41e)GRtl&zf(|Ri%hExD3#cMxz)d}s$#!NNq*PI_C16pCkFTc z{^;88VgrlPSZ#;Pi&<`Xktq`XhnkojGn7ql0q}LMkeKsoHkTT`-QmHZ%Xs%K`)I$S`!rFK(AJrt|e*oy`TPvQ*g?<;j6g)Ojs4NPP=`V{nxs-;{6i0Z@|PZwEMD#7@yPp zwS}W<0FzVzPq?v>Zq?gc2phTMOkGx%lzKrOk{?ty>nHYo3KRWVT&gcpsbg~?9~{QX zek1RH++(@_Ny;_X3kjjJ<=R}d?1_!HJE?@E;XW+%}j7S=19{K>-oQ{p<7TYhKA- z(r@E@*5A#`fC-*1F*fV{7Ob#Id=8)_b4m(LyypM(sG zrnzbIFm&G4nEGaUScejDrspUKPB(AZJ-9(%;o!gZbkflWGvf4I)C9VgF9Wq6basM} zx|=Xn?UujwK4MY)R3+j3^f5V{-)=rzL^HQNp6rnw65@8l%ruj@mNz*${%XSXo{N^+ z_*tH{=)872&IoxsXkezhFs59Y)ku>O7*xFc3OW*6|Tz6YElIYo6DhH0&piL z6VKpDv1q6BSD=X)Wyhy6OyuNn!;W16>(Pluc$Nmg5prZ|ik zfx>ynms!<1`1&>@X}CNICqxoX{((nDY65`gF+QV1QlIXePPMrN0t)Q}a;F_XG0uwb z=a!EWUKvRLocV#KDu09dUW$jadG|da{#9ls*}@un0kIU=PA1ft5aYF>U7u4;6E}2zZk`}# zaqgM8&-^6R8fnly8XZ54_yO$ZY3*X&%<^CXtQ!krv zy$&%s+4R)1Yqx#m3_`@93)J9^$UbSMA;U{`^n!!PD0!jiq!}i7Uxa%%wzmqKy$a&M zTZ>FK4=kjm#sFlNSf(eF`on=Cdolhwp&k%`k>v0c<`LRt$%dmo^$DX4z%ag@4>9d~ zXoIehXpIB9n%JpaWI(fk(j;Nk+6eyy<|QxQM_6pw?pNg5dW6PImdGe& zN%j&}buf~Q&~jr#7xa1YlN>h)-V$`0J#(0Ez$=RuGFx$8`0fINjlfdgfM>M5e0;P6 zG5!3uJI`#noR}rIE&|i)@&K7-BAqf@NTbozvsmn0o42u=g4kB&+0ssaI#A9OXC5@t z9{XoAM{c9M;w-&;-8|o*%5*(PVlFrnkn3d~=VC$#LmZ>^`}u()PIBHNm=ROE&XY%W z2&rB|7g^Pj*|+v7Z`BfQ0L&|-eJ|4aNUHkxQemF&s6|WH+;7z4oT8A0uIo%(nZqcM zLaljT0{;kXD;R{Ou1%SX#ViJ|9wCo7PX&gcw$w6G-O%(~%<5dVTJTk>r4=$o{*Ox?v9^_diGi^7HxV2K}L z&k^aH4?M;my+Z{Lu4ZNeb!WPqe6r3j#sBF?Agkn%exz6S<KRAI$yF9;s4JDOKTfICgB za^_~!aCrCiUhL$^$v&&U(@@ntv|OOl!>zj%i)di z5(tGbCr-hsJU^dZ_Z5(EHTr7a3>dFQGf)iKKpw|>A`5NgXvjoi-7j%QBW5t!rJhk&K=%ndoT(>FfTdgPM?CNkkf$C`&zc z3uymiKfLnRj=W=*zv*2WCDDWzJ1{NN8DG5Y7AV2r92e(O&(Wd!B+kcrY3R}D6{7=l zrAacCNV&f&sO>B)@*j!ry2wiZy%FTlweFmsUj4}Ni&wFKI|3}u`%1=vqkf=mlsLLa zQ3z4^K^9hVjGvx9&r=dMfqLz8DK~Voh;7+18Lk-*(e!}0=bz!{R7aWDc7YL7QJ$A) zW;jKq{sY@`v`kP)$goqIOBD0g`NRIEXWD4)>gHhl=n6z{@<`^0-96!-fmOe}cn6X- z9n<}*OrZ!6>~86k*hk-0xPxx{RTru@3Jh3WR`$F>zk1Z&txseGCR?wt?34c;z#`Q`L7wa3(dwmeoabzS zViW@p0(T1YfTQ!PKJiFc93o`HKc3tsgkn}c(7sO;YIgSf^dOjB(?DLhWdi65BEFG% zEol8Xhx5yZ9`7mS$a?fhP|23-vDl4GMdmBNMglM#_eN%U!9If$ZWa{;w>}V)1ygLN z)^WMp$oGn~<wNG8wOM`gs!oqY|J~lI;ojEJ$+=|vUemxw3O%%gK3gNr5F%Ky2r(L zN7mkf(iG3*O4Cz?cUozbG>Z!xzcNX*)!|CA7E);cfMDwK{U#hYSQZ`f$5~z;>d=mt zUGh>m@#=seQ~kB(!C0c9l;{9V^#UhNG2)nhNaHB5)fBP*gHlbnp0H_TwCm*6mKLzM zXtNMr;o-z;dWH!JT@KOGlD4nFf5XhKU+(?sn(@}g4^C6>`0RXXTp#u{gE=fII1S?< zoWl^enMrT0t7-}tf?IElIF@s2MZD%hP2kfD(^WjqzN73LAxNKWQy!#DT>gG8+rl*M|>py=mm%5>Wb6RM?oiuX*77P0bVG zfR*F9dQStLf7C-6KkQ~*O1(?{h~!|hJbW-Y@EQ%o7to-Eq7fAxsstiAI81Q3V;55= zhwxC}(E$ExG_Na+UUaLhIic1m_ldxny?{`msghMRV#gFr^gd)~>XT?vNO$%-5$i*eFnE#1fb?ac(#42Oxy(05n{Xh zPV$(XmCM+f_7K$J#qq}t38!GGtsF$7)Y|0&w)N8nvyVA!N8F4vnY55m%Hbih%Z|o! zd(+;U*cdT1BMCePEmoxPp^Wr?)Rc zinCy;$+jn7Dq@Gq?HbJzB2kA`+Gn)DqMzt`&ZZ{5x_yPUKngf=dY_rgufpxB&pn4; zT*KCE%rX0}^+Dgehb!9Qk{2KRHS?OgnMgCUFMyY-q^qXYD-oeCkYrAG)<6GjFtcSDRRBeOQXnMk;{M~TH zS>?x)&*Z4Z7n_q{z-RX=OF}y}!45-{YI%`zl422yGA&;c-=K7qme@!;7f(>JzHNSW zW=h)God?ohVNH6m%$(gBzHuvx0k=AG(->U7f+A%KTUgL`EBD!EZ(wC&YtvQ`p7Qssa96cbFmxMl!QB#S zs0LXWetz`=5n=aV_N&s5hh_zvv72X6sI7WINl?upd<$L7F%hoq)Et_r^cH=EpC`FT-vmu zpJ#9pu?2@B^KC^EZUA$zHp>k5{y0kO!z&6o(5K!$;2bbmXhnNCJCg2BsZ)XJ#|g_r zFC@+AX%+>NbR|gC1fXp#x{f(rz)UN9S6YRquxkdFJ;BOl0jobLX`RZyH(#o2aat8S ztg@r(?N|v%M;qCW>`n+J>EJ@dPt$%;wVl0I%fb)-65z*QRSbJ~;>2OS6ZEM(8)_-{ zbLv_PZk@zh<3<0DzuHA~Kpugdy|2;R&pRHm9{s`blI`SiM2N2wKO3JtRg9ELFF0x^z=4t{y*Iv}>NJ)yF(Df#lkVF8MtA^?RGO%7;}B zrb{du|-XTTj!l2D#s4r)|2@t3wwD!Z+0 z#?|}-oXwn-FMC4iRXQSLF$yR2Q$&}K*MIL8&*9-?Uiqn1Wf7E2|LaQ0Dlv&quF&(i zmh%nn!;Vnl3%9&JxOgX6&`iKWi5SJQ?`JY4Lb=G`J<(Lold1YHEuwo82cevs^ctV( zhnc)e5fTuqdkoYpz?BUKJG08WFa9{=eSJV7dm3qIyVoDG{3I=K$t9YMnsmKh%viAr`4%6$ zRVYhe+%xN8bgv64ZZ1&6kfnRekUYNO9orupUWB9fJyaZ>{6YuW2qb~ntck6i=$-tp zg&)*t#Qe{NuLFN7Z##jLxUa9_Wj3J*SspBWwElH$dFXsn>Zrsy}3@N)BmJhkOE^w-{Xm7pDi z9f=21Ijc3@x|is}DI|7miPj8R%!}o|McUO07|d(1y_=lQn^fb$qO>cSkljwTLUNI< zJIvVmixXj84zDCLYe`)J2cw|Q?{DpJ{lR?F_|7TCsgbkLlk$ch8j$zf3i0aE*15xS zi>t4bmLG}qYTAB;3G>0`JW}Z+Yp^iKUBPTC`3!{2=Ld3?2ALn}I`&;q+nP4NPk=oo zGVVpP?qiYq60t%?@$GW63|8{O4Q}&J=VjlY-&^rDJqHDN276bxvM%NTrjSiCBZxGr z4+8Q@kw~RG(lu~-xQExt^t9d8h%pW`%8lD8HT2spoK&LuW%)ifrs@hEimgZl_VTlX z+?Cr#r~ABxJJ;To1or{5u0#6B68pN3$tW!>oBZ3al`SV@e`vRL3p6FEJ~l~*Kqs0M zG)+L^C*tJslhm6_iPx`W{#G+-jL&H7qq!`58+pli#xY&QK9bQ8`>9Dw(+tA5)^B;G zlH_!ZXDX#3$W$-6-qK3jc$1!GJ_t!ZTuy!tdX)t-yEzqrM5->{!uVUfc1}SWdYJDOL^I zaC?V$x7p-oGL;saYR((^nvrsT^LJ7EIrqq!@l+n!{%fp-w=^@FqbgsJNIEvwJby#< z0X@eY@4@2Y?Q+fz4?DN99_Dc9`z}9Re+e$uh3ZGjbH}XA9X-)kKAs8@pWo$G$w@vl zY1$c-8=Gh=)j~?=1w&bsP`MHVn5VDjM#SIPXPMq;dZtJ}i zjhB(T?_;T{)@tpmU|K{4y1mnUWpq^oJevGdAF@lveqPHTQS1?DxL1J5$}(cS>-RRo}R?)lbE`vuVReck*dl;{6jh?+0&o@t>*#i}n6SHgM4c*N6e}*@+~X z8vIug7xR8R&hFr#5KniZ#YL+;H(>|)2A9wFpx-@xU`joj4?W?MSScWSgr1z-fbJ>n z(A5uQH3a&_?{eXJQFdKtOE^py4HH5tvhz9-DSim$O&TSG+!y~n78$9YV0pl}gu_+L zFu5-5uK$Rx#-K#Lmc7ZeczQYw3(=+M{QWN82;Gbm{R2d(XLJuh`a;xT9Z|rf0Z-g; zoKt{9mdD1+lVLTipf7If#P{zp<9W-08E;01Ijk!jJ7hL!>Nr;$8)G-;DM5@!dd3sB z_;9?F`FqqziWYFy{>RfM=!j{?27S@j-qCnVoh5`l5Iy-^JF|Z)cTjpHoA6;^TY;*= zDmmBz8F~ZOz90SO7~&U~LESt(DL?dF`K6X!IgmpKWJP^Txi2m+yxvvHDPPH-58Hb9 z{ZGA7IQ=8OG+TmP%s~~ofh5ZztYQ)ds%J|Gs;Ym+Y5-#+ucgeFv;na+wf-h}hv)IN!F zD<>;dNax7!eBt)MD7xonGIAyf4Y9JaXa1+Yzg5p77f0Kv`C3Bu0R0?Wl1fUR%}T=S zp8SGS@#_b=hq88NL_pc71vu%F<=%zDC6p*ui zNzy-+E&Is4^Qg{_a=sCvH@ABjg|N`wKq`((<2jW@oZ(pW8=%w*IDU8(20B?58t zOX3rfK|b+2ZAUyvmlWg9SX-%u4(7Z;X^0uQW*l&*?@}_`iH2G~e$O4$**cPUc|{DZ z_>|saO??svFvbnrq8}?Ub2T{?jmw>{LzA)IC*G7WJtt1 zCWnm5@)>Adr?w_;d)25L4nK|#KQeI3eJ82u`E$}&QOi5NJI*#0A&cEGRr08zTf|B1 zc!qxSNW>PK3cZyPCZ@rySu+vFa@%VA{2nT&+EHz5|1xi-G<(@6RH!TU+5-VUV^Wme zy}v7%2bk%VQfn<>~V{@>I0$ix=12-T8g!aZ5oT!_w|KLTm#!y@A8nti0adZQb~G+3I4D zsbJ40aiLTWFVX%W?%&#e6194Vb30vYiZ3#>Bq_@!qqB`i)TJHM`urZYHKQ7>f#`OG zt$n{Xm1pW`lEE34GE@M7e%^5BNYk{5A}rXBn!_ODfJvMxul#t}9x*COt}lXxGJWT4 zO(0Z#Yc#bhKi_X8@MAC4iLH#F5Ua;GjdpDRvo=cc$mk%&)z{bKvifo!U}jsS65o;Q zbg8+og;4pAOv)3yqnBF6)|u^xLY%8(X+J_aWn|v!#$6~T4!{}2n8kQ72_+k%^SL>6 zQ&c}JZw>es`C3@_l<@kio(Q58Yd&04I3|T5>0a2&tmz5r_V)`P-1jQOJR`axXXIk% z)0ra{py=7~!J<4VwO$IW;##7&v=M9pA>TA==S7dx7M5IQ>lzP3?`n`=avX zqbb0ti!846T{*%|WW6zyh&zi||cGdp;{ELmS% zSnfy$HO0^3KDb@g`f-K0$2;xTf;+#pcRnF!?cS4caHh({+-6yREJMxafBCXmP}vS& zM)u@7y@D;rt8|y|!8$t~Wo>sr6IGEhiQ*NG0!Y`#pKYiCbMuTd3CL9%!%%0EZD?CT zkFj6#B*m_aI-y5sA=c_h%e9I?zpOR-*3LmiP5qvajo-}eiVE#xR zr@*gwUDPv3d`&IEMkgi)p)cRAhzG?VpRp#$zS|L?OiCZF)kJ;s0-gO6!cymypnhoaqlux zeKQzLAUP>0jm`{Sv0N8D&$2(>avJd~qkJ!1L z()xj7vjI=7j;P?9RcUivVDCl?G<`I**q~J1rP-8EUTAn}SCs4%1o#_!sS+ux($rcz zXWoOg0hQL?(e2tMo)Fv;4ahCtIe);pJ1Y3?xMvL{9O?-HZ06Ng6*Vo1p#oL|XZG^p zIGtk7N=IW@R?ICEBV31GgwD^T2hEn>WWMy?PeUrjm)ED)23b3O#I=sWhEsw2~6YbkiS^OuCi*V7)<|od&NTR zUTTW{lpZV|%WyDz?{vCWWr(-ruuZ72e&{QpwDz6y_!Gb}FAjK(cEOwJVj-k$(bM~r z$jvc=N{=9;EK8#tQol*)2jQ+^VcSR zi|@SckW5X@nTx1lsanonx2QFf)>Ux^e9;JbC>-!v>&%t@1&Jb%S^7d1SA5{hsv<9a zTr*#IpMyHlb1?$S_vskcKKyBXJeJ65{Ro`h-E|4f$dUWR<8_x6^x?+!kbM@Bze@5{ zmA64uP&WPq$xkl*dzqia#}UZfJ-ctsw0J86lyM7nM#e&Q(rLKM%DMN{T5kECPq;d3 z{(b9{qz@V#^=y>=i&`TIPX>UzYXP>O+DDBz(Fr zmFcTcOy(dFVhG0vu0O`re{>6ot1YzP383xV<>AT1A==+lv*SqLFhG0HyQ88xQW^n8 z+GeNPq2mvUlzNgJH;b^#Gu243Oh^{AE}XhEKdqbpXv`z(Ru!j#pB1v7f60ZdY`F}O zA*z^FyS8NW!-z2%bYVkfmI4EE81CPo=T(3*c|j`e+<2d>n?DPDG*cOyr{?p~k@;w$ z&Ag_~i<^Hs^p`<%DNN<{X* zLhcn3m;OgrH%XDei~EIf9_%suZla9|a|i*cJkv>q#ZpbsX>bN1B5`*PuEx_{>&4#O zqe_fNY_Pm99n`_MSZm_=ZZ+7AgrjZ!RYu;S0Ow;Xsv+iw*Tew9P}m=gdJ_S%bIYQC zJ@ig^6L@af$3-mS7=KRQa1Ea4xA3pcXnttgMILcia}jqxoIUac`1-YqCEB&Sso7cj z1B-(?+j!I;!rnUO`cIe^&;{7t?NY3Sz`X%37p&D`tK2&tbazMi4`wgf0b0Ap2 z2VPVSf|(#VfIHy%Jp8YyMEBVa5BbUL4f}=sx#T`!A0p$+l3${}9tm4df7JoMc8LFJ zyPV2Bfxe1zQKI7D!wd7Flr)bwm6v$?;RwCK6MuE$oxMycKyAF(6(XhCSYK18UxS%&U^&4b^fb#G^K^I~Rh>BM0sAevu*-YVL8g&)TU)(BXP0xzF8Ej=GF9G$p^7?)o|Xj*2~q=1$z$ zg>+4^A-(~7VzY)0!qWmW(gwC@^7_}tBSrOxfgqxj#0UYXF+TiO{R}e= zgXh}#kq5m9Ol#w;y*v{rB}$v^kK`a^HI@P%*Rm!z*XCx@Xw1t;QWPSeG!wOmhqEFm zmZHJY_M2F0a}sKQSeV-__M;<3MBpCm(2VEsy?K?aDIjvQ6HN!vGf zgc~{gz2N7gg?G*u?R8rStE|nezAC+{FGA>bXr^i1d}AUwbJfHS>7#We^~VM#*d#A~ z<8MEA=0FFfM_G5rDR0$TKe>`4--D;;usl&cw#v-$a3M=NuHm;Jx({W?Jvb2Ydka@( z45bnilhZwxZR~=b-uBwT!3(<^UQ&@O+7Ooo3lflLSf9T;PyJCg6KdNBCinT#(zZxP z2Z7$0`_s*GOol9i%Icab?Q|K7W>#O2S19QgtIJs&-(4}w0>ZteR2bP`7vmkrzg-ON zz}^vSz9tVR>jny3COqtcF0&l8Tb_XWjd#@@;*eycygvjS8Re@7q7ctpt?D?~OD`zu2 zki@qlmp1Txnp38}W@|wr& z*$_#nmvFjo9TP%*G#_K^wq==CH3W0#mZ5Y*hB+QnZmWb;8wzI`3lrH7>X) zQ?Sj{{X z&l^hs74B?|WT=M>M;e9Jr~hOd@=d+*eJl0;G=I0R)@jxXkq-5&kq8}4d4uo(3E=iF z1W+yPM#pPbnf*f^VMmNpvO+}f{TXiJXSHr$sra9eZ91p7Rj!y&aw6ya1~j_5UQV*~ zr@^lqj%;DOeOhx>33;ZiNQ_>fLGG)+cwl5Go7c)|K)GPhEiWvr*>v}s=h`3uFxAY4 zIk{n1wf*{GxpRg$Yj>!`fvl%Y)y!Gro-uApZ2VJ%K45B~pK!L@Q8_5HHU5)EA>hbz zUY)a!_C zo^<@8so!iR@aIU)WSM?DzaXpePhahX@T=tB`SPC_yq^)1Ql2m~F$|h-qC5%<;~X?Nt)*LtQH)HVT=^(s_Hg0x^YzuTojWpJUk)>xeA#lt;uaXesY6(-;RT91{QMDDSsl6pgs;Raj8FZv2g- z(15d%?=5ZBPmV&~>D=$UUS&8MVhjT=FN?W5t}xww*2|IbMd{}*`>|L+ws&S=|Cp`rR7^j@`% z$fo>xpwT=uf5hlg#x2s?ar zW?&<@HCJX`YC;7*#QYL|JWn3@p7yv)A(~OBhN1mB^2PISXPRJ8ZveL0R8Sqs=#h-& zkc&KX56^upAWKIV(inoK5N>vF9b?YL7V`3oDc+42W6>!cd*qJ3o#k{E^9L5(Vh zH{*6VKBYF=4Mcmew@peQP5ZNy*R%9ymwrEU)6Y%(zT?=jV}V8Rd0yzeZ}>WkpjsW= zEM+<#dmo`F9?MslUldxDFJZEB13Rjq5+S;MS;e$u;#!2eP+^u_51BW~8%U>d(nRoD ze)gG6EgdsqlB`qGtH2@e3>_2HCqmby^OzMge&K2CHpfc+Le`N}IIvyY^%L+~-kIE+ z`$$nGg$zkN&G+ApP1*)>1{a?BJHK61)`F%Ken=L%*r+@K36qXJU*H(01 zn4xC9q|&CwCi)YH`eyr1jE$`+IMTzZNQV31bzumg97e)G;WQCjV#y$Y6H1_*xnPZs za#nk;hdF=sCifUmWjp(o{I_!mImPzrz3vFAjl8Pq%1i}om%y9PX|bGa&#PG!dDs_V zmnZKYx3jZOgrB`Q`0>iZ=T7SeO~_YL+ovj*&OEb zGa~ItYp~}7GcuT0mDtVLyAhon3v<8KMxFnBH0Vn}r|wtp&H0oC<4Cds4=D#ozvjIT znEPVq&Kq^G_}h?}6dk?P4rSd9QvWV-Kl_Kbe-N85sBX05s?>F}y-8k$jUU!nGe;uYFl2E<6}>n!uV_YgJc7a!KW@6; z-+itW=*z*VdhzGVd2!W^mWTHqm)#@%_>s#mlJN^;Di{^KL6_-kUeINrDK9K8-jMn= z>PyI+L$PXEVrH5{H25LY!?GI>f8MJK2?-t9-r25ctZ8FaKLQoNqG6o zmtU=wG&B^#oGs)0ZV$Weyx4HXZTFm` z+|Eqcl^>aCVr|2_f<3x#I-m5RLywy2ye{;bX4PD`TauD#j4o}*gR9w6pO>+-lRBXN ze(67NsRA?J`fBM@)Lu)yBX@P=EVlRFoGP0Z1NzSLRR8&s>&5kLSszAy4k630dj_d` z-$YLZN(y|I?v?wrzHBN0>h9Nl8N1tUzD0p(9h_~9#>_z4`Iu$20#`TgAFrl9F?cW& z{Os^e`Ma9Njd>9{lP6{GP6Ln*(x;1d&;NYo9er4Sq;WN3Bn<55pPBgolZHw<#f_ndT!jcLykQ34|Zb&X=bTaq8|?hUjh&JbkTKj%5wE2_XCaFf++n~ zAyvn#%*ImSJ3>64rf$OOkx~ypDc^&Ne9Fi6+1}o-a51U*q8Tr^Gtz3H1pl6DU0j+M ze37AzXEbDG1mAo`xBF-|tj7kmy)(#WwlJKsK1y5J+(XI8TSpred?_5PQad}iUW zu%v<&r-`H+i^*MP*mtWnBb#+zNxN^$Vo35@k%Cqu8*ftGo}T*30-O;e^fIFMH(lV| zYA31DJoP6lzQZ}f5xvAZGQRnFR%&_RAHr*>D znteQzbbe**LjQLw`V6iB&Y0N6{%7g0tvCdCD*C%}&q2iZetu=Nm*-a!HP`=Ryb z2M-Q`!6&#w2sXI8>j1;RZGPvRE6;l0=dSw)tTjE|J-c_;uCD#9Pwh^xY7ud9J++wd zpnD_JP1|vMG|X;FOfE6^UhSTbO;FJf)HQqH)W+*2Mtnq={E6702-=sTf|o4DlB8nN zm?+atS6w0|GJAn^xUAFi@RXQ{bd&)XHhts2cURH~e5_9#A@FTyxHK8)T5|FD#2_Gm|s%odd(ktS9oyok`Is0;z zd!TuH^vM_0?FQN;)G*jk&U`$V;bLqQ@#F(@-gmpNsT}4*v`yzew$%;(LO7XYDTgb2 z-+*qMCu<0kfT7OIF*3(bzZR>i#5L}CqrIsrLKg}EkGUhyOw2nXv zhte(>n=JCvT%}QnU?;yXUZ#m*-{TD%iBUF{ec$5m}^bsK2^uh|_0T(_cv0V*r1`t1AAYw_M| zzw-7vA~yEbx0m}Q7zxLi9m|Gk(unK|GuBztRX+KqK@<2y?NrCXTXSxoPm|a9ZcsHc zeqY|l(W6~%WCAo;rMJ=xSmm!VwR~Q^Wwuu6>$T`%6A4itEc~+jS=YK`7xW@b1+)~g z$i`w#jb$vDqk_8(_?xSmWS)zt*&UTH%_JmawC=&xc2fO^pO?8SEL; z#T!>>@aMfS&Lf!TFtnTY3S^TQVkcIppJ9#4VTq2PIGdz*t~hD-u2d%n`LU7n*Q2BO z@Up>4z)BmOd&r213C3uAbaV??C`M*jmESK+z~$n?V!dk(6=lw7 zHthMSa*geCa_DTLkULf1*0G|gI=MBM0Ex%p)%g7EM(SN!>H}b%EJ4f-quXbpm1YL{ z)74&PcY!OJhoJokJ~nw5n)QJNUqqUYg9N{4#m85RJw`lUFY!df$*Vo6yRDA_S0b6; zcsJ^orgN+Sa?*VA;b68~_8MM8b>69`Ng+YH!5vuLJXho6Wm zjPlWUwlB0c4Dj*5oxI!;i-N;{GRjHAkGVNBWqN|^xWsQIAd$CFOmyF=MCH}UYZob# zy+MXh#_B(B$6;lwrMK^+$jW?@sG1<(kuA?{j5t z+=hT|RA4CMfip(@2V%XVjk_8?@6Ej7chGqf`D_yqgx*^W5n|j0hPnG3)q(2<_L^r} zQmvgwJ#VCI20ZW$T3Lf+LX_9fqi4kWj?VQQLBJ#Xz$#CYbvA7LYg&dQk`y<88oohP zH$y36KizP=$uaZMZDW9JPw2Kbj|LWGD^~aRqAGv9tXf?9=hWHgUiNsiMGeOUZZ|NZ4BSbdq|*v#MxQuw&N}P} zP|khcj`vra&?SA z5Mgcf+}%?If4JN6iNH0_Tzvu%uSo+4WL+~jC> zCUCLm`hebMbCwg#5C|aKpE2^}iIQGSET1=Aocy&8#6Lp`i^?^O!I$d~Mk&ozb{QAj zSQaFOds*4=?wtre(0*lddFYTMzXVI>2!U!slgH*hIZ1%hMYuPom6;P9TI=MSMcSAU z^6A#2oTnb+i5CsA>DWe=pTRxn+PclHJ=X8(LklNB)krY0)E- zc=R$g)YHZ9o?awfYGPY}ra?Y=?BfBKnWzFSDa0DDFD3!;WPP2+7h)qWute41N_f6E z0;9Zc&|%@<9UL3}22>fbDdHpa-DH($)1Fnn9$qEULMUCX6Fr9y?w$#)q_Czx|=vSsx|K{EK zSh7caPK7j0;GJ#XsfRalZ6e33difp*!3K7_PXLK|fc#g?z`ZRQOBQ4Uk$ckp!Im#$ z{!yW`hI(fXtT$t+OgTCpt{}reNPvQrDqe2$)ocHDL!@~eF}Eesv>TX>U`}?(osz`L zG3eu9m2yOHC1>oqa~}55(mO7vE6j4EdL)1Qg`qUV7LUN>GujviE6ETk%LmVw+eyn8A$y=Lx3rXLnequI2^{>OV!6z(q4}09Tu`vb3 z-epClM(-W*qd%ohd`TXE)e7n~<--r)A=PWtoH9RSR-4gFoNvw%dF_4Tv-#=u*fKB& z8WU6`uSR#J(u#Q*ul})7sd&t{Tc7BR2c_<3A&wlyFKW@8kw^t^2dJb2+%xBmZ_xj` zAeC9)DMel%HjQcL$s56H&v|3veEQ-lpxf_}D&Y6)Wstg8E~iog9WBGyQ(_woG}<-P zavvg1j}}llW3#_*I#l!ylc+>LFX|?KU8CUqP}JYmR4~!6Q>vD9#;3ZF)G;un)S4hW zp1dyTwHA?^ZN1vP9q>v-1Zx+4jFy17dTkklNefnJ9{O$gd6)Hd{$&-xT)|Dgi|b0_!ZvP)`2fb! zix$dep+JZd2T1&xaA-^q&QHZM-TWKmlca*n`AcA^Gu+{WS^(@!0KRHSH8L23i=>d*?CT=9D5)xmP z>l#jWd-!~}o)07Q*a`}ZytOwD%Y1zbJ-;UrL`Px@<`%)anGG>)xl}XqsLN&h3l5;y zfZU<%TH}0N#zUI>kc2&!r3+{bVj-+s9BKpW0zi7<~1I>V#g z9Hrb4u>UsAjSi)y7U~pDPN;G)ykqak6DrS@zTy*y^|~!^XuyUFs&;E}1Hz1`eZN5* z%8-)rw$a@?p}^q!(j`>`nifHX9+f?nDpWOm`xl<-Pbb5H$$e2#4|=tkdAHyN}!5_GUYsm z{0;VL-bur|3 zrP#Q9tM;?p7eXX;@wlweCd>u_e1n2P8^r@I{_LknZ@|-3A|kfWVzW{olPVv* z9k>0BK{n_0=If5?X&%R9NwnN1dV!#655APDwGR#kghWJZ;if*NgNc5O)Y{W_tCJM{ z%u|s`s_W8Z4SD{)oJ>JnjWc!VFAy6Uu(-zh(jQ@sRk74AqOw5eUEwU0RH)y5G3Bn)>tPBNUln zDgdr!@kd>|A5|AwPi)sEcc%t(>v=^_qyjyw&*D&LDGzDjKGV}ticm8G;#bu=JR2rL zwi`wt7vLCcu}sJ%uWV;g@{J8bryMG}&a}$7!UC$)LO$AKxDdW#G;rJ!Q%A1;QjzE$ z4>-ydcgz^S2CNqT5YzL~rxa};s4H4xf4|W=jkrysZ+K?$K}Jog)C(4mX^ON*dc^d- z<>rSCr$C1YixwJdrRHL41SN&RHGAe4`{;AWwHC`H>%ml{e(@Ni80^)(H1V(z zf~zuiLqXfR!T5e>*UIF+nRV77E{vUF+RmsG`dYHqH(=!tF0&f{acpT2Yc;1 zrdX38-zqO!rFXv_m5dkZ%mo$Y@2(dbgKX&{C(kN8WQOq*10pabPTPW*JN!EOV2rYCr7$NiVw(`+q@UYD|#s>~)7G*V>rFiIyVX!R`C*F9I`Whu?eB<{()s-ouTGq_}?_zu6It|3Fs=Tds}FX-uY_ za(LtPuw|ci`&}Y4?pycYg{O-0O+sA`KX0wO-zrnm+D{am4dt~SwkzVyNV)(o)^O8p z`^=XTdyK?-db??=f50UQIDj2~ZO6K8-7Jq{GKh8j9!ZkQ6>dv4J3od@LX84TgHx-V z?N~HwehZd+QS8e1-fs8hYf%;YnAEuK-klc&%o&+qXpy$wII!tLp-;Au&W;Ud*ti}3 zysiEcc!sB`MssP?A=4cMtig~K8oK5dJz9})Z zl~o~Zu^j`WPo;dgGXG6Y4vI65L)Vr{MvdLjOe5&pCb|%&i%PT+7YQ$v2|+{0toP-z zpZXlNUwH39la$(MlKGQS8;iOu@Y@;wbHnl1C?4ad0R|s7CGeNYHv;VmwtkO&h3l+Z z5XC)^{NN*rqh8zI7wR9K5exfSZ0O71+Zr_N2)sju*<~@YYif$B(T&)~euL(Cl=}L% zZ3OD}yE~FdPa=wJ9Ca)8%JjVGIkdrwBxaGQN)u;LPOam~8NS7ljN0D)8*R=p=CT^! zh^npSp9_(KDXw@o)xi~w;`<+FYYLwV4ev*$hMTv@Qc)2nN9{LI`f)Y3lqqmY{XA0Y z!9faKpsJuY>Mlf|-r?~o7_Hg~$IuM%0b&Wn;BFbZ#v6o{uu!<4pbO+e=;4{)(sX7pp^C^F}LW zE_%Mz@)*OgPBl?=`G`dcx>b$MYZapB5E0seKg{1QsV%TRjli@=MzW07l1G@6@H;%~ z*vhJr7t{%!7Lzqkd&5uOeQ4sgF4xiBAt2}`5}K14fH}qUdrqX^QG%D?B=YcBH0SmTdc4z27>P%k$B`s#K2DUZi`>nK`!mJ-;tt(H9Co)rl9oGbY1eluykOHDNG1e-_?nvBA9{ev9l5!hTG# zNZoPH#?@0ts3V?VX1gYPizEnO|s{ zOjLb(#tdc^yodqD@{0qO(_q%pJ7*@H7TjM{T*zJeOFbQjkf!{B z5HWk>EDu@nQ@!3Qy@Fd*?<8wcX>NXn9{$l%++W+h^VD=$?hGB>?k1wy#p<)`^czii z;p2*Gu4lB|0m)O(Pmfv>)_2qE?VnH$2;j^%b@CVK0R zS-A<^i>X-1bzGjC4{jrWsY$cRL&364Oe^U@ggwTQuRu^1YbF_rjF67w%^IpIbCDZg4A~u;4%=WF%Kr7^0b+@csMqyI(;X&^X5Y=dR7 ztm8t&D^9?{wtyy~t1=w*%#GuVLNb%>;#YJ-g8{XI*3qf9NYZ|RI+mO37wV#TG+aDeVW$p{wNZFiOSK@D}ioJnG z1{8!g@fo&dnufIW6KWcNt?B57()O&}9~0(2OL5eYXwV1Np7?gfc@m4(a4IJ!@`?w+ z8%UU`z!}`+5^&vgQd5{Laks!8@50^_oWWnP->!LpghXi6p+ik)`ViU|zu1T+8_l(e zw;~N;?H`!Arxkoqr!n0|b8vCo_V=)PeX6t3;UGVNcF-!@*@imfp5{RoVqBpq-{#wV z=cZG;{0h%tP!CM6kUZ8v&Ct$I-nC%4@cJ7ErQBeaRBj9tw-F?SqKPu|Vkz@g!pK9d zOz$Q6U#&0xDCMhr<2b6%wHAbTJ>pS(To1ZTb{j8clpG<-c(# zx$UF7WoyTc+AyQdHv;J32iHYJ*9ZaSe82hwGLoU3Yz(G0htaq`TBW_W(8u|M3(&@l z-8Xv#yhD4mFJ=1v<1&Ssm5E#t;7G#p8OI7Ft+pd2?E+o%-^by3IU<)fz1_dMpM(C* z2W*ix5s)e_UV&YR=Gu!9z*RQ95}LQ-$zp5v1W>W8+q$z_23zDMoQP*-0+nqp&M{7~ zR9%~@b2w*ub6g8(9VTgOo zh`DQv%w}9=d3Nn~cj}LS|2$#RUNgpvib8)G!>++1a3yb4Cx3}GC`3g|no(eGAe0V0 zBF`u;jXTAEw=T4Cg9mZ_)j9yAio9TZI%`&?$?@J-mlooSCQPUJ(6fRt7~!LY?&7{} z5@XoA!Nn0A(gURDC~|&y)OxK?a`J8)?~#+XXW2!^*u;+m+Gt(I2-XYVt=}?i^6|e3 zxqRDcGgYWcMz>9_IXutJ@A?e?MCPhZw20a1~GO^)tC z%C_{Zsgb8Iu#k=SJFA5z=t(5D$)N8IiImGSACMdRTPY%F+}4U!hNYjYd3K()-AKPx3mqiucM zUvZ@$LJ^YTx3+8{_A#bFoKCMYZj&{Vv<)v=e8N-b$mbxD zjORwp1}&n(nIl+BWn%t3oq!N5fVK21Rz{`|kWrr!lM3q?2mqPp0W-L(xCpH$@wvv# zIO(b~9?59$$3a?gV-2@u&x1do&c?R~$}U=8hsA&X&?nZZbNK9n)RFZT@jan8HDXIxvgNTE+Pdr1>SbZ>v8v$ zHhB3xEx-JQ0X8MM9Vh|NVkK#d<=74TX1^6g6Y6MW!`T9)oSU-b<-9jJWcF(gJ_Q0RUxZFs!Raeh> z?a;w(wAz!Ln#U3;@Vg4hJT}cqMjY)Oe~B#rn)qfL#l@{b!RQL0i4kHnzFLY5==hO82TqQ8)-VuBu9i5mSa()D}q={e5iOjZ`>{Z$@Xib*_q=#rt-}T z8#qnNkH((O)IB61tl!o2c|t4%Ge~+60kMTZvf$f3XB5VSoln%E3HMb}*%6t8Kp|Y_ z+r9Xle?wn^8ZlnkJ-j=$b&E;o#x_8|K{)L(r^y*jX7U4vT(Rq0to383Mbz&r9aOmv zT=T`Ap1zcXFsJOpQYbjr5^gEfk8So7%)BK`JVe7pJev6JIj}TxTzER&_VdT-N_+q} zKm@p(nJ6N~DLVonO97ubgq8HYyxNPA)A(8cNqa@=(b`nf`Klg!$NQ4>_hauwA0M#Q zKz=HB@*7W}zlZ^AIkz0YtWL0t-mGHGU)R%BW5ObeLN6Km^yc4kv=tynD;Pg|(Wt1< zm$&x9n|xFm1r`~rmVf!8$K98My}mogqzX36M{OtZL}$NAe!R@{)Ji7xP*&j^*MOkQ zgMA(I>`Lwgk1GG+@t(HsI}4dY;pqnUG|$>!qHGzu9YuOrmR3ne9vW7w8wXKqKl=Jd z+89S!>UxG!^0N$l*o;qvHCZfhkNBg9d#TRWTIN#OQIHni9E>nKs=!~iv=0Zi%7`&J zkT(jbFJWu<6WEzFl^Yv=LAB#F@AHwsJj&b&9xv<78+V`ylPIcdHaoM9D?5_XdwS%= z&mYLGbElhh<|mgPEjCPTb(*8D$SqV3+JE=y#Hx=tZazJ~U#1yy z81NNp9x;xH*`3GB&^){2g5h>Qflok*A3_DoWXnDjfHYc4x(-@$NJZ3rqOqfNQD3m26*p@f_Q zgv5j@=S7@lKctN+J-w*SXBzBbqMXf-vh0hX_ZN0m%QTWXSOvd%QK{qp>Q=7f^CHzh3L-K8(KlF&wq8wq!;xnflOJ9Kp^G$#Ngx~nr1A#KTZ*nCqy9)xy;e)#tm^2E z&Iv@m|60A>mL-^a7At5_g+rC?X)&%r_u%|A$L}^=umziLFM855N@2zlxDVB6N4p|@ z^x}|Ky1i_=(!;{Dfq;aE6`Gek?xOhO;e9n5`D>uIgF^!|m45F`F*xp0>>T}QB@N5Z z#ng`05PD7UO{1x!=x<$6yv9I{+}?LdTK6O$QEBssgqzJm@M6`P7n#Mnkkw?pfk&&b z4HeG4BhqpJJX)|tY8J{HlJ134U^3%Hp-@=u;V5;qIM3ARFz9lv2)xd~U_i4;4?nr^ zkE0a|YzKFJUZ?d(pG=?@IyTi_Y&Oa@Z!6?h940{=T@X3|4}ok>42H?182J$(0UF7_W9-U*5>> z8xzId`j+}iv1lS=Mj{)4Nyg&SF)bJhARH0>}S#50$XmP2zzqj|Kr|c+`KO>aW@H}y)0mgibj7K z2?U=)K=8>U-yP!|8!CTb!VMV8Jc1>y7*M3&{UlV05+?yW9)T?>M06s;V^}nkE@*xP zCi7p)SREW@?~~0 zO>!t8ePhvczo%P0W}R|-Q~P2zfPv<;?jv2d^KKc6f?z=Ok@c$r=hC_5L&dil{j=|3O-b8e0O@$Qcm z3}3yle01yMB6+gwa85FW(EX{_Hbq5DT18!O7|sAG+UJxE|N7X@f@ZXY?iLaYJjAr! zwe)Z_h{}B#&DT6lbi;X`(eUZ_Oxa;UFG)k4c(zT~hM7;!f8y9xqi(qSW?0(bvbI>i zNe!PVeKu9UEO|e~i{Nrg(CHl*%Yi31Jbcd8V!WXEgVC_INi_Jx*$-iz-kM-RVtTX{ znccQv$}w7^45&8!ruOf+v^ZUC7n>_oZ%1kJH$E;7Q7_r^?F$Zpx|fq}_cw`L@pZ25 zRWwlDAU_0Ahda6han^bY^jRGiIghV4wPa=}+}l09wxm7hADcXtC}xO$-Bq9L*`!a@ zgTqW{AkX)fAirMEB0%{$cK4Uxr8~2_nKz<*@Amm|KH9xD9`#B3I!yjaz9s%!p9;l_ zr46(3yDvOSH~b55WL&U&G2*zDLp`>CQit+dAd;ON2U^JgtPp# zH$WZkr3ldJKPsd-l;uxivLn|fZMK-S4bsh3z^e8g@n*|8G^t6Xm}+?J+EzKNF~nQo z1eDX2-@j&~p<^iy6j3u&>W6Ly1wG5`ISjZN_AiW~pK4#;TnT9xYyAU&vz`5ayoW)p z2kP%_N?XU2*#c})iHEbk{%*pmzN^@LIn$+YL#e=U!CG$ElIx%|gV`ulM5x@nrQR{^ zz)*C>yy#CvJbiA#1GPl8EdpRbJn+AY^T0YHgf4V=qy#Sg(f#xoEwtHuos}c z$y>Howwh_%t732UaSa6r9jPS)_l%FbA#M)HYT0s~XCImt55J2Uq@rw|$(27}!Mqnw z2u_qsh|43i(GEBwV93o7#R|Ow{oYa`Ke-w`j{iK3y56h3L0Q24%icIWI+a{sfQa+8 zR&ND8%VkmN;1UD}9ox8-iomaDDEgzVNj*wpu;#0ol##%U0NPcgnV2kLzyWMFDZIz* znBQ;ub#orc^m!4N6RuAolL0|nSu;Eck^vw;rDjx zuHwiU0>E`V2Ow{%kJNASTm?+h{#5nfn~^lM!BA>@NRtP;ZL^1SIHjgN=tNns8HVO) zP(ml>yhex6&FbNRh>~gGt9F3c`7m)jHf@klsIh3NllEXVlh%o(f)j;&crHo99Et z!X_@F9^cHdN=cP>z@*wYZdJD;F$e>GiwBHTXPh1!si*&sgmm`8zk%n+DekeASqA_- zdw6W^aL0@+1BfrxV8uUdtbN+p0x%g}|C7wd=KTEJjkLE&?!YP%WxFmp{uG$Ej?un; zr}x%FEXU%Fo7Fm)p*k$r38bw1AhuqY=7oR;d4mcq0L@5L;bG;P<|-j#3!)*7QU?C8lAiJC0}r zs6hICv%&567*_EHI<;H?(E9B1)Tm@>`5%lt63`HUS$$DhIH0BYv1!!AhHy}5ufb({ z=&MUgvOw9^4@;nvxi&WL&jjRo;{M32dTHjI2BJ$DLe0bGLxcb9Lk>Y z!v{M@=kxkp_T#)q#63Xm$32-+UdHQxh}J@#WqdfB%9&A3D)SGi*@N6gHd=s|l{vN^ zD!Lr6w5UEhSJ3q^Uz?q-P6kys+rP)yjPZD6+3Bm9?xAulQhI^ewMoM1wGTn(v$d2y zH_eb(yXOYozEhhz4pq>;FaH9rZ(I00zZc!__s00KaeGr^8U2jL)u?$5v~MVKB6~g_ zPl?a1cwwL$N=!6_u=|=4Hdy?DxU${R?VXVTtfs$2jo*((Cm{doz;XiR*I{_W5o2{p zzazg_BUtqYduH)L_Wy^67C1`b|Fq}+MkhdN(vIX`EU{Kz%wh&KEocr>aeEgXp<)`L zdw5e8oppCL?%M)>t=ZDX?50F%@Z$!G$0F*5NaW_*OUvrGrZ7fB`%M6#%(MxpFT=C;nSWf>&Yr zhUp*`_35!id`I1{n@4yUP)Ux)5G?@G&t`r|%g^8nF0#`tIwHAcGtUDi|DG`?%^iu+ zvq`@R$*=vIam_8hc(@>Xbt{(z^d>mc{vLBv`B3~DmVV|2(ZU;}^}v|sn xqAB4 zcBlX>K}(boHR-p?h4n0&b9~zaYhmCKL4Ub=?;0aCbH$M%5X~M56b;lg_sKPz(0pup zekl-jfu$U-Lba}4n=9XL6uUIk`Q@pTQQU>m`Nu@Y$05{7e7if~)BfKp9~y=KmZri| z&r65y4%wvg+K${0#8WGx=K8*DWqMSq1Zz9!2aG zU4O62EVTp?)L+V4#I{)8D9=Iiplh`1F#1Y`K?9iEV6gyy`kyK;=aIFYwNVO`U<*>E z**lfBb*;Mp3m2`VvCh?7S*rY!lS>~4uRhJVstK!~a22mw22f~PSK=8kZ*$w^Hm*&b z$AN``P82?fC+rc8mLe%xk36+J;|uNZ|ElTFN&KpnN*{l)q4&`0e6mK9n{@W!Kd3pU z?y`+Imd3$!@7~#@Q8;vMhZZm69_7CmLo^s)uXFuH@LM2QN}Czm+udedo`cF4IUTrZ z8eUT3a+|d({HdCW#xUfj5*eh~5UA62RVLwH&4cnwZMR%< zBryE*Sfb>Q=9)=z>}r+RMWFl9RA(f& zfYv(k>WS%u?_sy`g**QZy~Nh01YlN*rV$l)Pk%BG0DTcN-$xuvY{>grm>?f|Cx*f1 z8bnRvD-uh99rHawB6DOu`S*{`tvZfu003V0B_;&$RE|xKt)`C67yQ*Qwg%<+-?U$f zxvIu#K(-6^{B7WttsjbbM1yE>Z)f;2aIlmc`7^n9U!NXLKnK&F5;sUwW;S&|`|H<# zZSwGQ3A@tIlL2z*)(rqiJ4{>LyAu)A;~M_am>Ga$p$&@TyX6h;9`& z7JJSNiJz?hV!az>6Iu+e(rySRAR#AN$}&Wbz_S4wR-k37eg|0y}Kyj*VZsEQz6NnLpOH!7Xg z)4H&jZn6CdlrE(PyE$ZZP+8gQSMndWy&!Ag;|QwA+yWkGchC#U1(=KbD#z!4ap-%a z$*0g&XcO-F-eDhF)h0s;TG`O{eEo(8^9L3y5P5AFsvKi=b%CES%|&}GkQrZ{%TT^P_c)t=S_Ew_kv^DQY{&J;mUA#iJ}R1IR|lzsGsvye zp`0(?M|pEsF=yxoZ{6W5ep{u!|%0HXBkM6BJ*G;Q@m@3~(1rYgJoD+A4-VAvK=84ja zGX6W@*m9N!VTCiggn4`BJ!(f2qw9gm;NmU@^hg zz|9d9!cN(nX|#~;0!e1mbOePc9P`D#YC=7-j5v{D=)L=?G!|g43C$sbrTf$jtYSO{ zAm?toK*l1%2H4{Zc}(-!yyJ`(eayqj3t~Jlkkr{8$zsiyYJ{#F+v_i8fwP#ef@?dn z#F}gee;=|bdaCG+@-ircA$c5;!c?TOVt+Sz+5>?4A+3MBuo0|hlNB$c;Fj<9mgtKu zF$}mW*C+GM<`gb2Ixy1cD2d767lw?og}oLRMm(SXak=p&aQpMd^8{PDdc@?4m+!op ztIzJ(ajm6*09wfr<9UsB?FpKsMXBZt1wH(^#*Gp2k#)q5BY%4ixaf#U(Mbg;k6E~E zY{=S(?aO{KJ=W6(Bvfe|)A-g>ta<8*39giNTFtI^Sak=+s|)N4|&=g z*t|wQAg|Mm2?2BLS$r`?FgNc}w7s3B{KL6z0gKk}!-?y?W43<7);}sm(t8z8p!H`@ z4h+ZjQ$mL28XO&;80zZ+OP`U>IrTzpN3Sq5) z$hcPnui7TZOlpIhNKKboDfPi|D1(F)b71t-P#)zW*j`LbG}Z}>Q=y;viKalJ@>8wVO{?li1ymK2xMHyq2_$b*EwFb`r&RpXOsYx1I7 z6h;j!v;FBrpDo|@>>X2`Fr|lru$jrBdLMLMX3n*b8pN7R(pRyePO6`xW5lj){^Z<0 zX}OYqeX36rD79fYEGu4ax;<$1zo6Hr5w(uoyKHA};&}x>So@sLx`_zW&u)+c*)p&1 za4l_YXb>xUmYq+u`H^y^#t+#xY*^Kw0nkhM-2od1vsPd>Vw;YB-Jy`)P;Ab*MZ0jN z%t0d;v>sbp6&(nxBRqI8^gF=J8y=PNcc;-r(!{;-v$2uXD`2KjH|`#jFqZarCmYRC zmJ{Eow&&1-A^-mW=0_34n@)Z3ya>2K*+avu*UEkKCSB75HU+{G_lJZ?j4u(=Cbj$+ za{2dDPR125rv`GrXmq!dUcUsKWOS2IYWgkCZ{&t(`61?D!%>Kb42wI{ zF|C9M7A`f)Zwpb))Z>Qu?V)5vW{dfH>eKVhCs6kli)eMC>>HzJJjoevKUfSRD)fXz?~ubjImngeF9ZLPW8bs?1tDj%bqx&OejD%4sF} z=`%!{e<9ZQ(4#v_&_|^5S2%oA5`|Z4v@Cpn>`~#WKMLHLX z3-7MX$s!Aqm*7>vKtBkIn!l75Bso&+xbxs2T%=Q~VIt=QxJ6#>P@!S7nQ07d=Uk>u z^6vHrni%WO;AX5efj8*G<*PdfNKIZe&N1B83>e=U=&^DfT=t{Pu} zC?PT^*lXY53;%>Y?Kc4J=7Lt?&G?=2Dg9S^{NDc+l#7`<{PgAejOO2UdJQ+oPU?ib z)@)aCtQa^q5*@${B?4CSL&pybB_y^}2`vY&ZIO^vvoc?ewTFEu%6?bay!8S`UT`-5 zf5PBD`!G~!>K`HnotSZ*XR=S8A_@NPndLJv3X3K+5$a;V)5`32@!%T?YbPvU|;EI$r3OjV=yNLt^Yc-;tM1WO)$0FI6n$%J{#d6uj#yxRtqGmf<>k^R{cg}nf!aSn z9Q=ck1xdTw?MZE&bMEJH;>uBWvkVvHfTj9Cqj{W<^J^IPMYhZDa|PnkrDH}buhgt; zR*JI3%E_!(tT|RTD*XqRsRb|IVJ3~hEk3Vc+6THS1_<(LIB>=<13)Ce$wH@e+ar_+ z<_FX?@1)yM5}Bx1n><{_D%NPxe!o z5tLRoil8C67|$Q__F7jSNtxF#=a&{^U6I_U*F202yYoB**2^TC0@5`-6gF8j*4kHc zSdikT(Es3{$s!%x+Kw)6{)hJMKtpqy-`>VpGA{P+?780>?Ic2d>Q)ADJymy3|CgpRiQY@R96qAH=^<*dwrfdT9&}e~1X;@Y;oe#M z-H0qhe2mx)qw4GWN^)8D?5}1P4CbSTCeTfjo+ix?xAeEq=G`y=Vz&7RWlzxHs=ewg zK9})cnX(0yH7Pk9@#Jd_#(UExvuD;E#*a{V@0k|CiqZ7M(YOU+x0kC6Eo=_8R zQL?3IRM*dD8OV)3SC#WE6O&YsArQ#mUGQl&m8!m`|A0F0E@p})F-yQ6PspC)og@W4Ast-K6R;$|2XLar${L<_iPuk%RyVeJ)hzov(t8~8r{EPw-wCZnh=3zTsI8>wP^i_xFhbFDh zAQl7Az@y&S!;mM>Q*#g)gV=lv%ij6O&}bI2EvE0sqw|V5>ZmeNM}xW2%d_{Mu)z99 z`1oM1?^7bSS72frLP;<5-|5(v83OOX?%MqW%@MZXbJP6=@`_C>$OrwJIRF*~GR?(0Y8aRW3%ic-GUXsvBr6fNR#)bv^$PcO{?juoX%32n&`(adW#A3QRI ztuXiTMkOea=6YQfSOcw$m ze~Jm<%y*C%6{N4OUr3mYNxHYMRIOj(-~Qrw>mGxtx@aP`AMsL@_OBZ?ah!yBWYzL~ zAb7^1HwG8BUPbG3-NuWL*2h?uP);oyz4PQ;$*0IvpZNVt zl89G-OF}U8FeA<}^Z28t+5rdfq*bfkmW-Tf#YY_%XBlTQG*pO5a1t zNI|m_OKQ2~0HkyeBk&0!5z0?;2~0E-NJr_s5CcxI)BF&5#A^vPO-H5H61vUTCAm_2 zV_!O!QEcM9L~wC?_Q~=PGD9*HO3h)u=_rO%5Kkg52epWAl0TQt6c1SCT4K;c=*`3I zAmM2*|49Byy0gJJicozxeRHopxU#WFZyY(H_MYO+hAMJ|;>9N`$Gve}lvWA=Rkgd2 zzbFsD8($@_wi~iO5Z3zsO$(rV!5&{H>ag!z_@8@ud7Q#wBGosG4vpj|ba47jI?@cF zz*4PGPY+nh;9opD$kvio zs{T9&N_**(Ir-zA6UbuYHR2I0JCB9xEu15sAg}v%$z;-uL2`p%1Va^KcjJ9slE=?k zi^bKJhl##`CFG|^*a|hGL*vLI8Dv@Cxnpu{4nEStRU6Yg(@RwSO))OGNF^Mv#Ae7U zRk;^Nho#1&K_&Vkzpo^M zt`y83m_fq6g{gxQF37tqwyqai5@>^Dl~ji-lWJN&Bi%$&m#PyaeSB*t92Hs)B%)Rcl*1hid+@A%CPHxb7SZ24F4W$?Wz`=p77y+j2(zeF<9WrWI z!}+A03yl&t1`WA;18*<8MivQ$rqvT2JGU}7{|VJhLmS*!xEnSm*jw&JU2-@}EWU^_ z)nkmMnFX4-NJeR2&`q`w$;LgF7*py&euy&WNa|Y1&*=}y(p;2d#L1+NjPfgOl#|K- zCrEU$vfT<~f+ljhtja+eM_d=SOnKj8a5b%4U~Y}9bNTT(m^&di-{T&W9K4r5)7V+W zrTWnoEfs9kzOf^u`ZH6qOx)vjEHw);$W?#lC`5<5v7vcA#<^GIO|>c zGgeZ~4nM+*<-v8K#i@1sXNV;fZK&cnD;<(rCou{C6OG5&h6KyQ-Gvo zFUB|krNxjs@hnrhw2iPXf|(hA&X_AMVkG8-*(DQqtH3yfy$I5+(GOM)8*I__cpC5z z9*St8b7W5%@uvR;nQs_5cL|=%du>2?)34EFY#R6q_VPn5mCKXWJ41)Z#sK`?czBG* z(G>UiT1Z$Wi*5yxb?7}ZOrw3nM_>{sUVaj?xVIJ=aG;;X;?n+!>y81xQ|~l07_It( zQXXg}_J^}o#Z2sd&G_7e-SEFaOgj|PxPv*`|7dAk^eJjT*`9&eT%G_a2Zkv>)amCN zWq=2u5In)BbY-~SPD6ct$kXM}I$Ft_wBOBCbVOEU%M!nOM3`i7fxC6g{;M)TXJ)f- z^>tB5dG{95nQ)6nFVCvHj_K+=H~O-c)I2Z*wapcyk4yJyMLHlpEH70Z_Duzsh-B#IHP(nXA4wpc0U zZ$YrC!LO82?2~4qKSF0T>-?3xAlQJl;Wu2_I(b;cOzhRADAXq{KlH+X+uAEBFGl2e zf3m!LK%JEyfYbH)Ho;WkFDf<^{)0bh7?##rQY3YOKN|j_L=QdGk?I!oL?~M z4_9zvyn#sk2mn`R^^NlRnfi)SF~{z8S{o?{+gNk=r!VQg`Fo)3p;(z9Bg@Dc22@9A zKHUwgU+xb=3^1`{01W1Dpt1y1uJ$lZLoN@tn$5sKsn7x-b^_jVwbAnzj=6sU3DzeV zewKH+mN(Pg`z>Kabsy~qmNQ{Mz@5xwXDZk=DuB7FV=rtO{5R5guhJby_h()2pJjI8 zoG`Sos$c(rzFzw@;#-Y(HSRqFAc^bYs&(pgo(k31Px*!Ha)+(t9*&q9AxL$V{O)3% zzDE9qIWAmR`a}(6=>7q`yp1hYx*$zyJ~0akaj$0bN2*mG0hBX@$4&sV5XTbRQ{EgA z5?`-{uVFwX(9dpG(6QsL)Fbakko;8;p7$^{XZUxv-I;3H>lg8HhYODHl?$)-Y6cSP@dbm^0H}TtbGr9t zW)a}J*}E?~f043OI8l1N?U6=tjZ?exVnBl7wVUrlfVG(+M<$%z%#e4FqhKrBeLv0u zz{aLWq^jK00kyVy3f+^(^uEbZyQc$sF-#0ltu(uV0s0Wj@rWj#zJ1r(erwZg!>(Yo zTk&o80K!a|^sleIJ=MPeswW;;Ta$zH)}44Aw>42rq>_mm0V~?=Tl!m$sF|)k38VRa z7Py`!kRr#~+UCOr_Nkf!{OF1#h z?^ow%r>yUeA@0>)o$iL7I1|^uPFQgacrcM|y;o@Jl?scFksH@q0J<9Pw6DvasU2-g zBR(9C?#}M)o8vK8VS#Z!Chzle@I8iFVce=3pqR27rc}DeP`{J){0SjV1&S*TJBu_* z?mRMs?j3xaiudJ#u;GI)BtUynd_7QUaLaOe?jd005e7{-mTc5!x#& zzZuZRxEEx%Ky6J4AQ5^ub2xgBvDUA2_lvqsW>JXilQ+_E{<-2A1z@r?QeOZpCjjr% z2vLs_QN6&$rSSEA31Rx@R|aYO`c2=I-vaWR#XIh79z#`pOdrD#2IGp)i*B2(uW$YW zb$ea|%vcUcS+f}C$|90Ny&&xS70GWg)pAd(gqXAf-7AT|t-8AVPBHa0S%Ndym}3$` z=V+d?#7DNz^|k%^eDsA@eyT0;pW~eXTmvkL@LqcCPK- za)RJQ|41&0@kGjg_$ZG_o_%+G1nzc6NV9eX`2ydPEq$gfSNv6e_fdEkR7s*9V) zgQawGIs4}y1wss<(}y0ZKTR$4X%$^+5KXN5W>;^Z!F$`9m39f6i- zals)fd9b*BRqxSNAyC-Pw+Gq1;+8%q+o4yJh4d5ayOojmJME3n(DE!hwdbx@zMvsL zYZ=Nu7j9{G>)5QHrsqitkjr|wyyVB_HHYJSe42W({Y%o>{gmEI8DkAPZEXhv3AEf; zN|?71#ld>?HY}xf0Ya{aQQifx{b;f#2CfD_W<(pXy~#=e?UqnTOzQ7vI4oQ|B3qHb z^jVsZa%DzFR-YnPR-*(+*k%5JF(*t0u>Owmlswp$-*O&M-!Ca6&`qd;)DkX82;i-Y8HsYz! zlD^q$#z|9)kN`Q~XmuUkhhHvea*of2tyt5NB@xoW5NY!m#HAMZd zy>la{FyRNZ*pB)JiX5Ck;V42ae%XnBclge-+hlVA?<V+IQ1qkWst4-vTTa%#1RoQmr5Q8I#`>Id%pY zs;AsOpe3$o?eJD3z=EQaXeB!le3CfR(ro5X^LWH1mCX8EWqapbfw`YYy&k%gvPuSd zezy}xjmE@M7pRe%TC=ww_?c|o5_hsyzv*DRbYh0CHQ*L%{Pfh;*Kdf#5B;O(J35p8 zGdQFq;T3rtvyRd zobG`#mMq~vF+ougSfA>bpFVwGJ`QHV1`ziwR*PsrL@mGZ{y$cU;*=O}(GgXV`||0F z6B{R#LKX}4Ck(xH(N;iq%n?>#S6RUB{U#$Ms=W1~iWZjtT>;fUmI>V?wsPzT3MTA# z(_>DjBjuG_UF&15wt z=l9LBZR)LMTaV}%wO(5m6lfRl~u1kpU=hbcSG{ zfk3vlw$3DU`4poe{QPh>DS5up zhDzcbF`w`VeA?)09BEAnLL5Pg{6HQrwWtgNm^H9>ctiHRv^S!iy;kPkp}jg2w)0=yhZDK&vW zLMRK$<_((n6oGNr{QP1nEm#Yc^*7?6S%86u`I&flP(ds3u{T*Vm_-a(h6X60baDef zzMi2$98tfkC&Oh6b)kufz5;3;2_EhSTWbxbWS|h#h~39;Z}RRP^DwX!Q1|8LruGkp4^L>*C8GLxwIM^7+b>N;A`hPwey3mwe#ipj=4GX zHKTMUKtrN(twx?GOXA``nxI65cY@5r{D|3VE5Vy%m2djxG`+EYGWSD@7ZnrvVvI!b zyf~zbzrBhP)E~pgXhR3Z((v3*k5@bu71a^^kN_0er4R+TQ;?v!q5*N5osj)7Kkd06 zbNBZs;_w8rMN{=*8 zpM-^l!Ao2kp*%#b*JpCqcr;ADCz#vs3VcpT>GI*H=ahk!Ym6X}xaw|42*y{5DkXOI zJ&LAtPn10ox0j2;Q0I+)izAlMM3$*iAN;{IJ`B*InjOHw)=Fx;@UgCH&SH_LuK_Dt+FIMNK|q}3Cd+Li=>q8 z?YYitr$;-7@i5vhu>x|L7t1?PUKVM>qZHA~hnK9kHp5v{!O4hQcxYm@m8oe^V|go{ z&b0f*>2EbEu})=|6j?Pjf{*lTOQSvqb3k?p_G&?uKaj zSb$iEA3G-e8)FikMEM^`pal`@6LIfIc%KZs1PfSX-cO!9S(z%${5@AHA}T8KMQdQH zAwYuGtFaNT^@uLH1Xydm@jhrF;W%g@-%m4ffLKvFJPOXvTz9SS(E+dgS6c{1R<_}1 zHm3H)W{;j?LMax#;;B+DvvuoOG9e1k;Vk3P#m)pReDbyXDZ;{ScRdV`T6A@FmLP#f zOv>Fo#gk7;{)O#S3HbvfiugxK40-;yR~L8f(}LcD3N0Bw%ZwUhwM(gg&CX-oJ6UUA z`Wk-1@)94bLo;|QJ6GtWxsE4=tY<0+2m?fQzZz$MGZajHAmZcWhFBn9^f~JuXJlCb z_Q6Pw7*RPeSFcELDLWn8Uk-XhIqWj|y1;TD4)>;wqjh*@PbZ3j_s|LZTz^V+qz&W( zfGDFw_@SCy>950={I1NVb{he6fNpgs9Cnd#254b6?!@hfIk(#h0%uv=Pp#ZYY@yMQ zQbs13$5axNt3+=P(W346}$Zj#5B! zK)aL|dB?U2x$!=U92T}X{S!G#b_Q1hAw6#fV{t^iU%@!(^I3juxYW2iP260^j7W67 z=N6^Wr-cQDxEYxyrKh72H8wMGa_TI0|gM*Y$Z?B-v z+F{jyw;z7XCzB@1V>sMfBwh4Ac_kv?F;S!;2iV7=S32bey+snpc1H_Of%K1Mi#}iE z5^|glR#4JkGu$6UUruUvfR&vqFYZmIk}FPZm8!&8KK%PA7%VF6`6Z~a5wLOUQQSN{ zhv(a{!yT|3%gvEjDK!svaz?GPeXRvZe^C)vV2whqzHLp9^BFVCI# z7bFBfd<@+!$%OB^RN5_&`J>*9&K6<=*6QA5E1n##N*f5K2Gd6n9Ua6`!gm#cJ~Y{h z-Y7LXz#Ew;)mLdr(}WCGz#}b9$;O}qg zT$q2Oi_7Gd^&|HXpe`c6JBK~>^mcE*I2!NP%cp>tFnJz+>ZeI&tgm^6T8$30V^L&s znzHrAQHIqoVM*H9Y#7TYVW~$*>K#5#VBIFrHjAU+3vO%-^W>t-2MP-(m|ZH%ke zMjHFb?7t`xt)il$Jx>mpUEBN&F|I4sd!MpnY)l<+JN{K(;2W4Rl$4YvN_FrPXjppm z_|ySWwTd)<3=D)M*VYQg{1+MUw5O(i42*;*77$uZ%JX@fJploSdkBHu!AC3l{f&KO)NONuXVu8p^DwtCR#bnKS>tiA0zG1UPwuYn3$?n!JdB zhhwU3fh-0W7O1iD+-b|5KZDQq4UFpF+cWcHGx%#|syBz6fk#|i52C7VYu<7JpYqsW z=qSxhuc)dE56L5<5O9bUhA02K9_WM-AnS#EE-I>Q6ASw|eE}ZlLJ~14oznPPdu#?H zU78f-5|i-~v?}hg9zCtj!4ece3Orj%Th3lpUEN4VmFN!LFn?wB5W9s65>SQZ;ykbg zfG^SHp%f^Q>5Hc!6Y{PC)T17voWqL;TrVCXpi?4kgW&s*;U{k815wF0<7FHY5|VEK zTfiTeW2f_d>L;WUxqVx6B7$`1`}R*3Mo6u+BHs#deb-je#=qauVKVe->FJ*x7T+GP z>8dMIN`-(xJHcIm+Y3xyktKg9ddGO4SZ{15$MCbz36ms-*p7>%yb707WMYQ5Az+CA zq5gFtud`k9XkhW_UOdL974O!V)Hq_%iUvAnnHhVo--80Z%GnA3-sx&Bv{7Y-e!%s4 z#`h5xIyO#LT%)U{+1!a&gVX94z%Kv2CQ-hdn0@q@Rrq&x|F>PUO#fvu{EtE7zdrrn z?7sgk?Y)zgb@mS{fnuLz3+kUk3!c0+M7ShPM0lt+AkIJa$RMq z-ap9e(jDWuU10j3?aSRve~0%)Qijb}F9EUdo1o>LWTW8MET4|yy;W;JD=DF3SI zzhs@@|IgR`Q+mw*f6nVa67-#aD(uYAGA-E?liza;dzTr!c5)H$(P2`)Nj3%?+kA1kca3q~)%DPn#v1F;b~+B=u931uZL<-T7b-c2 z?~LFw!rH!W+AWwt(pjrxQXEDlm0N|jFBn|VeQKFbJ%V3d;`kMQXQ8EbE&c0n&^Odx zG9y}e?8H3&(YVBmbDhaSbLi;zBD4$v2T-x>si_ex$CDU{9^S2O29YaC9}alruB9HF z0|q(2PbCNL{~62UK%#eOLFx&K1aZ049jdu9Mc3dI#VO#nh{d446tmHnpG@bnWBQ6m z9xLG962pqLjI4fC^j0Mo{%#c_+K2@82zjY420S$ixxtulG(lmVPf6703z6w!NK8+U z(g9Xok{C0G)aO6qSbq+lA@`L?6H@;M{*0OmP}`?r!nPy}d%>?$@bqPf0ob*$*abun-c#al%Vb_!GLEfS~7H3s9Q zt92?+d!rY94c{N^s+xIx=iB)G(ARo(C?MpUp`IAG=LHJ{V3u+IWVN;`yAUikW4$^_ z@&TO`dzL(3i&=DH-TPm2rT+e51dQ8y<-YyLUEKu+Jd3`!3+)2|crXZ>T|*r#tOq2I!Pts6P;s9j7Ar z&RlCKIijV%9G(_^HILh{{=FM{rA)E5ohK!#CEh*Wmw+#h53m--0yswpNKC;A?V*29Yeiw*$>z6P(_>Wz?%3U;d~10nkoz-hM9QQHp?U`S}KR% zp5iCaG6Q*php}qQk&o5Up&&g!*E9jU(D1m-FAmvF4L-*#Rb#m$8r*jJ!g<9!Jx!up54ec^_DmX>fKic4Nldz{ zZ;YlGM@iqOtCAzfj%{cA$MV(u+@N*gVR~XWKIe4YuGo^(8=;!5!%Nu&NBi!+4F}TQ zJ^oC?Wk;x;VkGnwC%*0O?4ZHTo_B&pH)W$Hf4J&hzf2Rb6}D8# zjZH5HclIdh0C_7>0d6n)wAu=pTZXu!-&eR#EX{#6T^qU3PlC7C1T3(p2WPN`fP}lH z&_pclLcEg9+EC?VekKkMsorFm>bDlKQSsOb^-+eU6fYwyE})P99Lr)e&ch4il)J0F zYxpFKp@mB96fyW)(L~ahX-A3frQGof0p>UcnMZU5F1L6d(mndQSzV{|_^W1&eK&OB zwDzj+?ba=S&v%fp5O)<#7?>~QDX2-wZ~W+I?WWZ^Se0BQI&oNj!d^0XKiV&*`WU=! z4k4X``krok=*rQAQjs()8`7dp?xAD2tnJW|>raoYbK)0cpCcu=6h z%M9cMsA2&sS>=TWU#t;Lm6fp~e2TYkW2V19K`4x7mGIu2`QgL|&`KwL2~AXQ>|0hn zvOf`CH!tX@E8w|`_f&3q8>hp{ARBR*Fx={YO{W|QWI(!?@_C2Wzg2o36hPY@8t#CT&edG41C(xp_ro`4$YQy&M!q%40Vyy{kU^6c$YNB(v-aJy;b(lhhOVx>Hq#_Bdi6)o z>s@F>q%rAgN0SzOpI9u)=DFLeO=snNQ_ZY#+Yg0zMl{@l#rlB=xVCB>9L)7eOJ4;D zyT5A`mH`qymSTzacB#=*B6Ze|(+0GMpWCc-uVBSW0cPCMWHRFt%JhzoI=Zoa(QY1Z zHx9aWp$31ss7+;x_IR2ieOq0@+F*nl9gWo@!M)G-{!nlSPcs3_bag^8OUC9~FTZ<5 zjYB-JX952&RI|xymwex%u*!VXR;Ht2VQDp7uT>avuWe6a9E9uz;{u*NX+^264Oik!ZK2(HXHO`-Ya8kXaCl0Mn65z!2(T0-2$P{hmI=E zB_@^nSgOB7P=FQ(=d_d}rdl3XqwtL(Z)Igy*A3NQ<1LD~6tZ5Jc+=8g_6tfym(Ma$ z#Au`rnY;ivWv>WG=|X>hi~ackyV~1Y6-GlN>I+ORkQz-!ek1#a%KJ{&7*HWux&=L8 zgCCHIr0GGYDIr%-EILyAXT+|5U6GAlwf#Z%VK)wh4uIWZ)DR(r% z8z@hfEe@1AUmO0LKpVi{2cRQ=5~-M<}X=4u;;*59spWsz!W?zJpptIE1V& zSp@#$W#=5IzdX7{kKhzVX7s@8{~1pF{Vz@F|4puEKE+P(PuA(x0J@z+STfNst-r&g zoa)b*IB&y#ebT}-qC$Lajl(YnGhoC4>hJm~IQ9cLf!5eoe{cYmti`>b$8-mUz@R-{myf@o~M z@x{jrcJY7!T-jRE^4BaAF|3ap!6feoZhM4AK{vutJb-(v zSyXTUKkNWMA8XcSSt=8@;Uiw9FJ=kU-ikoyRbz>+J5Bcql4?fTk)_^Kk&Wzc6@BaHUn>30z=C?|+v{n1wLu5n z(3$3q7ug&6-xs=uYy}c?yivEOs}kvlQ%ofCBzz6$+kgH9YA^CdFA5eq_H)0#TR8yw zctl9MV161u!b}HW<7J-R={9HtwNC|a#6oa{r; z>I7QIz16h)}nmxbfIc^;1Re#U!#jJbFMrOP1lvrmr`5G z8k=Xs#pEvB4o}%njq?kZ+J~;;29NMGR;Q;4>Orv7m?=M-ESnbHb z7ikMx!?yv>B=P%8fr+P-hr`J1-c~aQ=ka|ooG&|0e)57m^FbkGY+^g@NdvJ&RCQcc z-{2ejZtO=z$q%g1KH)A)Xysg^iwHP$_r`I_HK( z2ER69Qo-Bt55>D@!OLBM535V+_wXI!Yk{958^N67=jt^BNq3{KVEr4M zk3aK#b%AB<&O9mXzU+Oy8+Dcf591q1$RsuGoX|_-gc9)6JlUd{b_r)xyrtKWy30fS zFDng*2^7RM>(;{loZ!tJw?t4%ax7Zj^eR}~b~8l~#~uwNaJV)BjYPZX+vn&gF*U4# zacVI;@8ljF-EA`6eC%NN=3yY9&uXxTQ43-mfZAF04`ob9XCC~xWW|E$_!d+_UJ~#ls4O8K67@;PuLgBftk=B3)f`4=xFUvvJ;&VSYzF&aw-qrh3 zcsKKf8T7KV48wpPLYdVeL&gc)ZPft(quTOAe@k!+e*Z7JbpYGEMOwr{Rp+MlB_?o@MsT1q~f@c z4U~5@tkXvH{u~5Co+!MM)I1(7eH3_ISm}DgvKz4?(uCztt>c8R6Ym!qw*%`WX6uNE zFb>YVdc&Z_6&-`S+x5elf;1;E%rNsodeqECJ*9T+yS~!Q0OD(b;SLelpIqfTo*YpN zlM=X;Wpyv^uVz0Ds%AI*?}a1XvkVjo_RFn!v`QeR-pdA4-4m2tK6A8Zj9yZBFTL*Q z+2&HZ*&QVaC6x-lST%o5Jcd1BX+whjZx8K5{oeOGwvhSY0gD>6AQ%}qy@Upe#kzi2 z4y{B1xZHdg>3?H`AOdv$W%wCz4b0?WPk~KaeQQ`# zmT$>w!gS|3{%Q|@;GbQL3yDeVmbn2u%hN#kZc?PzF}rWhqZwRp>KIGR_(u(C8PL0@ zW_5eLrmN@nVuX@AZUbe5;vpeEU6or~c0d0-9*3Uo$3}Y8uXXv+KI1r(c6KkFT+SnZ zV>}|=+bRWAE+p4!_JBj2`pHrQ^Dy^f1G~I;9?v>R)f45~$RJ$IcPdeU{_uOmi@*n2 z%DUkd-39W3+#j1kI(H!>0risYFEKdYFjqU@Ao)$U<)$x64LQZF$t_z~g1-j)2u2@R zt;jD}J`5-_I>7C|<$PlQxx1(V9QdAR#nuX4l)ZN-iPo$m1MGwin!}g-4OjTRI4t{k zoW}|A>4DNRQsWw>G@PE-jUimv6*aSGHUKW4f)Rej`{3|#;Syzxz?rSi^q9sHJVIW? z74v=Zxy!9lcGZ55@^T~DwrN+-3E?vBp`Se}37VU%zA`1oCz?RGL6%qpDV<2X)3W_f zp9?6a0B6?OIOgkRl-Omm7#VT?_91%R47NT1lsH=VXgSB|mOy{WA7A3#-R{?qtu{lM zeALy2D!qf#q^s}(Rv_^BP94Kv+MdJ31U4YHdQZ{aq{k{)vlCow#fo^~PXJHA3nv0S z$?EaxnqJW`b-zx1++zN~cV5qSj$yxP%VT%33D@{=S`571* z)p_|^40}vvHvRRQe$Jkih5u~9F z|K;d_iur1=qr-{Fm~}xS=A5Cdk90wu`h|G&1>IPJf9S7@gU@mBVKT}ZvTL~QX{ED;YDcjtd+Bgb_};sK{UD=bz({073dZ-csUjoKXWF4P&~`|CTMfVk@n01BVCw>2*a^JT{<73sKd)v zWRs=!AlR18^Cr-H@vGt1N^RRhJ`*Ty_Plf50{o8V{Mr&EL9RFG(aW|s3Wc-0D%tO} zU*V*T_DD7Ph9NMka}#L`8Z4P>_duZgHoQX4t54>F4|q`=(QAUp0etLhyu!fCx!W$9 z^1WHLDk@7C!B$R`O!%hj0c6s5#GR}zCHDy$D4ogyvbmy+(eD2feb1Rq8?K!~*uxpp zq6~g*)Qo~n4Qj{1ok(y=EKXQc@?^|H#zO&%oD}AR24c(XWH)*AtkW1ygF@%q67sqR zpC}7)yc~3#7fr>99}eea=?IATBp$c#6#IaP6n_Z9AYYG3F)q+xHKZ zskVeq>haVjeJSmO%xm9OHUn0-)9i@^+F*uYXTT^&cZoV|*6>i5Nz(88vg$B6wd?gF z(1R3G5Jo=8X24}leE&VC>(&A`{3LaowCC5bqu<**wX{A936MxSj8{;y2nGQjUHQm~N=jQpE)N-S)t=xTfz_ z`YpI>-<4}}$x2lF<^XC<^Ki*qv`!|Ot$AS>V@=1tH*>LI0LV@-%;(}NkDILq=}Gto zhekqm#R~*nh*FMV<_6&vuNH@%9>gAnTq>x666ZTSdTkya!Pv41+lU7+j)|aQjq=w5 zsR>~JJ^?TA;dO{ch16R0r!()2soaVva5^iL1be&nbxdrnL#JbJqB6s`w0c6be`$P)Nq-NFeW6_q-{-aDOi^&$xQs zWnk<80F(!Jn@r(fItqmEtog7GWFJjU8W?+-j(vYzg6Q@0hxl!L-eLViJ7s^$P?9;5 zl=%d3O?mhz6^TtO-$ghO>6i=XE1#tQ{Ck(RbC>JrQqMh3LCQF~*ugi_}q>qqS+pAAHqvDzdS zRp(PlxNVuX{!m?+9BI!bsA211r2OBK3cz3`MuLI%K^!3roz*ZMiI<8f_21qpsjOIZtoif!_h#Or=LNu`W&+Sa zgDVpr55hrdE#5f7M;>Jh!VOrr7@D|X#ElULqZ(2fE1Ma4-I<^vRRw7_<4&G z3}xMdj?x>T!3A{FM(e50-J^HAR1ySq7?+Ue)pGMo?Sl?`HGP%KN@OBeFHV-B56)?b z^K@TC5Ka4VY$r<|+VtNZly+~EiJCn#q7fxnHdaA7=yx+tzK2V5I%u-C8zK%d?zy7QeY!ob z8_Iq-{kj5;%^OYp=B3S)iX)xL4Clkw34NDRAWrE_6b&w!)^cu6A76BzudjQ1b1#;y zr>dnhNp~w$j~k3%4DY57`t8>x{Xh$35I-~qeLBU8r8V#C3+I{P@gsW>j@wn2bhGx> z&f9GL!JgH-mIku&i^EE0o=1`)5p*Vp=DvQ!3m%ADEcBG4=FD{BwLy&I6$uOa4->GZ z5~~P)Ka?AqghQ1P+xXT$3tKY{&h-5It2yYOOfAEGBwEps(`HUN348o!)In`~CL{RV z3m4nUv00o%8gje6m%PR;CkY)UDzPaTSAX5;Tuj0HiD7`1aZue`d zP@s22FX9}IPq*ELtGix)K)sLe%(97llGTJJ@jakXcLZh6=Rv!Zo2_9krRU%+hL2HJ zY!nXzB>2jiwIaxmaW7|L5U~O8BTRZtrb-pU^h z5NF7ByFkO;LXErxY zH0!y!Ov*H7>Dl|7P#{;fsblZy!9jLZU;EK97xc!L>LPWd{#_8E%6zF+IGl&pj`ip3i=*6`S#b~@L?RWFCwp}|UtuPW2~B)1pdq3 z>?;$bUsFec?I}?zm)q{SHq1tV(upQ1b*t-O_sU zjac+&3vB_?2<;0`rP6Y9MH|~B>F;WT+L=% zT)q`b&h#F7`Ot|DuTN$9r8fdMzt9RIk91$zdf>ccz>6fhW%G4e{muY#?|kY8RhM8? zih_rS9KjNy7t6&dzv5G7@>r3uE9Upfono^5I|D@WTOw(Qy>K`CE{}ec4G0g;BfSV@ z*kDkB;xUT#Y&z1oc7e_J!~3w@JHfb*6O{5Q%6&uEf$FytsskLlX*XhRkqIGR3JK-R zv`yjdQJ%-hSJ!H+Oudl10DZi8X?VT!1+V1`3S;^9I<$~c;N#YWNGSm=>Kbl*=HvL|i_3Ga<#dum1_&>>`pvV3pbe0t*(?707=gs3oL*dyjG>lX#I|{ zsbDN=7n2ZRmfyfUbbqmgq7S`wiHOeIh+2`bcyXR;YYAN&6AkpPALTXnBgvs=M!KBV&~MIaaceLOkEw8NC&AT zbGqh)11V(8Gp;Vk>E=*27s>R|wThy>LJrIOi*c~LsJHeZ;`!1yJSPo_xBRCn`vkS1 zHW4W9IaZ9C!2;$RNMMJbVGTrUOQQ1t6gWR~ePHijL_T-*Ex@p!wIQ6yc{(T)w(`BA z)DE?Mom>3X)TBmx!H5(50B1jKvB5BUZJG9VUhfzgK)EHpzHH)o%xVCn0od$?K>{Q2 zRG;9OYB*po)cF%GiL#FgUD{ZQ(Q`Jkp>ms+iED;Rh!IQ z+7kQWQ@ns${M@~xj*vVqU2b|%o{lg5-21mE(US*B?qrcjr5mc;9#}$ciq4U9x6}#w z1w|){!QCk*jrG!)RxAbv{#sv1pA!q@Y12+@7`~MC4~o#}-p-y>_a8&OedMzjM+rQF znOER3b;p&jIkwV&mf_c9#bmd0mjBGUXBE%D{PB!vW@sTxx)JBdgNJ397^$VgosKl0 z{5}^Dkfa(BRQ=38y<GP}7R^wx8j7w5SVpMH2g&%Pv8g!VnMv>x|+;%kD+Gi1vceWZJEEj;uTxL_-# zPi6H6;!6t8&V6_=GI1mn-0KX(%BDEM{U!exvM0JK(VeTbSQ|2zm>k=s9LA+y;FZZ# zRbRwNvt??fvHDTf(YVhJr}G5oiLW1ZouBW+!aL&E3>iY7=9S=TptZO024SJ)_^ZW{(3uJ+ zLRmbTz|Ufr#`*GBPl_!^db(#_D-raR9rTqHpY3x|$b{3B_7Qype=_ubJJRQ4)+dRa zPwfU+`*e)koYdKe?M zluyE>6-GVJ0|qMH`b}`i$7sQV8B=2~+NudPjI$GpVxGOnTP3)|v%r^2P?fdYB zfIA-z^wNuuz}eX#XY*~b5w;t}7_D^pNnawJ(M`F0<9ms5#r7mhPo34R9WHJYHgxpC zj%YB7eh0)dCigp|G&EgLb02LgVrSE}&u-DGK=5v=X<8@a+;zD(rE#GgvQ7OZlO^SN6Te8^OzfZ4Hex?H?VM%;n5nr(gwlyI!ze6*bJ0)1Im4F;abljh|?Xel5{9H z$hMyv*V8C?H-sUG)~A-F>bZV4n12<~nP?(XjHZV3eU zpo@pao#5`y;x3E3JM1F2zx&@+-CfmHZPiq5&-Bc^?kD}dZ(hGHR#>KTTb~?9IljWD zH{1u1M;l=qUD5JqpjTz@TCe85VZ$Rd#E)|AZ*q?V_%ZN{n0p69VAIgH>g~DzaQC5e z=z=b==Ypi@SbToo^M2V70Y2@Tqs*LZ&AF)r@k zLE{WukCJ)W-vq52Z4GTqetH=c`Z^DFT0!a?n5>%{?xl+!G9D+>!-IOTzrt{hrfV*a z3g=@E_?;hmhh@(@Kcwe%f2A8?(;TJsNkgoWeyV3ZQzM(mS(@$|_Rx4T?OKQEe@J`e z%GYK~e(pd?nZ&LR8=$r{Vx&%kVjR={EYI!hdJ|<<@WKB?#$&oO7>oLLsnsK!MHgne z`@fto8+5@;wgGznbFy`S_@b}Fub%fttN(XfAW9o2HV3a0frWR{@Dmd^BkfO`L| z^gogTeOrp3%k}W5EJ0UPv~TeG5d2&zryeqnL2A{@h;>@m*Wv_P!0)?itj&$+CtRy8 ze;hd@s<51koA^IW1SUbuL_>XHe-Gf86^!bTPI?=S4btBk!L78F!>5m6VsR&bUONY0 zF~K&Wg{foL8&kxyW^*X#hOp2S7`b~kyx9kqDdk~LNWG?mWv~f~KmqYV4DwGJlK(9( z(v2aR-e#z8RcBD5D~Vm-_G2FM3p6SvBXpMAGjuQ5Lc&7`;Rtqh;Qfcrr8?xg{qwD$ zzx0>4{K9u+al(Op(?d?!uelup@zMFORJ^ac+Jp%bd<4ph1JHhGXSMuTb&;ioWvZrS zAw>oN08#tKRLS`50;&jtcH=8ub|WAmv4|px>6q&fKEANk)nJBy)XvX6*r^rXy#Y*) zTUl;#aO6Jt3*Suhi0ldxoB?;ef;6r?I&($B3var1qM32ixbnGz!jglokjtlrM3^?9 z)m*p02kR-O2Gw$ySQMRzI z79(R?70rJj(ls$zTwiKEdY9fRn@zkP(sSA@s6a5E{#@Z>@EDx$Ff2PznT?*T`|Q*q zRV;hP)Zr~g@_-gfJo8nWo}nK3X@Rm2S|S$L`3{K4Qn)0*v9in}Sv$jvVn!21mkUl6 zaJVIf&klzT)u7UIb^{*uoEN6qL?(r?4e{u)?9DgCFGm*b(3M{6<)r zsZt!pBTP_Fywp8X@hN?xd5)E7KBh^SY_cD12f$kvbZ&CRXI=Y{FxF5N+a~zH8wuM5 z`glwKucxOF_*q}~?Y9!Kc8N`k@P2$p+ftVocHf%T6^M@N()R6b+UFGhiGuh^#9`JA zB71+aJ}`>rAf3UTgZCYIx>(OD)hNNv5-^mMKLRzzi`F~)nbIKY*{FJ9oN_rE zIi_S9;JGSG}T^WIL`# z?@IP06)ie2R@=`NY!;dc!GVUnc5X`gYI6|O;E(2!V}W0OIYk5fSAdk1&C?}ADD^jE zlpfySY~2>#s6beLdQV+mN_CqDAjD9yXx-)6PC~Gyq9M=x6xzXYk!LO2cj4Dy8Kc0S zg2VtD#EQsK$}h*q%WA$Tbg`v+BlfrElnd$EdRb$O$zx*Ci^7QuZ+obQe~o6oC-5rb(GKIQOzo-4-h?VMk@g>_lo6-{|vQTk~&mr zp;{~lakX8Pt14uSy?I?Mr-udRJU-fN37(=SN%_K4b~;(z;dLCR3Lz=Fo4YvHpqH2# zq6a)T;r{Um()eZDr;r^KYYi?z77om5-t`D7q=yhhe-zX!+cbp{dz&XZSyJBEOYC>QCbkO-o>YbBKYy#cBpxP= z-dp=-OA3eF`PfjC-8=3~<+B_mTp}3xv&Ou380F88Mw~|Wx=DC*CWCw}jwBV$??MH( z^l93ZY=*6N3SNlsq7E^Las*ml?Q?i~GR6NDxzBqynk+p8x$GVqjeA!sEY#D4MF1H$ zkjXRqTg|RH&h}PWJ;qEm84X72iT0YdNqU#r+%Vt{@)bS_V|j#Cn7{o>iE+$WU#+ZM z$s`cz%)N8_wa;SG=09Cjx|};eMp$P$?_1QEvOpB%%QN;Xbj;J@Om6<|WuUb%TBp|H z{n_qfjU<)Pz;#8+K1lsMm)_wYDfDntV@8d}@-ENzLjUd~?eD#B3U&;`@YEN!Q_0iH zcY6cRb+&BZQknO2BOP{2iMAt!D;a5){TYzIlVD|BgC3n1Wnm9I9)+bxtgQz5hsm~u z7|dzadQ?jJU@5QPkWLUyH{J9*F6pD7(7xNH{(T#Z-Q80RRqC0=vQqAq>)ELnYSzjX z8Qd7*g&2fmVRPdGCzV-y_!-M-Ry6MKxu+VX% znyi+WGHIS>`UAdt#&{cCA{L9LHLi7Vwkqe@>!1wkGe5jF9&>{T zTif>W86*rH$gEUU=r!TnlzUxePgndh8{0Z)buoV<$s+t+kgE}y;GBYRZ+;p*J`f-0M`6{ywHRk;p8YQ`eo+}&ZMfFS6f z>$GbT&*Zt_okMLx$}D&U(((~Q6Zt(okN3PGB<1er)qJn@sy<18!!cWbjQyt(j*DjC zPieId968NheP$y$%^J`L|H>IPh9+yiS-p|FSRv3qY9Y;lbh+AVh`t`XO(`ce&qlZy~wj)Bxp0%wM+E{zE{{+#!D>T|7!cZ>(8xM{a~ zvt4i|AS6C0{tNU3tDWvwCwPp@;l;h>r$F}tKVfMY0a7|DzKOp3EAW;Y1sk0X0Ts0u z3C+pezGEtkJ0$d8j<1y8HN(FX7%u<&sUeqE>;7}i3c=d$)@(j^$9tecy;E^$7{Q!V zos2bYN9tks*&YCrs{Hjdwv;+g_3~bhhtYVxjLY8&vJ=M#Qf(38KR3}tDus@+Ycd%O zW=-xD_`~R@BaIpuORR4m$Ek@kn14~UMW-8IzW!P?olc(4D_zN+*CVX4J$k3<013Ny z4RU|>*fajNJm}V6U^kWQl0Og;Yp=~f@z7DU;~23wV`8IXu)sm#SpVz`U#zQXlG4CV7*%|DezrSX%6&(N@%b zogNxr^2Ex<IBV?KEwdBwa+=}0zS3v?O<#d(MLW0rgG>@ZVE8T2~_Qu0W_s&Dtj z^|87B6a`EAJb4pA|BD1HK*bl=HxsDm0Li$F>GfFS$#JhI^|(xzZNJW}&M>uFkqy=2&Vw?4L=Wpg5gl(prxf24C z^5}UkgR7Hrq$zUaAlI_#1b>+$ zW%mg2?O`wXw3)#J({5ex>dy5~m0vaTbzgB3y3e}ZHd?7`xI-)a_o!t$1O7=!qhCrq z|F(L~iA)#yl|G=U%C=xftA`|SpKUea>d`gJ z0Xn1WgWA>W005nEAOUP3nC&stx&U+!ZCMtO$K?v%FO_O1`;qZ!K>yC=$#^Qo&s|m7 z&jgKeK3_+#a56wvvrW!AopG0!H~m902=q;S>~Cbk9P6a#Yuyt!e2Yh)&Z~bPEEsm` zxy0z{tP>kP`id=$(lhchJ$z4g>0O-VWYX#pH|s7(<4cgP`-9||*?D-RO%PmP_RT5A zgjc+4VjPXKG6khadz5q5?}JqVih!$fC3J|X5)0SU^TZ*>H=Iy|KH<)LOPT=$f53A1HW_0c?@Bl|X?2OIxray%3K18|0V zlUr!7J-LwIGn4PC=fF=3s@sZ|XUuH)ze3fQU;+IsgmNdnnaf&1LCf{LxSh(zhn&X7 z2P>@YfX|&FH-iW${BEw?C>~H3Xg(b*!D!el4R{&1)tgO8C`Vh1nB1Yq_&YbXa4Ti; zmwIo-tmBAIXEws6g-|f(ak6=p9gr1AoT~qBGt@GeZ_1e2CX-i4kR*Y@oT243j%8241xGgIj zhPs@rLM1t*)9fOF)Vdiou^4t;x$M&i<9MfH zB}h6+I-HE0%z~~-Ig1<1k-v+rmTIRpHjqyfzvpe5^w$NqOW!U08E?p@yE*rQuiL|1 z6A1dfxosHsp)?L%Lhv<|-}0J&dfixSO2eY+2x1yH2 zA{0j=I=U=Rj4bqPjZ_N4fu!9DH^joKpd$CXR7%1Tr;UG$P9I;_J}ELl?-fp_46KI4 zrj0&-P>81d?(Fhl(RX3=wN5sxtr+Lw987x@n9pARBl)=d5Nhxe(SDO4^tWiFIdmxT z8>{XxJsjeekE-oEVDQHedWi@?Ktf{Tuby-QGK%Q!8jjCRrie6q|KYrO48hkTf z+Cj`3-(O{sHV#BUX|s?|^_KN%q+obIWudX@(l*zyH*M|LR3G4kekX z03_xGg}>OcF-y0QG2|F4DwK6}luH_{EE88!aIdc!ktE+zs#@vxnzFSd zl+|&62My%b5>l;^tsI1up{HxneCc3dI{+T0Jo`#|uU&T`LF5&S)autrg?)zdNeE-^SC znY!r(GL9?}O+A|~JJ_erJ~J~SKgkL$NCH@X{$5Qd#24xwLZ0AHZc|F^L2q6_3cIzc zo@{F(W&rvT4wkv_HlShL-G<@_qRy_WHsYNw9tK8@*DD^!WLLW@5Rv~R^d%*xK4!>} z0FLCyyr{quNg@%U@V~PuYn?(<0b-U>{^YSFFoByF6eJ{;(a}h}V#glM1Qrsf6e5Xz z9Ujd_%$y+e$eb;dTdj!y@PP}r;miy6Hi$4b_}fl)4XrRHBvnYSk8 z(gcp08fA;vub}ba&q6f#G?qYNP@~28&zNy}c6JSI?Vl^+p;7WPm`6UyMjEMv+T~1B zB^ui1vKFx5{25MI z@wZX8xwlrJXF-^m9g$9w(#n|CHZv&|PqEQ7L^2=#WjcFJn0oDlq^x6XQu^oJIK-X( z*I^!yclqRhYgr zSTkG|Qb0@f_5gKApnNf(gkQzXlIdTcR>^t7A26A4C$HcEjk$#!9QleP2@wBsdaKe% zYA_HGrfQDvs)}U<^29&f6M94~Q?qY4zN*cj2PdT6-f|xOouNCGi6jiob^) zyoEn-vt}gb#3|;AA&X|RRb;OrY_J=%{**)ri(vBSHzo~FK_XDMm760uD%hLGEUBE+ zPE@qjEvlzLQroCA$Fx&g*p~+`0+P+>OLGOPuitxt)A^zwR-1ur#H@K@KsM)-ZYoTw zhU3Wf7-13+ahE41!D83=TbFe*oK!zq4$+vFhEkdS&X8elEhzzmo}D_ZT=VXg zGM23!Bh#!5&n^c{kRK~CCTGN6+?^k(LMr(04<0IoTDZ)k`|% zBdK^gM+p~5ufFJ@9w=Bc6T(7^@>OpCkIlHiS9D`}jz51=-;k{0vc%4t)8FiAbLZtC z@mpuy=WFZyi8)OCGwl*0gATF9rkPqKH^Hu3#d4T}?jw9ztgdQyq}+dmt6u`+)N>Wv4I&!3jEd&tvwESR)5tYgpCJC3nrW+wPE)0w zzeYEW`O;#7wk?^lwp>YC${)JFDp_S(^=*VK3t<2A%wk`c&-uo*bgc4X4L;$}q)A&m zxxtQlN315J1fu(j4pW>kVGUm|8(LAg9Vt&q{D0EenB~egbJr$@1MRq2ESO}W;riS{ zAp3&y|5G$D>GTGPB1$5O!Qe~WDEW*P!k?CBrJ9tEND5{-f=EiYD}bzKc!F|wc*)w+ zZpt|FrM^V0y2rpP?l^Lx;l%Zur~Fu(fb(0ZCp{GV^wZ<6Slp@M=x*nEk^CA&(Rf+2 zRNr}u^1d>=m#heGBmHTnH5|n_o`)s9(Zg8SC|z@NIdkM&lgsDBk3CWkHBNho5WuKb z{SYOBjLX5^^j|jnH($J)M#{~8A&Dm-3L5sxVBCP^-rkFz<$XqEAcvNl`Rv1OUqv9w zea%rCJ>zPJR9WA)N0h#>DRjCbPzh9Fz%FoPI~KiOrzYs+xw;x(~QY z$!UGsFlVP0Jq$~OKfEAU=|W~NlF&Gjv&QzXgv-ao>{nZ(`xfB_FGun$my8}m7fBoT z8*C2pewD5Ve;*4n*{(0GXw3^LaI0W%=%~|jMn#|Y^e`P4$Sxu7_|=rrP-OCE#G6<$ zFwZZO6KtRosEL{(`SRpziBosxv}xL!>KmKIcLn`XbP+Su05a}8$8k>nQdh`d8O54T zLA2bhBTu4E*4G1QU$91<#U?G`v3}QulYA2%2%=-L=Dq53CgnaJ8rEP__un$`S%=m5 z^sqaKm%HU^9J>hA(9w~;aOdg}g$^oG1GBGeXyUIy(vv})W&fw~kXR@?Xj8|q{=eqY z{JWLe)`K>f_bXLLk^H7LwXFqK1TexdjqT^rRh5=tYbQjp1PgJ=iDgQ+<;RmBoe&QU z+=;Eqw&m9;IQ^ZeVF~*U2$A39pmBG`AmUVYDNjbp3+9{9??bXr<0PHA(y=^u{O`yu z!1a;-{1&vUPNKIXfZyxzeoeMBVa((fehUyR!KGF`h|h^iFI>jmUl#(bL=<7ui4o#_ zrPqN=gvBZKQU#u$WI=4ZA%N!qeq;v3slRW^mMp^QLirr!&zz;t_ePHlUy&6WEZYsz zZEW@+LeePtaqO-=j*V}v@paS4HP~>IIW5OL7nfWWLdVzdyDvTG+@@SCUw&{K{;W~J zU-s4v54a<|<+#vy+KiJ^#k9ik4n2%4@y5hfq-JG6EsSug4w;DnC6xoN{|RU<#*N)! zZv}tOHW;$P@T?1%mv1zh`4xgcpMn7(HyF6B2p3MXmsC*bi}Ca~^WTDX%@9x#gP$`PziZNbZBY$Cl4GV-bdRwntKOho zT*Q>5uANcJIgPcVIG6>gwL>}pX4GD_lZErh7^pAxPUmZM2ODWwm!K@A!W$yz$t2N6 z+;*%b96tj$+4$~2QR9OFX{I4cHLMb{1J%@auODb7% z`?~SgnXw71dusIpuALRHJfje3OQVPt{u}0(mqA9z__yrTGvVJKBJsa?=)aI?1_-%@ zxr6_HUv3ny%K0hbGT4izS>et(&*vh~nb2=`9R-mQoGlPQtM17n(R?2$zPpoe(kmR4 z@Cl^|{}%^=tjJq|`cm-ku?pUq*eRTxOWzDgwGdEz*RS%r7P}o`lBfEYE8RT+jMX=c z?#MLl_?+mj0idm2++~v}{8MVCFC4fp2!6|H2#QOTtUtyS6{IV&SPa^Z8(V9om_)d_ z__Tid_C#Oa12K7jHR=xYPP$>?MJkukkMzzQlmkvzU*Te(lm0I^BRxSgRd0XA&4hZRVbmvPVKlgSNz&fbc%VVu!Y#RQ z9ABncjpJ12LpUJ!1P=Iw1Naz^3>qJYN&KD^D-6X&N3Qf5gAwEZmiRNq593>#Vy5_z zHmUoeWzClUpl?zyS+2sYS>HAgP#mhh$fDsq3wZW)eWh{#FOytd)psZ~e0vm`=K8Dm zxmYMaQ z%|}M>?0kJg&$E=u20MQv zLIs~x`ZS=m12T5O=^$Ay*>J^VwprohyseJua(_O@Wc*pjjLM2L@?n>rD)f0duW~cY zY+(o~^X>5c#9$W^DD%+_IPSe)QcHgkwm@3r#wKtd*HT@b`?ZvkuhH)8>->PC4nEKG zs%N)I_5GoY4ztkIr&wGT!&MHtL-sKChNJuMPNn`JO8IjQGU%i8y*<*}X)RskV~I1I zZs2+xsYSBgFLfH@6U7E@)$V#&^^7q1hWR28V?dU$ApogGEpAkL(_kgU~C z`>pPZ@f z2rlO7W00McfS!-UspQK0sD%Un|2T9QJG4~IYU)Fo5y?`@aUtkHI`qXHe*M5xwP&yv|RUx(0L=(NVeuhY+mV^NAUNmKR-y(OrXuA z<@$==r2k)lIr|F-O-MnqJ3HJ;jPYyp>)cq>I9W46efp#nGqwFD<9#gpFdqZjPrK*! zLl0Vb@B4iymDE&^REDJr@Yira7pvc0A7X68FKCUJFb>+*>_iXcSMn9(U@twUWqYX?gy(kaT$w-(0<99#}8vQ5Uy@`_N5oOy_Q5UjydSJ zL5L-5F>9_5<&$xEYEINNtOb-Xah?h&K)RkF9GupFCTh(`{hrNr--b6HeT^OT*)fS3 zIjzqR#?x+%@8j6K$HJq4dpiES9l$R>HLSeLMs`sC5-9?`fY{d48aSS zkbelHR}-z@^Z;$gi2M?eq+ReG0`il-#nqa_?p-C}a2pYgk!GFAg0JJ(n;h;26!unp zdx!-egc3-s`pqiZ#2f z=GW_UrypYMZ2Yy_;$*X~=<4iqgLEJcx!9dt*OX)PoMPt{lv}vU6Og+S)c=0%i08~t z*ybR6{r*d6oiBv9$SEFv^7Z+35tTHJLmCKCV|HU};JzCrLtR}D7m$} z$T3EK328U!YCJ~;HbKxEzs-o#qoBG&UbZ@czBHbl9g(3+Nd&8>cHk#wcY=-18;o!^ zf#PJTm7v(=`|8ZtjI%o-flki@<%>mtJ3>vpq=T85q%1lK$xH=`9SA8`M8DgWkmtY9S^4eI9r(k3KN(_vpTDN{zO{ZUneHwZ38AaG!V~4(MI} zGOp`Jzwtb5U{qVe8c=&|+s?YJk-=NeRz#|zSN@4D-KUP*nP%vec(P6NmTZymU2zSB zZ9)y+NvTND`%@SGR_R>+5U&1O!15)OXtd(}!TC{NM_ZFjYi`Ln10FRi<;uwuV0B^qzRA0{afVk;|6Q{JI-~VlN=0)U2S$d5e1KVxJ6n)f2~op zP{yt(XFK~)v}icZqQ!?F+<;h`+eMfJmhtHqJA5-Ooj0u$DbltwbhlFK}a17gxZRJ)gWFQc+w z%+|;^4=glZIid%|_yvtf2XJ#sDtya@NN;A|4D&o8 zF8j}ov2CNtex0N2^A$Z1>wAbA~QmI0J@(fO|8z{^a=2wl$19@}= z`*Ur%NQKy0GXNRltv+oK=>Z47HJi0k!i>ZARl;~lhsywO(TSU43o_p4lKTc+2dySR z5Jw@zN3xSu5&g$I*5oG?aKWkLjj=O$PcBY`$;g>D&`x-0oZ~092j`E2VJ#CAG?F6# z4EI_LRNASTJiOD0NG`7ZJZtY%}>;3j1dSuVB^H3t1HV}^jZvKNEu zp>NAu21T+?!`J6?2zfA-2&*pnD+}8{Bfj7qb>sSLxptB6%#!)KWrkP#(Bi2ZBI3LJ zlS3~TzI%ePsAUmmBw>@2N0H$hpLVG|5%E~Djj4@q8I?hx1MdpFtjy~b0U?*J&Mc}_ zkwlD|17$AXfGgxUsUODmoUlyKyX4XC5;wC}tV4N31|7@f6W^5a}Q+a0oy`nPa&GkS2_C^9;EgJXoz5p zM*llWrfSLodiPzCua+oz8PUxhJwyopi%RjU8j&k;IeH|l=Dr#1LB5S#TxxQJr^}Dg z0SX4dzn^gVp#@k7R806VxF1@CSFWfYWWl;#&iA(gIbfSf5v9sB?di4YVWLdQyb2G9 zl)~e%flfxoRZx_d_O|kGQBCv()JWmAg(wrqRAPb?snt|te3fYW=7=V7!wEwk+~3l{ z_i4PA2GQI(amMl=*_|mf71P5~N?aTtXgkP!R3?ybAFA=|UGR#ltRAH`YoSv5kz+E8oRLx8JiUlTebj3Y zmS_=A$Jltg;H`Th?mOQ zI`6qlVrAAkTr#*$?p+sAH@}CQyx7>2T2}~U`t@7G@6|Or*@VQdL$Sj!aw_?5s&<>cTP}c zw(RwpVA{-gaAl>GT}asOsyblZsCMzv1DaE@cxFvyf5hkeFokqYv|zn07#qARX2s5orV@r7qUs!JOVEO6qGDwo^tsQb*|FlqN)PW(z2*mi2sQMF- zq@oFaE@lVlY_*ES5%~3$3EN(GK;|i@eBq~x`yMNY7G)CQ!+CE)r?9>b6a`w%wHE}` z+YZs~w3@P7caxyV81aR$m8EbxnXDCZ9Qu>0u2a6Jd8qNWsBu5PwjFyjtdL4^)t!?R ziu9VSu{ol2WT<+g;$hXb>;Bqmx$d^E4%4%uv&%GRRFM#m*dld$n4YPeYBy%wpcSbU zrF8d7b{@=(J+NDzsA%O+|1_+3@#-GzmENv2t1xWI)MtoW zd@S*!o21~buj-V|fGTF3hisN`BL~{AP(@RChYreHTZ3rXy*cBlL}H)xJ-aZ!(;leXf z6IiJpB*yA+g~vXWY#C;?{ycX}AnYe#G0xxxO_%y}~nrjpd=Icr99PvBXRz?KaRB=uqAN%zD6s zIIBz`sKf>bO{8f`7KePTHRe{N2%(_d3mN2{_7OT-y*BvKEJ``loQMt3S9{ySI zx~Spf{Ly`nC}{z?B{-|)w9{U$Qc=kI+&^i@1h#7OnF_pwd*0!>+Nl^g#fa*y3Up&2 z5$DDAm1=QLOsSmDK#oJp684Wnf#Xg&S_4uzdIc}}ACnGhhWJ#+3KgflbO(E&w4BZ) zP@a_~e(?UI{U_3`ObQxC!1kMb$C4q>QL#xFOIoBWBcUAm@}jQ>{?6Om9oy|~Wg-*? zHcVG?tz@D^HlsXp7n^ltG=Dy|oL|Y-SJo)@nS~=iKR>w67Cc5V)=NRY8%=Wc5P@x@ zYO|Rx-ajuue0JtivYr%o$C2#F555zFXZ8-f}XXlO{0t`Dk3mRPDl9X!~6;$BVU29sm8}Lap%*lKpWbup|B# zLUEPJ>=OmJG-Y^jxlX5Rhc#`CyU1#h^fOA%XTr^M59Z(5Uv|}z!Ju+ul7`H>qI5>V z{Py-2H5+yNQ&7lGSFODs>ZfAyW^%yqp2rBtMslOWrHw>K*}cAwNIVu^l3*I;@nnC zxk02_L(ULhwo>ESXNs_Tx9w4pI&Fv-?v23J~AF1z*5qS0~z z8Jx~rJ)wO+XxH}k=cBU(?E0CyqYIMUbef6;U<7wy1O}zpTzBTFh zw|eU7`jGIt?(-8E5>LnpKP)vp(!GKMf#5b0%g|`tRUg)fxms-T=S!|5 z$w5?~!K-&B@q<0b=`zOJ!qG@~4u5rgUXmGN=E3XW!p>E5{h^pt5Q5c&{v?CbrL9aQ z!~G8j!|sJ|jdQkO!ua}*-|O<*6L)g@s2Y?avyhj=%l)^@maeoyH=_5%e`E_Avr8{< zC5HD9^GGScB%;~jQZV9p?kP?aP%`a&?qfOz4<_Gr+S?r+v~NtgGQf+J4@|x5DySUI z?DWvjX3va4=(u}nL$SAdp9e^?c@Z~UTU$F^=^(RG%-tG4;ut;EXJmwNN|=;KzW;_r z03}RJHI4uJS7b@w1I0r_86Ko68@pF@6-Z){QW*#sQras*(PL9G*cVclN|jnJ|xVKY(_UM-~;ZD-1_OU^N1y_!aH-M~+`6U(IbvwXD<`2nL}8K6}ADce+sV zxxk6xL_UvLPxo@YU;Pp5h(<2O(5X(u0`h}uAB<(6fmGeI)~PkN4706=sR2pXPmFP{ zTqmllT(Jmb3pH;h5-XgQ;r9?c>hs0Cg9IYyPR@8ELWfV6G1yJq>(*!Pv1VJ5KjYHL zwz9B<`V_)-=e*?i$7`dP7c)NS8?FR?U z>h$n4#1)JqM*Hx6D$&i(eRbwE^Su^`4JU$hh5%EQC#L0BU9 z()%g^>}W5sJD+(t!&FwwdU6p`?FEiiREm+rf4sOGv&4~vE_ay0XAe~UOChYrL>i#T0PE^wor$Rj)_Z8 zMg3PR<{O@z%_==P--au?qcsv!wVpN=zp_#%=gayvPFI+}JjKM8Y&wDAAJ`O9|4fIf`QP$lxh9T@jnuxGfsX zqkLQ!8GaVgge?r=C%GML4!yJS(=HB{;{VNOs!r0C{;N z;jOu`)XO5cDAVlo$(#1@;Ld}y!!)4k6*+hkOwG1WO2jgt(x~ZR+fw9W{ep}we4A%6 z%fDnGGMECrzG^!W#R|gG@~qgr$WvA{XcpIZC7nu63+FCAEhHERJm5&>>zB-yegWH{ z@CD)Lp&NjY6kd|?$KUMe{isKk%iW}GT{5P~Z1}E;CiF&gP|0lTPo&j~i52xbQPg)r zKN+~R)<G{ilzN%f6nIkfkR?3U)8!iu*( zK*ce5He`Oalg(89BUtQn{A2DEU)dqlc&|z9;d?aC?X#y>3{;{YpGD)co*BRaoz;=I zC3IAWVG@kEEJn2B#ZPzfTws;-^#R?U?y7};3vg}^Fvr$L@rv(;d^W8z*dI5rI$o>{ zYuiE{(>$Z;DJP*gPTo;fCg!iZ?s{p6zOk^Ksnh#b&x3<+7%_%90MXU+#ahl(?i{zYKyf;&)j`>U;dfglP{d}_7?t&J)CdgZ|ydQG3eTOc&P|wFH14o zGqncUZCR{bZ{vy74Z*MUGz4URSDTMijO(6Jn`*_)A4-#;Ll`A*3O1{y!b~Mb^EnZB z7&=rcVZN0sn<2x!sjQd_j3t@KUNE)D^oxP<}8W;|0!DhT+Sk?f%*5 znRyj>WV+JO{SNk7?;3(4()njaoWV>1wDw!L@X-cmHRen@r>wDG=bq`ep8;2-Dqd<+ zwqxIj{QXxT?OZmbJ9G+W&&^&XxrF_HBv7Uhvh)WgyIu#YB0hW!ZN`}Xye*9sR9Yf@ ziMS<+Lt1~8@Sb#jFzs6wYBK-WbLQNjm{(l4!8EbVCr7Q1#qJhY)dqZi&>wE@^LVCa z`H?HG8JSDlhvuo-bTC$%RA?OyaT2PLTk=;sr?Po?X)z*v1yfg2=h!5&i9}+zjHI;lydh+=?!iIi zitNjZ?brp{&}JcwM~4XpafQ`|4H`DujDF%-{8&B$eM4}k{z5uAQ}s8}>>Q~@Bd})~ zUbD*|10tY|U_h{&3%$0^fv--&5+_Qt1HPk&)8mby=BlfFS*AVN>p$YIurjP>g^9_7;cRR1DZ(SZb?gc_INnh*|d7JK)8gV}=FB5Ult{#JZ*E&o4Qd&{6Wx~^TA;DG=kcnB8U zB}i~f(BKxF;O_2DaCZyt?(XjH&fxBYe$6e*VMptZ&`cqWxdw5*N7;ji8u4+M>r|GWcV4nn}kbaNg=l0rQjb8*O=M0_TSA-jBs=bFd;?`DcE4g<@8XLRL*AWYWhPbV6Dkv9T>VA)UZpfdlEe zpNX_4S-rhtq;MPxhQ&0LI??LAR~{vyq#woQNaYQgvnQ(dg zPO5XF4)z3Mz=~vf_WZrb?{}ZKRcII=I^m^Q1aKf|Yd@A-UH4TrV@3%s)H>3M-y}lL z30*m8i`L_bTc?c?b7#6*rzk6P57gz@kuFC zujeI)oA5N}YAA?5O28V9D5it3pNl`6bc6@@-HY2|Dk2K|T(^KZ#moYG4}o~Rg!ig8+Ip?)qd$TTs*9^`O}#B#n$Fs0$FjRh*TK)GeLIqTE9w!^oZCN0X;sHB`(({ z0!INqkgA3yZWc3#jDiZ@NvySdbtPUTq@@>(HYYONxLcX^D^)Dt0;cNPl?gjK@QyQq zLV~n#F#Z^3Dl51}Lc4g`;p~hxuwqg;5AG_Eq<58!1+U)DpF7?>8rwPP^4xiP*1nw< zxj3osv8GRi8FHXQEgTLG%P7W3l!2_@j%qzk9IG|G3%K2pHepMKquin0@917!Z%QQg zmX?uKrghg?$TL1Ee?9aY2`&+sOV!6B^QKV9F8k71+mwltPkWnDQnN|Ibe;6`RW9J} zz2g??fzRfOm`k70bo4K|eAYU@F_}CMZS5Y>wX`bwtoJaN>8y;GhjJ2s50iC^hF;p0 zBWp+5CJ9l40fy_e`+BDs{eT2=p_`+Ur?P+>*j;Ea#20B=)dob<}mBRc8p0bRL+(mNs9-wU`Y(-@19(}zL zmb|gdQT3LV1dNPI?A9j;_dlpjFDN`5=G+BrZ|>%9Gdi3ep zTuOgm4{<=@v6@hdea(zO4vG6+SvxNs2m3^Jw(xcT(pZ7|$_T7^1hVzcGC19seH2{9 z`RZwI;W0Y&tmv!Rf308Cmxrh`lCto~Wx(oT{#NjG)_b)Dgvgx5{j9#a6 zGCmX$2Bd_URGprfKv!N6b*`kEm?IdunWelz8w|M@&ET;!0RN{Jk}7is@&vE4wP~!k zrdJ~UvqW-K`25Hhu=}c{(cqR@(Y(8$L{|LTMzqxwHt!C%Gb}ofnq2cmxZDdh^k`Cl z6n75}%x5l5E}#(d^5SJjp5{QhV$58gbjX4W!TwEc`kmc5-ToV76Z(%=_ns&`=@e#d zdN)_NFL9sQevy^l%tPSQO7xvj^BA54X`~B08w3`8x;JAoxxv^uo&KF8Y@^|5@R?ws zukY#;B1CD}eBu+l<{(G+KsdrMIQkP%#A@PHGgzH~X*q=Wr5^UWXG zFg)Hm2Z&X2s<(2btJ)n%U(5aSMDIkkP{}&4CDlHpdDdgN|Aj$#DLS4|aV%k^HW-3@ zzL6L`)dUw732(%gzVCd(ll2a>(TqwTEHJy45O=%KwZaefT>M`9#F1*(F)0e+tRRd*+5%wE6rM2R0Z};e62{WIZ&`+Mj(Fnk6 zK|7ox$T%mld>ZM6MeScDy#i)|gJM28W;iq@B{EAY;=~#=ak7MSLM1e`@MG^C_S1?4 z#DzlfNj;J_bLVtudcS>wY3hAmGpKa+VAA?_y!u;1kazcTjK^SlPsWx)jU?+ErId}| zPGJu$U~bwT2{9HnAUe0Sdfwb&9~un*+WoHQtPwIVf7t#9HT@E7kcG_x5Q?o$-;y?< z(uDiqlzmu(Hhuosteju`DZc7(uB>wLsF5fkb!jZi#%!S7foF5bD*f`gsN{_D$S#nO z3u$msY_9LAmr+0E%BiFfd%VnTF ztRivI3QlRWdsK4Oxqqyn7dDq!khl1Li8)jL=F_n$(T3!W3Eej9*hGbAFRS6(Dg9!y zN%_@Uva*3b`{DZ5>5-Y2XLx7z)~Br-mG&0(SqC-c{%&DE@xLmH) zTK^KZX=ZmP`1G*kVKyEA3F<$dZgLbF;J$(vg`44|&P?vX#7yXnGa#SCqPw3vY&5<9 zRxW!i{h`c~n6)AT4_zT&y@AGRBkygCTX?rPr}9}BTQQe>0`ht1p{Ocq;v zhw8@(m4aIn+spc{T7p*X1K@zaNcm>23XS5S2<}QJ>z6D}wLjtSBBu=vN{F%uY+60X z+&(=}dQ@_5A~U}2aV!dNB1Xm*9X08FuYUeLDA?iq|w~5?+%#x?nBj*MXErC6l~Vj=tNTqH2TgYLHUv-8Gi4u!|=V zM#x;<#UV;GXbg6AxnZc~(S2-iUV4~OM}L}m@0$0c@o1Ip4z&BzFQ!XxajlQB9^%#r zLLuKy81b409veaLCLBIcLMTAbJ3B%@s)vkLfE47p{tzv~2&9e~#T3d3*Q4L)E zFJ+%J?j`h6Gb(ei_AKv4hBc*B(iNTn960hi0G4g*!&I;WdicLJjcj`ngVs3?U52rB7J)08U0X~)68@*yghf%6`cH+)0KT(p) z!LGMY4!{lZU8D=PJVqDXO5GY$70Cm`CA;-b6*VT_ zbJBgAR5*2&>kQ2~<4Kf)eQeR~`O2gF5nk8O9a>}nkz~>@Ng2JV3xe7r>;45O}90V z&=@v*p;b6s^1@MyJS|lvJdP;+csAl061a9`Jsd&wJ6NML2XD@4zQ+%M5AjbSp4|I( ztat(cBjv120Z+53`jgZzKC{KPCwwSpis#IgqD@ugE30_;Cd;sLY6phJc;V!toWVTT zNSr`XM~FDZ{5JtidW5urU%5TeLX+hORz%V``y0Ng9#orni*I5`CUUP@pG>lZjMf+_ zs=Is7HLzY6DSb1c8GMj28*g-3UqA5@48?LR3T5j~+xn)sCNTOqIx{FR$&KKaYQ81W z0>RF#(!%q+i2j-0|ILK2(5m3Ea(eIRMdgf%+)^LDod*8`m%InymvkwCQUOq${mukY zyWc4Kx&CRe1che5LB`l#W31t3WsMPeg4vkRu6Lw;W0!N!%pl>VUV}N)awIb}X0hy% z0BJsxuLHF%;^QA{Q%8)=2nkXHC3h9CZ3$cTl)l53I}w%ZDJ+`Ej4QIvc+r6Mgep}4 z@oI3oOhZ%{{BA7kt&`+*+RptLZp?B{ZJ@x!woi96|BHlO246_7hSP4?AgRGI5ZAx* zZiW%>)|lB;E!Z0KqUSczMvaX_wk0tooYnBj{bQ|MHp)tRC(wN^`_ufw* z{an|+r0H9SzEwl}Yt;rUJgQaQQ%tADX*JXGA|RReQ%cI)lH%Nwob$(v-nob<%@UUR*q8 z&2m99;5Npfl3Rmq_Yy*KyD)UgaiI&ixIiy6Zt7gIX_(LG`AsKt(sOb8ttBd@r82w! zRt+SvE6m9WPCJBDIw>lU&C78em6a<{u`7(J%KWCH4@awVwH(heI$moi!g9Ige2yRk zORI2@5axoB#9Zj_BgT_v(D|E*L%Il9R44^w`QE$`nFBUkBhj`xxQQN3q+Z<*^mtp3 z>MixVAH?D#E+?<8ijG=gUQ`s$YveQt=CZezPA^nYdbl=Do~>W*^g8KM$d3lqLvBi) zuEKZaTBVE2?H&3i9TM zJnhaLZj`ZDU(3$wR>2^&WK*1~1;fs8m$PC1TK5daLaMy)I~;i0iuc4G6(aj{Q!E_a zccfW*zRAGjSdzNj<`#P_MV+<0i?o47IQ2VAKAle?lzSBd4lIy5p$GNZ8owU4dD%(ht(^94R!z3MN1)_>mJhiy z7op9-A?G+Ck&okcAbg;=&>gryn}jh;zQta{sSSTYGP2e(;+<#lCxzR-pJ8eFfiA)p z<@f|ajd@Bknjp-h6*6{sw^K4nt$CrSz|D3oS-;@92)ii|cd_KpVDzSjt;GLbZc849 zGxz+@98^19B5`{)sFydcU-R8Z3;f>H$F=Z|j_h+@ks8lq0}1*#@V0-2L7`SX-71zs z{8-z{8uYQ1S+gN2W2H>D+d^wKdq?eqQ0k%4hdf+M`&a~|WNjR)_~RLvQ&-_9n<$&8 zC_3y&|K#e`9yU2Vzrbofz#Rx8P(nkJZXgiUyapWv?H{#3=C>H$-xI1RM0t!c8>#Wu zCa}0{-~QmJ$+{gMy)()tI8Emiv_5u@VN=sPb-3Iv5S@0kCE!2xFI8^qQ*sE^5mW;r zC{@C(tyO9dk5YjcZ%_51U+>IM{T2yZ48CwM`6J9cmWQF}k60WQFgPOMU0@5ib6b8g zL8g|oF_G2+8{6LLzITrd1|Ob^N6eiIAbhR;$P|$a0u|MU6q$nCoP6In+U>@=mz>Gs zP8j4@4AhTi%v^r8U!f=|vd8zxV{>5C+2G+@boWu4qTpViex1~&hz~!(R0rafGvOA; zhQmt1meKdeL=erg_1nUm?d$kt887rJOvU#uF|j~M6}?o0wvq>7nQf7ujSzw}qUtot zXq6tw`twQTmW$k>9D~ZdcC=NzPO|&PI4E;?;li~*sSW>&!Bc-GOz_WU%Sfh&#F^S$ zY9b}HdQ?r{JVn2}u-aT+Dnty*IfO6o7R5?dHQNpCIKVMF0I{*j!~kzde?#_8*juvx%0hko3D zp5RN`Hs*}}+Gsgz>lE%pz6=ZHcKcZwq?hi4WnzrcYil-Kdr_**rrz{DPaX%I|drW>3>RV)h*r zo_Sm|6L?MB!uF?p4buOOu1BJJ|Kxq0vtlcdkI z!?+pz)9_;L@0AzfhzAkanZd0$(W(#NHot%zzC+y8Nz%i#P0U{o)Y5Bm63dC#j}hD#E zY_$zB6&7Q5oVQCac!%MT&+6D0*2K%>S#{D~$6A>5D}pJmR8zBuUG3&GmIICE_1(P~ zhxIXhVW$!$b2`=K9c08|UCBFDYmbfJ98Z*0R5q0I*p&o*##iXCEF*;rC-T2gAtKWG zK~q%)eZyslU~`%6W-0P`gglI++0_nvUb1gyFFd#$ryP zb+FrXsk3cxVFfD>LBlt78A#2rI!WVMZgNS^eqaxvkBgDT!%k()r&F>ZzuZmx%+s5K zQd0%TIxuk&?19*qvAgTK=MY9XX_%oozcV(qX8sicyfvT3ht( zx01q|k5u!F&Q?JE80wHHgb>W6#WZV4%NNPjHUndFrD>ECF%KQqu=z+n>r(q}Oingj zSsgU%7ksRK%%%(DD2b(7n{F}|qSLnBm#9!;(BNK!2Yq;ZL@G5%WZnG&Ex#K0Cr$LW zT~UQq<#!MI-P>#+qx1gP5B#;j>~&WbR*-syYlLR2mL#;$<=qpOSZLJS5X`j238dtJ ziaps_cE60?+-Ags2dVnzRKzfns0iT)_2&DfKP6hh&-U)S-L4n7tpoLnqBL}heweE(Q3+v?%z7W%`^JU7*w(uPHjLvA1I-|7{7Vv zZb5?8MYl^4n~b!0VQlNL%~|YPhI_%@>z6URxd;6=*9_j<8$N^wSB#25#n71ZQwrg^ zF%e|%w{zH8q9@Yv&v*b`HH-D zxdV5mu9D{)*7-7hAy1XTP^t?IVQc>n$>?OJT=PT0v2sg zR{iwxKx5(~=M~QFD<1nSELz_z5P~50f|byAcw~+6nY9 z)uKS!iIOP9!$U{xEGLad$kO&F7xf8GPkK5=BjMtE4snl`kHr&~;cBhV%^- zpvjJmG2|fGaXUeczWRs|j#sRpjBT9gOC$i718sOw*G+?8CcCyIANai1gF_FMGLS}Z zNCRdNIl3T4LJ?GbaD4_eca5lZcJ;s}k9mjpTWpbqSGDWKu!p0`v7XwQECtjhZ!0No>4 zwCzypdD;GBs*awkYj!xN(tOR8aBVQ0crY#9)%zyoG`hzJk$*5EHjMP}`X&GbQGyf$ zrS>oDpOfRu4CZKA(lmJPoWbjd`cj*T5-D8qEQy(aLx`Ncn>bAxTQ)NoXTRUrC3fSg z9}LT>0u|``ZPw#DC*Z-2N&B)zA$Xs`0~s}@l->Ai1Erw03hT71g>ozM-Om>|RwU?w zJVy}1KAjYsK4Sl*%${9|N)T>X34e{lbP#yE(2_PIzEkBo;2Kw^GS-omvG?Lv2m6Zd z0$eYC=-OeY`h6F?KUt;jimJ2DqLP~W7`zET4DE56I= zj=_QRa+N%tIvRR|SMN|gayaCNEmmi&vH<1go(ihe3q~3+uux#9xSEwMQTM9i&o7bldVvA7!vkboE0{<+d zqdBcgxk#e?pG-P>phvEIn+;h#S+8dm`D)OAz)_$#tbo-u1R>bp#n#D@0qi2?Yukmr zd@11~F2Cb!3cDNG*uM2;m172wJ_PU*F&W6z??{=i=mx;GpvLuRX@9Hm_j3jgn!w+G zp@1QWw9e%)0Wd2{fBjS+W;b}L5pR&Sfdx&J*BJ&$dt?nfF0cWW_WkX@jj`7pfaw54 z4eDrMbYf~Xo;TGe;8Lard8(OzcCzm#)lt3-ql)}{z)g;Tc1B}*27LW@4~hl-HAG;l z%P!M~P$Idz2N_B^cWHV4e#}#jfUHNnJn$_FnpJYasQC$^|@%WS1`X< zf-!;iq(NiA7l8>%s0=3<@&8(gHR(dAEG6^9dS`Ow{~WWOL9>Zf_|7Ux_J5kE;JM`6 zTYE7biX{L05OcNG?-h0TeWUuDZ2#kh<8$pLm5abB#a}-yr;T$y)?O|c-T|O*2}Uvb z*tEb*q^-kJ-@Vbg_75GT`J0j(jCcT?Lc9QE>;N!O(i2$`LO8aUj30Ed|G#jcUt0&=-gMp9U!We7rL$#|z0XKhKh zjUm}K2y26i<_|zM&dkSj6sZtX z-OuEv1jEIBCM1F%L)THE3Usirx;#)WRp;UtYLsc{*P)c~l9>yNJujH>y+~82sB)0i zNAy06aY=}2KTeJhY;;U%Vj~8~1T+k$DDj#w#=mKv>k0Rd#fw!2&bci4=MQ&gyBM81 zVq|>x)o?mIt9ZYpf_-0A`=G)nm#XB0%fYTfW%W%ih}lhiSKMwtv5`p z7%Q)h00A*iCD`V)54cYyKlKzNf4A>dWn@ET*{NP<&IAj}6o}>0ivcrLRXGs6v3N`s z?{JA#{;v2DG7Cfvo&vCR9F;8cfzC2w8r>-q0Jy>Jo6w=ZX#u7P^_mE)8M>5sNrLcQ z712H{;7nvfo0P|M^QjPE>oK1&ezM1W-_qa_wrigUcn4% zH{4f}^^%G83zB>S{MG!NOiYvPHhUJODeV0#s;H!_Nk0H8_$-PEI*l327tF>OHQV*JWRw4KbJRc6U?}fLiu~++;oKc+ZxtKUG@) zFb%17*eR;Uq40Q%8r*Kk^E}eN7s7AX;LMrGd;5*53Zdke9-R9}H3Cmf2ce1skq~QC zfS(p9sXCr%7+65?BzgEp^h1L_``!G5i77N3tNOz~*;JgZ7jI{9?Y0%Wjc$scBOlHu z{_d|q@hVT}Mar^dPs(!rW0<*Swp#Z1Yg+2<*?xD=w-42NtxQGZRA9FCSkx~NUmwTP zeW6Pg={^HMRWpB9sH*`n0&%$GI(RKkH258%k_#w|U7`EUYcy$16xSH|F8 zYb!wLn>SE-XL_H7D5Ii}gsbMjk@GlJ1Oe`<@%&2SCsjvlS~|Z8ak}4t@Z}OVC*1=( zo;wLxkWA!gz%)+V$N^n`S8Npy_{2YPSK_t2QZEku9fserFsFyy(qW5kvreBErp(GN z*EJlOJr2#cPo_o2ppflA}jc<1pUv*`OtHrSiiBg*3a@?A1Am7yZ&S5}mtPH8ACtGV{74fDB)b;jPBfP9 zu9Vn?8?tE%6yN2EPwYaDMk*$^mHj1=_Lv!ax!UZ`L(jp{k|Fh^YlOx8bzrySYy0I4U0ifhbE zrUqvEoKD=6%aqZ2K``F(QdL_*|okm1x>dL(p8z<5PhH1hSd%MUVty$nPVhr zqDWOV{J~eC>bfxD`Xyi^a={A_>wq+zisNs?MtrxpT{rLy>DN5Y6KgKUvN)W|56mB zlM?EF?U<<+=dHZq9=+UAeVR9uL8&U&gUv69M**53}~h@F=k z?kUVRh5PqAPB+W2LNga>c?P-^uOx%T`I4;xEk0zBw;-n~@(@LLnhdNLeb(~3pDJA9 zTbq6!$%B(ex8&+-#E9O~L5VfL*cWRj` z(wzK~?S=-uu@IN|3%b2ipk&zjgo_gu@?X{NRV9`!c21VGHQAznTF2oFz)B*LSlpA` z3QILq{<3=kbJ+2)Cb|BkaS<-H?Ih-*l5}8_>x?Vc@{fMCu`dfKSLt-@L+LJ+Yj~dl zB{BQ4fjmko-uExP>2D=c>_M!LMY~IRkw|#<%M1W=%li#)*Mr1hT`nSo0 zxX6@_WYb3xEuI57vi{OP|CyYlVoE(4%IHb+8(W)M$kDHJa7o}zS+c--KD|Ki;q4k$ zJSVdxCcHnq7}OL~L52X(Qz=eYt%YC=d&-?XR-JL{Aoi4@!oQ7)0eGhWN7n_wy4Jh# z@a8$UpybQ6!aM$S^mO4JaBkc)r0wBm{j}FY5k@}R==WE4ZF9sG4#VH>KLNW=2x$x? zfMr`BmE*QL+~vRyMy4pOD;FxGsqSlZdTUqwfnwQ1kIJ^0DMDh>K?#2TDC(Wj`4p}xs zRMZOZ{?vr-4*NMCQOoC*`f&T#i%9;j;C&ESVU=!&=Cu|QoqyEY3Yt?49P3pz?AZe7 zA^ta9zoe4fHeQUG^KTZ6;&PO$I?ng_4ntqJl8OKPYhk>U0Gq|FESTUwmei|pW?MiF z7-#<&K55yiB>U|>c|>&nj&$F)_u(B7Y72aqtbmV98~Jx5HB>|BBk5SOV=op&0Veq2B>o^jVvpH(VkEpvT$9l0Rd^rAx9k% zBtV+OLooUn3~CF@ta{~#dE((6u3ui8KM@qvTlL80DGQ6EV_G}T7zTm}{xZ;CR&&1M z&6^O>efUj6CmP)1#&!N{kTiI#CyN&!BJB-x>D%&T?)c4)I-IQ~ks-%-K(Pu8+F6HT zgi?gvxCtDTOA}+z@9}%ddY%ySi~P^(6pTo4VG$<{{&sTuPXXQB@q}EO&Wt)Eyhlq4 zX9mYt8iU?a>117ZP7|8LPmgUMy;P&^)t9DnOk5gSi%jF_?z>=MdhFiBdA2j}UkrHn0=g?Yobkveoi^ZS0y|K;bE;w_zi;Q^ZkE*@VIp^e?MB#ljraW{S_<*zQlq6tW{eiTg7; zBwuVcG0}2}VkI6QB$37w-mdB>*LNK8Ckk@DU95@n+j7|`RjZG2vb%BaISpDcSaN7C zq>(mtati>g2jCj3opIQF^E+4rrg&44%QLJ=)yu1D-r<6+TM#ngBMfSGx5 zo4U8LS~X!Ynnf#H&05_A!M5E$RYju{3#{dzLZNmH4Bo8OG!icj!dDkR>v1|j$?RB^lqVL`-NK0!@B)n4zs7MG5?bi{ zBedJKT17DFKem=Qj_Vkr6)x=j7`(V8_>sUI`*w}iAf1UGj%_vd8EXhjLaUsq=&S$i zpQM(H%?Ad=w&X!yQ8_T!qG~&8@{|)Z)LcqI2?-mh$I~Er<<>qMSg5|H*Zbw_PY6*@ z#3dT;!|zit<2D%YZ&wD_R;yd8f2hjTt?GWCsGI6k`NrW7$e+)s#~%U2*3$443q_u} zywmC2j0;lk9_TNZcc-J9b{Fd`{cXos$c3jlP{w+><2N!Eo&qi$lX!Mu z(jabsMJu=N(dS}<_;#F9pf{8wpV)YRP0Eu&_YUrJ2*exYNhv%2lsOr&JmpUOWl^0c zK6ih&&1J>yjq79LLW%aGw*=Oily6OBj>Q?0GRO&fw&bS(oTDmgJYg}hFUhvI=pDJ9 z>pdXTMGvoh>bby1r?FyBT({n_Usd2K@1+>xz{)(r{&VJqJ@$($C>!illLhvWV=iw$ z)or^I&9Tn#5&YnS>0pazwSm#6+((c322&=2+F=c7&$zCH3RCU}{F;=3J@%tRQw}4? z3A;Jqtk)#qxaGv`oJx(d_nc*<9xYucrOCb9RZ+B3yZ@m%9Ath`dvy546u4FOOCnD$ z2F309xTF!s)~b0fsV6GU@-@l>>yMoouV`fx=L73GZT)w)S);qTI?V|MJ94e^A4y}E zYSJ1AUX-M?5jvyG11HLw`VF_GB?_hsByEN;T)c`j|7JA{`}H>ft_UB2~d~wZ?7MzOdGG zY&~3<;b+BiQAy#S?;K&g4tE4@YoEl{@O^$djuLWH$d&mlO!hxaypE@*$xU@q`rNkO zmS%B^m9PoV2E|a+psG=q{x)BS40Mgd8dTBc4EmYJZnW0RbRP;CraWo1WA7emzms04 z|3PQ4Og_C<9gfNQkz(~&wcR;i;Ib47hua<62Xvo|zQakAUpi?O6>bztc7?_ucNt5foyV?!o7{J*mF>z7yOiRb>aKZgbEW0lR&A+dy-j^U)<3#KeaE_j zl|VVo7!eOFGkVsVq10|`R>HkCnk$vb#4?L&zUPy3@fHbuju=gcm)FrEkq5p$Z<^Nc zA;~p@4<@tprH*(jNvC(^HX9EX5`{N-$%ixcheAMIcG9?YkO#xewA4%g^NGgr&r2-s zmYnqJ>(#*(dK5nKR%J3Nlx8jYn>FXQ0aI2994<=0Vswz>z3e^3rnxmcUAjg=Fp?|PO!?VLF3g|&41B?5@dhDpk|M6`h>W-97vb|DjJFSR#a266 zn3_D@yD|VCAV8vevT{yaR3JE@4(%hKgDai7P)T&2GLDd-%edcg&QQOd>_&wL>U0-k z{pgOOU?Kv091Av|&l{f{*ih0*YbbxFYkjs{bL{V`O6`{6`M4aGmR)_yP`6T-m*r#1 zR%nLoL7?bH(WyKJ*{y*)h%qy5x8C^dXhxc;(ZsFCCPXoiyv-UuWU+YIXIqTsE8;Txyp#V|8?FN$+>ph!{LjQeqM4_OF#AvAlNvbp#&j!A*Ilu z>K*Vqre%VjMB0S+3KQQY6SIWnJ~8lO>6&dV+N(6PPMJ19hT)hub3AeS*VqnC6nyZ| z$9uW$Pzxhmp%{K@MzKp7!EEQVT_W;<;eoot-e%xaJaa8g(4*NGxvnNX1&v-np zkwX{e?#h?lF`Oo%TF=w>`HT@bYslFRP2kmS^|`$-c@?GV1O^Q}{=tWLd#>E>hpOML z=&W_FynC+Bl~P>90|(9RX|NUJR}3sJpK^x=j|7bVtpr8yj7!PI{a^2&u0>*3kHi~1 zIUq=m-hSR_X9RtIp?vA3J+F-RU?8#P+2hLHoe+<|fUZ94o0#?`UDF;5B=dXJ zv||YG~8ZO+yUtw?3as8HT3{VaZDplN>xth2=y7xHon6bu*jZmk!@s=<(mEoqBVhfbVX>XRugcf8GOq28yFxyVr6vb0n9=^_`QZIM!(z zb(xD#RQ)d1FpJys+@aJ>+7QXgD=?91z2((j`6jz*2OSU zpXVQke&m`5MtE~YlU@G8rEegR)UVY96Xc~Bbxw^nspvSjJb3sXY4$~yAjT|&>lzvP zX{PR|At4Dk@Jc2=%{!x{ORD`K;aA-OV)zOG|F~c;J!ml}+Vnolmn?JifSUC*Jhu6( z8>wb(Om0E!y$68BAlcAPkjkr1#MSsx`>B6jtR7XHs`xM1?H}Il ze`K~sL#xqy50zItjDbc`&f6?Y;GUK&<>BZ|!`b$!n=N3Z6%AlOLQs^j%(avLmy5z1 z3o|Br=*=d~zc#knMC)Ay6Q4wAo!@ozti&U=d1U-|+)K~lYH(YCRAI_u8APd-6BNqpxisd?YP?0D^O!!gG#JNYOn#AqJ+P_)*QfhQ*EKLx@K&gAXlPNI?cz}ZJgexStq zc{fT3+sYo(dfIGv$*~pMGneQRu3g1I=3Rlzb6VUH=c@D^1>i~=^GDD#m45+5_|3`EE30W{=g%pxpAu*a*Zrn7eBDo-MWsVWn2OK_uh# zY#HIpl79H`B=Us}{w9434a5;{;GXJ-(9NnjVwc=ZQFG%{M<7YV4ZbK5iF_-np~e$_&7r4woP819HM8tMl~r& z%c*~{;du*gt3jP~R%z`nFSb!5aDQeBf10bGus?&>pgH3#9-A+Nomy6p7FJ&aC15DnTi|&Cjh1t<4Jx zkuSbAxV`iZVRjnnMy`51K7(x;6EAn2wrV{05x3dm$o%gqzFA(#fMH}Xqwa+}qz+gU_pT-4?I)AjOHeAErjDw%NWTlyR z7|>vCGkQD=WC^0(FjT6qpxfQ{uz5^R{m9CmoITKyf}H_V0!L{?G?=Kz^JJN$vH9~g z#;v;bM?B9GETqdlm;S03%)Kd~)*QrnUgA^7w_JC&XcXSqhtwg348P{m&TEIVt*#TB zw~|qVdjdO?nF{e-Q2x}&-$kx#t}<_=J#UyVSP|74m2xf6Jmd4p!h4%tjm_;@n9jTZ5$om|%0X05b)u`A_zi6g>IlwJluuE`kriN)sXm6K5uB^xPHno` zT=BvvSEQS-<7vHkV9G3`9sVeRj%Mq>{%TA`r!<8KtUfrVR54xzX36R`CPe8eFD+Vo z*(jBw*@0jgyP0%eU?|IfX-3=o{!i6`>YQ<3?04SoYcICoLxBo9Br>6R_2=&jT)!9-XIy-*3QlJ35gk6V06XE=_k(^aA*+tIM#J|?cYztQp)L#muSw`(|OwUENMdK5%(GR+x z2YArAPGZ6}&d53@L_B*XaYQQTY^Cd@y|;zX*_afn$XKVb6>AH+j;`?jCU@WSP~X%- zaGV6f9dgr6jvrf4qx-OmQ{yH_2?o|Pu(>%0A0H++C+*mi$3)tg@oO|QGo13KM!wLD z?K*sa@7rlpB#&xg0m`EqQPq{AjqQ2*o&-Y?U-G-ofpi%|$}ArL*r`F)g`>tqbk{TQ zD3tF8bUt8Le0d)u?3|_v_!%R_Yz`cQ(}$?fccous8A!v5+^!QQFt%A!ZJy1XR|s;1zU;8n>Qmhr zp+s8cTt{f*#ntqQ*^jd3FIx3uX#1!7*D#;uAeirFkD(+M*UgfgsPWp7PQ%O@s$qHJ zWC4Zq{*wvAVOtpJ==;8cYb3iykJ4B5!a=tmfWXJwcEeqRC~-ApMjlk6==pFR&&+q2 zZiAJh1ffBlYI6qv5-LvQ^=bJr;O{7x3F^A&hS%ti$MpJUdoqGnc27F9CWgCUzhX{j zb!2;Q)^$>Ug~7DeY(78bok+;U`wnT?RxqQS1B9_&t8P#CEg);9{oV4uI&sKns3*r= zR@+U)O%ZvF)LlD8RPg5&k;R>bm5(ETul`gw;EGED7+SdjQ zU>S|F>~l&TVw6$_)TB*b|OUe*9QY~DC`g? zHRi&S?U(LK^GWH(YP{}%srV}J9!%!xkn%^iq}zBQLHkk)i>nOD6D^3ZBT(Q?1y!Qk zCNl)$Q=M4|hlULav0?L59WtchhKNp_NVnm@$`b^=DNsx{@2={70v#99hfN$)J z@leES$KT&6!j3vYXVYtFMF8IXyb0v4o>UP#^=R?r-at?{-7M#s^1Zz zA0H0>@tcMnqh}2iC~2FwLMAKrg`|NdM?cb34pAkvAODJy;7 zYT|+`?IB@|H-?sMs&vXH>L&^#*)S#j9Ss6H|KVBOX-K|+ zmAInt;e*e$yy+m`a@gHY+`hW*TvZhbu4`Docz`mM^=V1qx8`cP|#wucP)Rfhru-Fn4;u4ag!BR2n&!Od1hdF*6)%sPnSY*Y?v07Gcw!*}FGdQl%w&Oni>U&qYEYk?^vp|YIkxNS_UA1|X%Bk~G_o2_J6 zwQF|WC^smGMg3A;uL|qpk8G13oksSocT9H--lVUnm-Dg8FAvqDT|TphrZaoj_}}!h zVYC!grO?%n+dy9LmxP`JFOZ)D!+gfRVrRk%y;^J~6S#HiQuGP1MC=XPNTly#!@c`| zwmp)@$1}y@=jTh7v=<-e=5SvwTMTT|Gul^HqQ6A=o=4ZzKXmxU5@&8M+wE)UFCI(S zxS#|X3T;j2zvcfDp6^v=Ze1X5@c|H|d&qbJux_+~o0WL$H+=TA{ps?N@4^6%U`sKS zY^tlHU$bqxD>Q0K{ImJz{lzA(pqy;QE`ixuQLjUaywKkUvBXwc8~s)frv0?R;?9K^ z!as9P_^TB|I1BML8R`$TwMP_PMnQRIlY+94%#ZZMfkd z!fnqtSyfJ}z0Dp@9x6Uvk2ZG9;6c@vOh^fwX2<<-_if2-!U3N?afqQtdocEA(&#N6 zQXfO#%Lt)$p2+bErP6DXk)7@XIWbkts%q{qc*bbw=1!|NU#boJxN}-3gLE*@I3R}) zf86S)2EuG>Ax>LL8CQbqG;btx{d~& zO@*f2OH~_`Ky|~CZ3;yG36i(HPmp02^zMzlyS&Ogf`_+JsR09^BXGU9`D&xr8btt0 zh`shg_pan?4nlMWsKrf`$#`D^nt^tI+tKdX+zI{ka%bdN$e*)xv=bIFGS4t+*hpTIk<%^X-Jy) z>v7aj>7Gv8v=4Fj<}za3FPk6gDoQD;@$){MsG?x9r<#xFZ%vno*ANb)O z8r$e(1-V;MJH`yL%X(e+=2+a~N>hluum10s(U}NNR9-%I$qXGA?4jI|j1ZI0fxO4+ zAB3lV^@YXuiwfkWFm>CyoOS`A0<=6ju&O2G2}67eAaQQn__LC8S99Pl=T2NGLZM*u zaj5XsHSzOF0nc$|j|EUIkfW0XKQvDXW5gJYNGv1zNCnV$_OaiSWgRZ?w>1!^PtD8s zN|oaB?>XvhTPLI?wq`)V>Wp7nt87fWz3SPOayy)hYEfKCXE$IL&uW#id*FBf=nCo{ zL2d%md;jXvza3$)jaJJ(*nanmW~-mf&C6QuX5=H8ha3Fx&L(N4it=Q_k(Qv(U!WJC zaQY>_??Qp~`0+ObY%*~wcE65!lC94e!FVBYfWdrY^wsE2!)3&L-VM64sCiO_g#dX! zWLlSL>n20j%H+IZd)Cv0tOw)YUrzm|%J)9-))-A@#*9TC+1?ZGNz{1bU#f+|ml3yI z&4^6%>b+$@HCAhBK6^f*9`r=8V|6!kr-~m#79&LJz@$9wCcl{c27WNv2?%e8=W`Qi z+>o`LpJ@F)f8HKbVSq|54IkF8`ELKx<8(V63zqaT7=3W;se?3lrQsuw5$ zeXMRfX~)Jve01^>5)iTTg9vw6SE-{)9~nk~9qT}Y%;JlD(rvAglVkO}yovBCs447( ztAw|m36~q3di+b z{lMs(RwFsr1YsL&IPi$@Ee0CzNK@Jg%K^pS7V#^Iv*K|`iKi@c35JKcUs7Om+sxSuz zHrC&nPrF}5$0GPB8vMCj4Bg)~W11+XFS=jyEJ({>6aZSjBPXxWAMycI&JtDH0E#63)~5?%#1Y^z7e~MX}3Eao(+QzeRDOvUt(%EZmjJWFu`VH*7-Aw6gu(l z+c`yh1N^lAamLsRLZGzvaT*El4<>^2Ed=zO1C>lR*@MsM1+oJdC32+_oXV`?cqUx z=1MgcsM&2q!tH~}+ubEZty0c5B{fl15yz*fpLW$EM^;sRHt!L$lY@xnyZ2BWBncI- zcXfsVNCFo**KT9dTW^H+)IuL)mVcLGK`sr233=eiq65^+u>gC<AGdc~Z08bu5gqPxztX(-5?)1Co)?@t4ae)VXHrRm^ zN1o{QX%~+|Vs829l?HDJ&cC8d`U{zB!dg&?1%`T~BVK#g)VnBAX;=Ii<@%nFHNgI> z0%?@>>Poiel22OY=Y+y`Qp|-x6+HSA(JJA|y!Sn~QJ1SHv6SME>!9^i@Y?znb=&O9 z-@gn2*wo}ddqG-)tY~|4o}!m0?FeomHI(eM;rSbeMG{R7+U!p!7}>*RA(%OYCJVQj z2FjHWVw<8IB@I00Z2>Ol4vyY2_{yvtFi)eE&GW!UR@+->=lCg@MvJIzJW+R1hv$L! zB$5sHsJ*W%VKg#M&&?4}O@sBN#t2-kMP$$}b1~$5br^@SSjyudO7Dqbxg+$Wbuo_X ztNTwCaEbg?`yjgj?b~3yr^(+ycpsVfcm1WsW8+oM=hK%y)mpk!40gh%3-0sZeBO8b{mp!p#(|dwG>)-%WT*tkl2PXb(_0SY=f}}j3 z?1Y}luSVSluU>O;Zg856#K<1ZyL)*4g>JQyC|k5QT=tcDxM#qg5+T-tfs8*9WP2HO zW+<~ZZM7E&ugz~L61-pU+=IFb5hLJaMBmFAcvy35wq@&FQL<$S4X$M7@n=(OQrt_6 zO>)J_PE&5U^A-+Y$wXUQU=?17LMc-gwbo1(b4iD`>9K|*_pbOLWg@#Vd3^?=hUf6a zYUs<4{7D7EwdEr_{h87j?M-%TsF(`Qd=jcuSt&gD-zXYmGH0?mUMVg2RJvWck7cye zCgt4q`w4u_HciSj>BWPDugLuqWvxU;8qZjVyLXr`NMRC9v-%2FXFrNJQDtuj-P=p6 zEh`$lRp5$akF?Y7ZkUesIP*-CiZ8A&rMB-HdZy#{2}QQWP+PMz*isCUz?UQAou93C z@o-n6x9cef93EWHE3LVp*A!=tlN3t_VP}+Uy-gB<4DBJpDl3z-AG>}GC0L{u+Px`$ zhKT~+(>h@1L(GKm^35D%_$n5V30>lIwbYp)BSQhnqi={#z7c&K09U;T?G z()*~n@=RVS1;k@mHn#E#jX^cBj-$V<>ke!K|;sLPsE$+}zwt6p%1)MjIedM)B_aT4RN~ zI6BJeb_Spp#6x6>5+C$9C6+ALvO(s*%lX6?sNM}Go(cyU6gw=Oqv4OjqgngK#BF6< z1O;`_1di= z@zy>hKeh&ONDf<%vh>WQJ%SJKPe)dz$-unEfnKd!v9=;QwZDGC^No;0>d^7=2Pdb% z7G#68h{dup)nsoi9RUxkZFgwgUZ=hq1Q^2eRG2pEiej=a8M=Q2{socLWwl3N-F?H8 zhO>iDiSs~Hb;}U)yD`vFt&2yOoD53I7T6vU&1rYk7jf$~ST0{pt~-N^BzM^zjduMorrqjcdPrZ z4bE1lZAnB9H2>z>{N`jwgv6;iD-jM!csn#4mD788b)1)CPxf8O-IrA~f?eBDwR@iY zovQRk8FI7!uQp+&2w9@^^Pmg4*gfU=^Il86NoSRR= z<0}?FR=GR6{6~W0-b|y|H-e)vf^_MlbBV#=Jh#^C=(f26*RuJ~#xw9y%2Y_G*?Fa1 zddb1o?6^R3%Ut=~P9Q-|O;DU`my73Kg`)4(q^GNUiRVr2*3n}AvHa-P3pw!$;RGjl zf6Vtrocf=7Mlz`0SV~{`mYSSpHLYe?*s?Z7+)*(%fN=DSfj<%hWmBTkPf}>?3Aboo zjI*9Gpf%T&qTGjeny}yI(3460yx2-d4n5fNlCioocdqQDO8FFU^XZCL0YJDnk|Khh z#ngnY9W=iHwf4y}=Vjs>Tt^dUkQL4Cdj|e#MRf3;#%`?mWV(|yzFe4|=N{h%3X!LZ1EmY2$+sb}+px{;tQz7g{tHZ%&R3r3kcls89q* z5PL*#MOt#yis#a0#u!X4ZQ|L;SxR5jou|YUn52T)5(#igvA16GF$uzTA5vM6?7}De{i=_9n6d zk>XPWP#iP3#TE6_x%UiF-KPZ_ZqS_q{ZPzt!`(^lOw zIBxwEv1WDkjozSK?co*#x-4_m<+P#9f{=bk(^C$(`~plXnJ7Q&I`In^+G#1n!asH1 zvVI|jtkY|CiXokJTRxo)kUUr z;a3NxgpI$uZZk(TrqBAG6{01q$NKJC5WEbmW<9inhh6aYi8Y|3mNi&rfbbOvj_r=- z=)?)inSqG(7^Gq8kq`uKo3#XJVXKy@F|2*F0BOo8?Td*~n|XEw-Lu&4OxjHnuXH z>*_`VZ6!>uA8#I=7sq}kc5lQ%?+KIh96gSYkn>^?eu?;OH6Wt%WuMzy-&VgC$6CJ!4y}CfLa0-R=`Vz+UW|KYp@&St{H|lUgm!qOTDOw3U)Tnfc3+G{Qx`YI zMZdy`b*yHsGE<42L5N4{+r^i`;J;$H_$MKFX^X;+AJ0~L@Y&yP7*W3|@H{=J_CU-( z(UmvEACs7>K&&l&!u$cQ4c2wh84=(iXoj)Yc5dQ-(D6Gnu7*TpT+45zp z%qtm`B|L{D9fm9+EsbbGHkFaODWBd_NeF?Q9dWZ>f^Bw7ypR8-YLPLN=!BpkyM5n!odi< zw;Ro*27|}%s0sO76Am_XJG<&4ofKj5Bnzce>Gdk{?S4hT(r=DNLUBB z$icsvSQL9y#zZfRIAkw=8%Ljz+no$bWaw1Dmu=~t5pjDU(>2;H5U;lE>*Xje`dtb_ zrA8vGd#UK_z2vUQ6UMtibN%Kp89hhd1EkCQGVhvKAi{;l7{H)Lh;8XNQ?}@%h%@jU z=Zsvv(xYUX#A2z?7Tw+hr%D=eL&6&6hv?lV)P<8nHzUCLJ;=cEQ?3eBuxf*YL^v@- zl6>O=QS4_smBhW_z}4iqu5imb>Bp`)7r%ppI^kF4PB$7DsYvf1Ed*47cgTxvebF3r z|HD{lpm|Q5vY^=U%=RJMDmQEC2LL)BPk;`Rw!++(Q`wAFaSU9Bg3ZIf3B?!*E zTFz+O7vepQq028LZ6jqs`~ZG6RY1OL=g>7p&yAGbr&m z#I(GP1}FU@+oHP6?{fIVo0=2NS0r-)1ic4vD8tI~LtKp! zrC@X>GXA#gg@%Y6AtL49$}s6t^?yF;|F9B-*j$lsh+>FVhAHxQB&kML_DyV3zR3QD zzQQw!m5T*l0IqyY&9HwmK;C@lpu|4;}H3z8I6LO}nYft_+$FD&R-TwfBtR%Kf^$2cf3SG|mlj@XdM z`5w+Q{gmUvKVR(Sh(Th_N)<1@*!7Aw>#>GSoa=kGYxm!~nZnf!zYcER`iuF;(7`Q? zN)Px&@m5=&J%Ko;Y`i3`eV&d)EQsox&U)0(YAnwNRdm?LE4&m{9WV9PUMFpumMv`~ zQ`+lB-h)g>!zLO%xw_|1QEUWxh1o><%zeyG^((W9wVdag6QAbvyoek__iFcaI$T*` zkE?#=FM$Mw?dzyvlc7^l{12(er3P6BAeu6Cx-G+~4IAE)j$ROA+U1fTdz~Ocfw}KN zOm`IAKY9f&#tI9IwL+`&-UVnrGSfpP)dyB5_rJr-SXsl}7bM5+RMAh-odLj<8JeGq z?EyHvfqxcgkPpA5&ywF=K_IW=1M|-g{LRrBXU;T~5PtK;7^_FP2BSUv(n94zKpw&gR$?tYS&XYXZHZ_AHMc%O+;+9s= zij$v;eE#-Gh;4tpr=wuSit!l<%Gb37aF{2{6B~iu^x9?CT9smPnCH7Ga||R(RrV~@ zD7~R26%nMyAWTbbHL^3Pk3@z;YCfgYr^1wY&DnNNwN@dU&+SvyfkNe+HS|e}m$?Fn z5$8s5$;YBnBV7ff(x-xTS84UX^0@DJ|8w>R6q0gP&IcmQ_luEAKW$XdWwf-tQth7a z<4I%TE0nhOo|+GW1cI)~cf&}*3<40B<0+?v(0vEwuRS}!NQ26}6=4t=`oSrE(+9`% zHt&}1PAG;t0&|F3EOSQR^XgATs=bRBsMH4fpa0Hcyr{;ecK50KHk?y7<*65!pQI?_ zx}P~0nZQi=93z9^D1AQXe39kfOxmqu{B6D)Gc4Ira!)C8R*^Y^alg*R&QFlt={x=8 z{+U^X{ucjFTX_%2@4B7V&zoyfMrd_)bf$~IaMOuc`7B&y=r6a8_Tbptk1|_gjlrSz zbh!q#KgX|z9xkq6q9@-dR;&j>k3wBNs+B(ev^0piqZSP-t*)d?wLzT>|GOWikFWXR zA4Ar#k`a6gS{>^G`Y*J2$Jt2yYw=bn9%c3{hX-P=W*x0svC)|8nlgE z9IyG;CahZ^$=;G#+^r2FIO4~VCePd3e*HkPhyHXodc=3X(xLkyCmS7k^*to_av`@# z5QmHcIUqqV+3u(aLg=~o`@)^(fB1$lff9m$x5EwA`ZF+)O8mb!5X?+9UJk6H-R9_8 zfT>8z%;38tZ`e{6fh}^kJAi}sXx=iyB@K-B&~nn{2Cv8OSmbC@)Ft*wcaTAXhW}_v z|6Koa{;Qnhf{z3L4^K{lon-@ZcDYj*P{6@N>{^hRy1Z zjisOYn1{=a)?~Tt+kd5s@&z6$pN6iYvpHi-lQPqbg-6Jt5WeG&t0k}DI1l$Jbi}!6 zTL*P?84njAmNPS*|Bi*youx7~*4)?8Mc5!dbDbXi**BK)qG{Ha-En{&fwW3~9z+1Y z%Zw!kK`S4ihjBOS%{26Sn+rXLaB`;Zl{x4){xCdnprQ$G=ph$l7 zkHqP?!AzChU|AkN(%-b(;TzFUF3{g4(qUIJ?jdc$EmCC!v!23|qB`rle`g~QbedCG zG6$+yiT-1A{nxa!?p*12#6WyX#-uldd9TAc)xD}xqs``sUzY2n39(by8q@t%IHHY> zWqqgBCg|vE+QCsN2V z=iYAge1c`MnF1KO9z7`hAvkNVPx{P$SJb>lTDTmhs6nHaX9Rd_q(b!8nKTa11mC`x z&UHA~-LJOjT%YRUyt7!@r{DSQ-6k#*gGRv@L@sDK?TFUDEG+;^FY^eNtOR(!KZ>K@ zYf1h|#)Z%B-T7dSY;kNfyzDGL^BY?H>nhgD!GTyb2+^xoQULVh^pAvFN37NNd(9a$ zB}AzP&Y6^`H6)er7mn26ObabwN4)eI^Y#heJDWiBjOkPeZqj&LEYSvEcw~a|2Qr=B z;-57w9$$E+y5XQ#YJw1*eDf3m|GP}C=(IkreDrVFID zU*|^snuLDu+)Y>b9|kE1TzPr8xb!bhs5k4KOON4cXIj&Zx+b+52oR7Cfw>l(B8*0-1wkk!KuI%3z_Td{wY+CB{$X+ zAI0}`dtvX-tEB#$Bim?lO@4xIwrIe|zT;j3-sBbe6> zo?ti`{Cfi|T`6X^AS#|WaPfhz_pvp#w}l7cOzSn&Tp-X zU&W}aO14$e#`txwKgv9QK|PGf^uUUZUR~qn*9OG`2HU}!?0rH)Azj~RSDAFl*_`t( zyb(m&vwn5>{b;K2Rxfoo8=p0=nP1_Ioj^bfAHw*YHE>iMD#Pno3_(^zCN{^BoM_p58Af0S~xYI zD)%X9>#iE$`WAV3INS&z+mWx^c(Uv&F@HJObQ)5C%Bj&dWo7|+Lcba83mr(`o~-tD z8Q*2YG%{U-gYw4IW*{{?AkddxWFjc7YXD*G-7u;fT96^bH1VQvG7u5YXBh#CL zRUb(^fo~rCF_Vj3q>e!AtjaDYZ?d{|Co;@zABtLI}497NxRhal) zda{qk_x%?}-g)|q6fMet#Vts#Wc}kGx6J1AxE;Fb;5lCRl}V_cG9%Aift(6@9Y$?d z9vJw=*-}U3((xk6#U~~MhdyO1+>?2XsOoohUVW7EDPUFeWn%^DxQrTH^{!YN$k-iH zUvs)I);sqjxJR6Nl{51ht3Hv!hwG(hP2N&XKM%nJvTwX))dqJlmP`F#o{t@2>exdd zap`__v>(H|qtlMlK*@W@{{z`P>XKyP_lgA9h)_##IXJeo8~E&5g#U_ z(n`lKeb>j}MFD5|Zp5O-paur45F4{Ot#*dp9!py$x?s7wUGCct|RJ&0IrRAY=Pc`6AT~FMJ`8;BIM5V3JbT(K=m)&rEIS^XU{ z@9bPm*1W+>GvSS0_|^;rSr*SmVTS*FOR#KhGTV<0qEj4wCJF1Kqw&Znf2C4yyMtMe zk>y(Y{_drOYp5ktt=r_;Zh9dF)xSwEH8C}> zc*t_wJ*m~IsH>a#1@y9p3HD0@R~gD$l|TE9iH-HlCyu-TM_A|UxY};`3RqucuzyX` zU7YJy6IvO83p$LXWb{62b4?3msr`@yef4WKwuS(pV_ovjNLc$Zv{%?=MIq6dSM;Wu zvJZBP+Q;oSl3$J`uf4nEpC70Q_(QwV=Uon$Rdnkvu<;H02uMkXn~ZUXr4>$hqCVw} z=1YFXZcgL7nkz>5lr1>07$vZ8=k)o{+g#s9x|EBthvy^1>?RD;>mMOD)g)9+Itr!{ z6pzSZ*zebH`S|c2M{zD^*j#B;s@3@P00-di_(Oup{RI?_wAgr83=)57+zl-WwUgH; zTv1_c&0V8J*0E*n_0Kjn;=bR$dyem62l;Ip@@7Vc>O`}}=n8kZ)b zS%kdpMKON&jUgIl=qt)Y?v!HmyG=c{H*IrGB_U)STl;<|6w4}m;EA`hb*{pz1<~ZM zP02Arac(`L*>!!Pe_u1jS&J{(wl9mWKIKl&F!V;)4|Q2g#H#-@(lB%u9zXR1aZ_Gv zP#)#REQ{dX+2^Nm;3vcCWziv{DXFmge9@K+&E?pnt?4Anmp5gizS&H3f6XtM&8n2^ zuV;3ipY1s<4WItX2BrXqrl$(>A$hrlab7}sEKjM%u}1yncYV<}!fVA|!lou|E1}2kyg6n4B~Z!eW3`cxLZ<17_BPnoy?RMC zsz27n2CjRVWx!D`iZ7U>goMJkS{9V=Co&y+FU`Zpdc}BsGrD=v%u?#}O`^Se1sg=Bon=3k)Q%jl!Tiy^hC9e$PwSh=7 z(c5E1)@^xLre*o3Yg3a_p_H<2f~WAhi*ohZO@KU>@V*+2c6;aSGf%y zH;8$QmaqkF*w?^!Zh{I4&?y|@@EE$KEyfE5=^L@`x|14__wUh3j}#UY$oZ|W@n&|@ z0a&_S*GxepZWPQ`Z#Qn!_=7GoXEVDpJQ$!B)%;#15uP83yBGUk)l0tc`zQ4NB%(;K z14Z`;>?nvnxpDd9j$IJkvWFUSMv6H<13TsF>{Y^_^l<>sYG`Dr-&(o)3)%YO zkq8cK0Vc9&*q-O?N2xSZK7(01(_~< zwHG&o#Zej~+Br#}CUt9Aa?9et2CL;)@%o+LxA#lmVsb<s#+0hMR8Y7ds*As?htsc$Bu&!C5Y8 z*>At8QLgH!Y{yz01XP3FLGb7YRua#V9!VgQ!P9V9WYVfH(C#+{bY>RFV6ZS&)S%6p z?E)eypNb}LTdS%YmMJnf%5HfbaEV(M{?uko#2ojOGC>b#v zf!wO3R)24yU&|~5t<O`X{R%Kn$`Sd^=fdLS zrPVRr%NNQ??XrEcTkM%!zc4?LYwZk~KND+5ptZJz~R?_100 z$CWs8-vuw}VQ49PoNk877)o#CYBC!(hs>DBylUFGyVFmk=zTt9+BVP13e~b!7W~DO zut%t;&igs5x0;k8dAHbo*9NJ`#^n%YJu)i^)o@X>ZAbTATD6_T)~C4lOYp=@U%k(J zmY3h9utp%*r`2x{J&JvQHiK-(ggIXh_9BjxZb;n57CERO9wWcssG!VPS&q$qzosz03Q-%YEof0=^?SXOq%Poe=Dlp=wo#(L*r+14 zigjo~!?pE4i}I)@Qf%}a0*tz zpgd4NeLCQ!Inu8Q>Bb8&8+s;JU-f)2V{F;J7R7v9tvh<@=?1b{f|U~YOilci{2lpd z&vj-xF^!j)kf1EDEKu}Nwry3IRH2^VJQ#hF1Pc>;AA}#SaNWQ9B4>GfpbDUDepP+h#?$CVCSN z+8hAmvqPI6I-hM^q8|0U8=K&lO^Qc+yPub+uQfE~!bsu#T@PfL{p{b9R|!dZ z`ST&gW&LO|eVKmtTb((Cmn~T)=sX*?`v~aFMgP+$C%|imx-4eYKWGk@^LGKBOwvAX zVyp(EUs5})Yi*1>^)J~qnE3k^{fjTca)ah|4)#z~4RD0GX57qyystSc3wlNqu|?NF zVM67;S856ZJ+4fIm?JxuRZ>%}} zXK4-BFD*V#p*s$pU&GA*5YS?3HJ#kGY6<#7n|^5eH?Hq}LNtllzh$<*n0EYNwtI_E z%3L(vWXds*vLjNM+^aD_MI?K$@kgUw?!p0*8r{70)oR>V+ONm#@$2((@)&%|-2N;t zOS>#gZ1q1_cgB{1>rp-KIoo8HD{MseoqMi8e?tewiu=E$perFTwWo)}SH8rH7ukU8 zs(k&^qtZ}WoV(sPyWOI&@17-}?iRM2UM?o)`=eitFI+??zXgXM2KpD#(%HaA7-trpmH*jvm9^c|rzLxt zKG6}5x9&>{D0emfg6xlHc{2++2Hi)q5ItK-mo zes)vvbR%EB%9!r}b7i;;|7mwstYrji7vsg@(FzNW~MR?D? zm1&}fLZZTh;Z#~P`$xucWmE0b!%aG>*mmx=aA2(%ituQ&KDyfLDaMqd$Mh;qF^!^j_XU%q}{jblf_{GvsDq zcb!UVuc(eT=oE-N?!;l-cBMCZJ=$-l$m5OTf(#~^-iLsHp4Bb2o9a$~dbpp%70k4a z!%QAk!i50}0K-SQ`EPMZeIin!G7tVb+R|kL2gRk_rEXpghyN^qv$F<^)f$V6Z-hdw zHhuML^F&Q;5C0?j-5dIQ%9<6!PT^2{mgy)JeLS$3eIVO9_Gulq3zKmp^8zLxzKe`5 zc30VJVjRjC6qInBY?vIx1xG4I-R7UC4KVvzzmI~f_WSOjPH?CKVvX71n`yEIa(d?y zaVAf6dp7U1oPv6pEh z*Lwp-WPG31)iY}#ywc?@GYgD&WF9-(DGfww(p;_q{D?v{;9izS*;)67m<_@t<^A7tS5zTmbu((Vw~vW`!^*q zM*TL^BMlb&Ir{>C|MFDf!c(rEH_WM{@-Hlj$D~r_B!7IrSd>6Ms%G+a>zKMb$2?Sz z`ejLH>wLV zp@%0DA>1p1Q#EbFskVj`8&7e3*^ViUvWY;3mfvDd{_hJlbg#D-)IgxZ_N{WH66BEX4-O|)~ z)G0`wz$6n!VAE%mbG7`LVpq`}<~{X#`X;x2RneBeQK~8vrc2m#45c{<;~UX9HqKMm z?#gzeMZx7F@mb?-W_a-VQB7q2P0UZG69s`zCXT0zp( zfqPir>*s9pzZL^RO@cM+(D}RXZAT;QLAz>C*z>YMAK#(Vh^tEv3LIn%R$8N~kNfUo zgHBWLr?Pi4hss%%S?sESQ-b|C zbAM9AJ&+sh!w^3B%~mTn-6X+6j)_EIRuW9x`7vTE(?3DIiiK`2zRv$$1!g+_hXIS1 zW(nLIyKR?8wHH4jB}J6A5Am4}gNFT@p-3W2IKKV}vhR1DI)fu0kDA*fcm}p_<%HWY zdsU?kfhJDvD|mJ1zwG{RfSvh@C}x=RzDczJibEgn*?RTB!>u ztm+y-J{ztyWn?nOXe$i-QL<*CL5)zILR(`sayOlYn<+Yz&LKzu&eGgpTK?6Z+B-OB zFJO%RR@vlq)`jFobH7$9D>Yn{9vhtL(cInertd2xv-Duxj~+Xw6Qk%IlTN(=smt%D z*$hQgw>LRtmA|fgwX~`f;!(+A$*1vv9~4jRi5dVHNUL4FtH_~d%HxMQL$&(#;JS1+ z-rs?AtYySGuoULSfe)mH*>d(dlSW()q~kT=WuaN$pRO1Ds_;01vzZQ}k4vsx}r%R-f6#Wf;lc7b?)%&ILln=6xSy5z=h zq`1sYnPG{Ya)fX!Op944Uk>|=s#aiojKkJeC~DK`8{%%>Z2y4^duGxDA$Es(4Wy1S zNPgCVLx4S^b@ayz7@S_LduDRc!gAj;HTV&PLR(%zJ@e|D%rr2XI=b5ywDqyYsmUPBt?aaZs%@CV@x z(q_qYK}Xr=K&~jDy|xR5x^atUA<+Jh{hjdF zX(ugB%um&6^duAa7G{%?t@+%TD;dPMPl>^(WU<0QumoRwYqLCMri>&VQi- zN09=1ROeYCGR>6!>IVcTp&N+p({ED&&j@Wdqx+24Cyrks%<94=$ZvV2(0ob)x&htZCpiQFTRpEnE;Zn% z$n}c!k4&KDIc*pyF4ZVQ-Fd(q?Pvu#OxNv+To0h^4WoBethynzxMGW>&CqNhZzVTjaAQ5s6_bu}4RUxD4GmPbZ#g-S{Fm zbCe}7D>~RLC{O&BG?egZRMMvVguws-QX@sa3=(};BHB9cemi~ z?h+t4!QI{6-8W8fcXxO0;;pK;>U`&Xr`_A`4}Q?vhP_zqHRl{-JiYh#SS)^pz@uSU zPJ+vPf?YN} zcJyl&50t0`vA~_Wi5{L&-nfSL2kWC}^c2}LswXl4PWgiRSDFowE_1NN;v?%vmev>w zk=QtDSy0BJRQ_k_)PafRaoEO*yBVQt8L-bD`D4E9XY<3)q4FWlBrpGz+)>Jrv zxKP}_;E>I9RLZ8FsVjJCYP@t^Oj_XG4EQF@%)YQ{oOANfNem#kY5>xMLwt?7Vw`D3 z=3yg$csPOWoAs`#&o^seh~<1UvB=OjO+@oEYrLhYN9lSV+4gi*21VZGteGU*0_I#B z2YLipzzGeyJ?Tif6{DAgRrcT&M$vet{3GhUHO=#nt^LeONREBV;nG!@mNxd?8%wGM zLtoK^sBrgC+{Q{{A;jL(N!D{62l_7#sR@%u4ho8xXi(#<7-+fohXXyg<692nM-SKa z(yNoe+g`Ad@Hl)-rfXzm&+GIqFf@4}_yjN%BNlpgx9sR(dajwPuYN$sCe~Q*HR=>0$8ImAsahTv{!F zW+Ovu-9(ewdgmMW^2%6+P}#~!WrToIgm9?(vVBswpZ;WN?Z`vjwkjNu3-i+Ua`*VK zsi)j^PAjhJwzo1bemt5h>G;Qv9uV|JwTB*MrO?7!h)CbuimTN2KiQC-AiK)7KNkpY z>ir9YlF2{2B6cK@$U0t#WXatJ3Xko-3XgvQIY%%-aoN<$756n5Y>h8dtFuUqUb^?~ zNZoA9c`eLzhFV)4ak*J1-}cMO+uH`BJRaG9m!;6x2mvrpK&YG0KM|hhCbPZC2`PZm zJoyN;N>FUp9;Zt*D^qHzOiXLWu>Gr?hyA)Zsmb0mL4s*q z6cq~~N%9(84xPP!v^gF-v&46Y(+KA%X4A_wE8|k#HU=$YY1y zEb9!}XNq)-c;(^j6uY%?kJf3fHw<56FRjE`fy7gT?KW0y?85iJ%Ok$V)aoLL5 zYnTZF#|xg4B@N@*GW>@sTVN=Eqf1=ClPeNV$Ah40NZoCK4E7yzLEe_wr-sp6U18UF z@58$I<5PBy3ZmZ56!F|Hsg3H&@LHvvGxr&pF#XoMb%_*ozj{yz7>V$?^(#eJ(}L}_ z2K07)hM7JhPwC`b6AQM_P2B&d_lnLUD=J1GzvMTcL7{_E1UhRDb! z=NiE?q;wdTNW0WcK}jm?5x!LW8SDd;$$-dc26#s%OgFid&e0k%BGT#4ba56F6DSRs z4)FK_;k!TGl|46v{Ov0qSg1OZARh}n+7jpvI%l+qF#O(b2*90WEAaT*oDsel%qoEU zh5h5B;5hvh0`((BUs$lNh%r&oB$EbRE0k7u0*3?Iw{DU>Oag&OvUvH#Sw!J_trrq# zWF7tp{-Yu$(P+Nr&YNr4zGbQiJLQ_|iw+NpY!a*APkNs?g42l|7WuqA1=?hy#2kMu zl}vR-pE1Zg>+i|@$ojO8dnEj5wmU%AN}Apb7mA3|CB94`e9L4IZ|23>bc{j~*53D^( zjX|Pu;WyR@rtrakoiO`G31Bd#gD>d-E@Lux_nijMGlz#WLhLr_TQ(s9QFxh_{VA@v zlp*{)=0^alt+lf9iEc(Ay;q%5PzG2ETJ|51u?uolbZH7?i8*!0{OOqyGD`Nbm*kFb z;dTvU_9zj(6C)u#9?9d1GhThnfzmpJ=3MMHCwFuJr**itCZcEn#WL-bH>zIqcUZ_e za#bHs6PMPkT|)xAN`b7ps-DHvWocL~i>X<6K`2#8=@KkT71{c(1I?lhPN|+1Svcam zj?TU&OF_RbyCc3delcu*_Us?78G|c58uty+j&SOOYbAlx-RtY{MY>t#ws2TBks=vnf9Iom`H?9aHbt#KawmBaNR+jdfYG& zT8ifm_aPr9F;cc(S{wF2X7JUAqwqcYI+##B^ho>&;^If{cQn`8&cw5vdw73fpLUJQ zTJf3$0gs$C&-ZcW+KfJ*M~(=aQnEYiM6nn1Qw{uA0xY z(=DBLz1k0|K{!;mA!|%0@MNU{ZQELDKFqT9SQ`Wi`J!_N*~KDP<#vS?&>UO+`bgSx z+pBy;<6s!1yU5W!Dv7JoxR0jnO>MNT*K7j8C7*LWts9+)z5actT>E^bs(dc#u;+{+ z8ZP!Sf7iOt!O#Enb>!_$hq5`A##QueBp1dPt^thRO}k`$_XAr%M50K=&xzf9jc>;b zev!ct?v0*YMvM0hbV4VyYzE~ml13DA?kJsc<2KRQCy_%l)8fvj$*KCB-x=&Q8kUf1 zXroY&^4)rMd9_~{PCZ(~a~uUE7`H68K-=I~CoeUEBqsk}bdUcd+{I!z{H&7x?4bJX z%Ps-i+V7c*H8RJzW-f(#w=U8)*SqSf^)se#ss)ptg(;Irur$9ZnoGd(sz>*M$NV0c zhbNUXRp+!Mvbhq5ypjt4wu84R5JycxO_Na~;%V3JX7!aB{WCt7ge=nDC5jzUHaJZB zJIcG=bzcbdErUTx$a2FdZM`tRC3>sbG%$9BJli!9FxR3%F6W@?SY&_y2QapAbaa`# zQXy%mUFp7P<%E&SDH{8F1-Y|hq1+mtpXdsrsx#Tc?Rz5JqOp$l#|N~uI@x_VW}L2% zYdQ!`AeL zA&QdBwlGC~#)GfiYvwlAmgXLMHI{4wsN+`rZ|VvjpQv~uy9ap+W6;xf^@SKXOL8~W z<;kp~Yr?LLA13pop%U}!?FuiT71E3z_wf``!C@ggkZVa*7F{;B*S(AKyt@}?D$K{Q z4Wz7d6Bw!8IB~$+{yqkEX|4mqtCz=J@gu2G<+rWh*seMH^8R+rTB0 zWyY6iA?y8^}QTO4aUE=K(URpX!oQyiq zHLd{S6!0@icF&$@Op=a@zvjnx8&i^tP&pY6#E@gL{XmOO#}cq^HkYFl8AN8Ckf(^;J`4(}=?yxU4X)~DKK>+0;onF5oDt3SS1&4!g#%x1zX`_nwdErZKr+j_KQATFd2u65lvB8o!2yH#!}w|ZsKs4v9^wu z!EQJkMMFsA;tAa_O{R1Azn&xiMF*YB-4yA$Y~T>>JH9hYex&=lGZtHYDtc?Lb0wz* zWpa=H*X_5Uqiy#Ucgbq`D_hl0(PVqZSNZpxJ12~JE1V@WqiIN0{oGEBqN1Oe<+ITm zPETZzOjCi+C`rPHba!j*OOxDaDXjc$>oe5<+O?+N#e|3YLy(D~z z3)HxxrVyxC!FCALfYt6J-Bx6Vq^@WOF;zE!Bx1H-v{mM4C9gPwM8P*A=Ca7E+UC=$ z+PkG0tzNO97D@Pu|EhMwa9Vm7;Lr~*bC05HuK&DO@dC~4I5$*ufD{Mk{Nd$H4{;w} zN8s*k{d*Bz2 zgWjIkCSR2Hc~Y39-0S11o9k*QdvuGlF7sSu9ySp!4W|>ndAo4xp9Uhwze+2MImTvK zfVHhU_vtY3IHlxdAaZa0j_Pt!B|9%EI1Y~h5OS$g8A;>u9FR%2CeNkrh|XnKV{wi5 zgwRlh?H^wr2(~9^G2ift92Uwn&+_qx6Qr>>t!9$lQX%0;>N$15B(Rg+)^rt{pl#<+ z8zHpkMovhC;D#-AiY43Ov4;VZn;zN3@f~ispkJgJ6Gn9~9r%s?%i6)Vlawv4Ud(t+ zfaKy|&)yOQe7P*N4j7tpMmMtdx7l*f3I8UFIIF(gOH~Dx`!CKx^=Na|wj@Tyg0RMpL>Zh_MV6tB(l~{+kP6VGJrjEk<~SawP`*IK(_v=RU*{=La(?kIB4%{j`M2af3jV z6HQCS@#Wz3&->Um-yxI9^Hw#>pQ$0l1?nC1B@c&-f>3LM3?`UcWsz3G>n@WZ+Pz60 z_`>B9C@oF!y#;}vH7?884EHu~Wd%S9>{*TIY zIUK{Onry9i3(0_%fO!Q8K4J$^f1ykDJB3m56j;P4@12Xi4B3E!b41$E(CZ_--79yr zs#jWfNI9*}jB(+|b65AT*dNfTBB+;@n!cC|J#R8O0YZpTte;QfnMfS(8JNO53Bs-u zt&^X~zTDV|K$WmN`R;9%&xIlL$JzPgc;iQ>J%{%se+esQ z?1gDY;6vP^vHfE07RAg?eQ2$PPi(rDL0x@Hmu!H1O}A7@PcL7c zrg~&U$82!xWknzJa4u1!Kmr4DotKGm3W?r`nKEtHf? zAJv+L^Ls$}91B(J@Na0U@}k;&3H7Xk6qWK=A!Z!dNc6Q43G=x+6)N-Gxdr zI;rbY(=}0*l^3+&jS;^xOQEQV!V+lHHkX=M7XthJN?=>md;7-3zMXUw84aXqHl97~ zsYE(Wa5yA*Vj2qnptSdh@HOiiw$O_Am<{G1yT%2+VNY@WCKectNnPn`+RWuglTT0W zO5j(XQd!5Q@VLns(3;yC>@U8gFq9_zX0DH#7MI}&_d9NX{kzZ)s_lDaO8a9edj)0) zS7F8sCvr=TmF_%07Y&%AYH<_7x%3IYtUX+e(To9p&Y096hod1exX()kGmRgn>Pgr> zj7I7`F@0ZoA-=dl*)xk)MHkzdl1ZbnKA(*Cexdy6>- zNT!-K2j4?q3e1dp#zvdKoGr@E*%DAxB4G1)L}kk0^KcVpU>;9vkyoF$l?ow7+5`u4@cXE!E2LBDhHvj5 zy;k~uUa0dEsIsJ8ow9`zdpw}d*YiOvuXjaR;Ek5APR>xf!xG2BnYYh1F{(E3u(qwt z*+-dP7uj7jIv$h&9d{c1<8_QFv>?OwTSiIY_Y)IL<++Qg<#FR&bEyM{8tc?poamyXle5{Yx(qT2Sx+QZ9}v8?U@SRa*N1P zJ)EY&VVh*V<+8$P6A33fyPTq`faG;*;iV-oPh7cJxHJ4Of*pW2rJ+@u-JAY@Et{PV zerG7ZbX1Csw$pFJXYlThxIUNy9y_~5c}*21$JE-L_AeN9?Ch#?6AGMl{?eYm z*Qm8CA4K%}TN|HBEA1*XvP87TX22XARF~QF`-WIz1Ao#Y1DUS_5Z*FmqXe*zZ94K| znuZ5H(r4~EV8zPj<+#nY6W5C}Osfc6rvb~?8)TCZSl7Fq3g+saFWd%6XX@6$CxP-u z{O>>?eh+FjYBh>fcXfXirKH@L8BsSL(^4QqxJ4yrqjzumX~$sF&NxT|*k8Oydx2?t z)R=YdxHXl3Zt=?=HbG`B2Lthg+0|e zeGpvt_)_BoANbvHr<5{g6$cGWG5A`WlbDD~`8*iATT*4@1$7U2XDr#j7ZMQtV`#YiI;5p8nM z4hg#N`~nqalOu+#)bL)~R$;8dBQ-^)epWuo<1a;$af`=(vFe4#$9BDch`M0^EuYze ztnu#OYaHIo1-sG^;|=*1(dMkKN&dKXTQX;ul8!>6mGJZNSG|e@zl|~@NIRFFn8XIg z%uB4W)yd>RLg$A)#Y#9;0cAe31!3__@9e94ERty~L8=r^ERMy8;+NVmjqE44CVPI5 z>Xf6)_KFublSi<#e~WX9$H8)qJPnT{CK;1>_eK zjlX-K3D0MLKhuxoAPFYysRCL-i1QzD_gOy=kw_Fr%QcQ9q+|p_1Sp64$3L59^$iqd)4Zwet;lQ8TyW1^nv# z8}!BVo>eD5lXos|fg$zA6O#%s%c}wdC;wt3$9CcP5)uZdIC<_QwodMPUXlK>D4A3r zZ@y!JCxDGwp`mYBN6m{6Pceg5aX5_Sy(V8uEC zPLALnaD~mvRlIHAdGjXjbbqOp`X*7?+oI<)rjylf^PJ^vG*5VO+PN1~UL*9ClZ9#i z8u!w{WX=^WQM-YdcV#-U8ClUgrTtH7jqgmWi`{0QgK_u6d7kq>MK$wp3!tXLIUNJz zRO*}-ek_n?jk=%_AMO~6h`t^6)|anI#Ek`#Qr3hJ{IWz2(OBN22_6X5(UmrxXHZPq zhQZYHYv&EOgD*Q+pX2s<7l3^S$xBp_n_4hj>*-97K$0#3^Fu2Ixfu=4*w077!mFfe z2afeVI~V0;2*KbAe~p#6EfrWlOqG0B)|1?gOjyvv#b)u>^+etizbu>`E&O}k5R%__ zgvdS{S)umLy3IJNka>9?yLEQmK{WVHw8R@Ld3=}XiGq~n_DB;>7}X>M$P;5_Q^n>V zr`21YNBQmx2V!;g`qscA8^X4!m8-akVFuPaK(Ujn)b+#;lV!x_LU`FJ)$>>9z{}5- zsrW-C`- zqwzE&wrX!rl2^lJW>u71W7Yw;+4P$rr!$&J4pDWG5?noJ!rnoO?2WpnR&8TKXeEpM zQvZ3zv(cBQR|=7U2^ytHL}E~IdohGKK+Nkg^E-4=gKZxL1$>4y5y{vuZ!M>`YN<>M zsB3(wH?jKBYRvv&MdD$@85O$o8X`U&_+l3qUZCNW|dg<%=qOmI$P8UExGiKbu@IE|#it9H!4@px? zZgp-UI1GwLE3ya$Y5x){75a!JIJSfj%vhn4 z>5E}qovEIWz@(}?w#DMgL&ZVb=7Iwhfg z)YNlna*AcQ15fIbC@|e*tg7EL(0T*Z3Cv^h7JQw5m{##F&Li1M}E;lYh`hf}XMVP};BAYynysGq`cBxUeTRR4Zkb9hd?U!SPhEP3_zA9QmLd05L%L|mRQOl|d!rG44j&H4S1?XeOWXmo=Ga7xXO zQWD;z^TMo=#(SA#o~Z8u^8cU00U=dmaB{3L2F_^RfDA+9&pgLNqE8NNKRL&>7dp!> zw;P~uqr)TWyeBni75VpbzkN2`ifn$QDdgnA3Mhcyn}k!gpu+~GK|p+iV;p@2St6&< z_fPkuPOuNg8HlS?N*;JBdy)P$r)_W@h~2noQ&u2pMb&5LbFp=k*}l=9A6dEF2EgVx zs^|00X@Z)2%$&=@u}`fP@L_#O-rfT`c(s&A%c*{8RJKk}T_&X;d9uNCPrqp)=gt;Q z$d5MqJr0aW?(QWr+q2Q;cGx}1N-1+r@Q5vFIq0*sP&2V7ezwrUr)@M)WN)ft9{>Eb z;`=FJbK{;XolSl=^MbR$37t<1+7b zu412y)qDYzcK7)r$+VHGMq{Optx9!sW=1EaSe&8dlQn_vAdhOB(&m<<#3=SN*W2=0cpfHHUdtlpGy4_KP%^cvpaC-xOGnf+c#5MvyA&yAM|{}`bmg5b|Ho}CYMSQ7nP-xk zlM;_Z*;jAY`spmkD~SB(E;7jEdcWwpz#=uK8Y(k3wCvl*SsNiwxbRf)^Gkcd%AmH+4}eFB)IW(eGewP?xgGd|6 zelH%5Nil1z=IPRnah;i+c(cck7ki1j&frQ%|LS0`87Z6Gn`pH;XsnytNM?xmoGQ*< zVIcer?`A>jdX;mc_8BGsP!QOCn0Fynz@ndxG39Y(Hjc63*F>kqagZRmEb}p&Oi1hd z-wE}?ne2Rj66(VnulpCos!Q?hvE#2}W%v{ll4Kn1{=X5LRnNzqRNO>3gB7P;E2f#* zB zs-*-49fYLUu9S4>T3W(U&B_EI-9+`p%9~h+A@$($G-3yr;Uxdyi>!R+nIN;$D`x}& zsClm3U#R&_;TbYKaYtvBZb*J#3b(jv*hd~1kmhU=INar_H!l~I!Dh8#u4b;xrcb)< za9cIzAn-Ku8*9Eilla+~lN0igBjMP`po{n*jI4RuTYY{2+T3rvcNvo+PGi`_8`&RL zM@SC-wQ1dmI-M}$-P4hPva`=6r4dKEQjpeWfY3`Q({A-RSK#=n^$M_CjqeSzr_<~j z1spA>3@g5$;=`L=3Q3W3BuDgyMuEWQ1Mz6U5~8^ zhPTU9h_X2WkL=zT|A$myi2Pqv_{vMNFn#@3mJ~fervU;5SY8q&ypjEzZlNFF3#7>; z^z91`HDNT3xRqGn{x>STq>t0USe@{RGNz^lzHM&#BDpRi1j$V6s6Q%%w54e$1vtX^ zKMwz=OR&m4S)7S<|HGf&z6pblGG$)RNM>klNhfaELLjd>Jzc z0Jvp-AEod;2FEPtPf4h$#aUm@KK>tl5KJcZ9(gN9v56Mpc#j}-YA2Zjn13VA6LJ3q zai;u|0k1z-Qi$jnPyN#>ull?cJq64<{0At#A2nwtyBUt^Sdg_$Yv}%mH$Z+er+Qya z(?s>>T>eM(E4$7E!0ZoAnXJs|4XT1SzfM&vCDqz0k6{MpS|b%xN&6o_+^tc zZe8wpdqPbh&gJ(fmK8GF5_?x_so5h7t?VCh{wE5$ZQr8!_Ws=+&{kA!NOulVgh6}O z#Mv%4vw~|puib?P4jnI?sjzN5O*k1BpW@44{&WkDHS$Ab zp<>x00IDtv%q_aFdM0z~`W*erxDLXIsObGqfgS^Q`}i|`Y?G_`R7u#cxMF>L@zxo% zzHqn1t0z&RUOR-JpPGES7u%G`lJ#|UMdP&~M8tXh>q?F}*Hivi58vdGlD85Fiy1NP z629B~gUZVo;lMRu>Xf*?huk`Lk*c=;GkD=J1KtDZT(KdrSJ+Ux(KBCoSjsUoVL;oS zjBMj6ld%-PI0PMV&Yu^n*2-IY@Hk^w^>p1VxaXREYz&1YZ<0*ESTt4Q;GU(?8YfDuN)DbN#ObcWEiEbyojXegc|~Um5gN5!LAN-S6vj zTZHq@YG;zVs8x}9pdCHrfyA5cO*x!;B5c}fn`cRgcs@RbkD%(7!&*;PyU&A)Zn_M` z>e-sa)O*ne*cbz*N9$4Xi31Td_XNOa41YFz^4{d&(X64L#Vew<{=cx#IHx=r9RHgu zu>x@BTlXP)+Y*+>{w99?AADpNi|=W$S)_m`j`?uW0r!5kz1_6Bqrk;{yg@yKrAB9x zJ#!{@DqKKW{MbLcG<>oz&5Xs8UG8h4QsZwIh0!!)Q+T!3Hc^WTj{^{14blS(bA;;| zjn+}rsPK>@+Y!!$!V3-?^8+}?2y_^evAQ!d&d{pORmkwQ^m?`zwv)ipUmeMl4B`Uc zt3(1*`$kepB)vJBk)XyFETw^>+gVo5wgOH^WILXZDlMQdbwwF~8J9(NQk%Bsm z@d45qAfBCx{(@I-nSgH*ftvy_gt{RUJ`FxG>>Pn}2W_sN_rF^e_KCJP9X?l%bX!wV2w`;rn2>&pNTH4>5$ii6$thfI^1#^g#9n9Wi)#nZ{%%f- z5|khq;=~%DUZ9i%`u~Kr-onxY-Dcw6eTQ{!*sUtA|;0muh4bMfG(1Z{Tlg-TkzMkFo^brC~3F+l4}! zD^0kI-3Hc)QKm+Pi&l3cJ{vs!_LJN9*>x}d8@-5+n7A1dO>TE#KU^hlxjsN5rrv?e zcgNbDA{CXPFnIW<+t!_Nq5i{566lpUpK(|C$a=RL`qGkY?eQP2m@ePlhMA}?p7nW!Ik^(01i zb(s}W&O@nOqmNntOwtT|!pgyxQ=Hdz`D!?MUT;^?Os`7K7#4D~@xvYCEEdqq{~oTJ zUeDz(Q1L)CH?JL=cZcUZQlC-7`nqW0uEtuXIl8w|r~Xl$1+48Z?W#%gzxUTtwK9q} z1gbPPjsu3YR|d3JJm!QLz6F%k%(>!2;NW48Zwmh15% zD@PN+V0VCebosl1#>(()j)R-PIXWBD*4vtFG`LfJyVpA^+quCDO;^*R(Us2cLVcWt zF(6IJ#C$m^w8oX?=X9C>PiLD$Q*e`Q9Zc>NVFH4YZ+8j&eg*qUn5|@Da$kpz$*;NQbJOi(K;}OR!e*v!@2Eho8cLVD{TA@g_Fk zm*6x4&QZ;QP1oIZ?rURaI)h(1*emijgnd_h$erh6AL6OET+rmVF*>cac>?LTunplY z2Kr`klj(ThDeP|99Z#ule+m<0kNw_N_}TX2f?`UsleU18LWz!S>6`h{k&X3v4*O{p zFfi0~MedmXll~{LUPhHpiyv5P6SIpkmmm9nM<;&0>HQ7_9vm;KuWg-t6ZOPknT`c* zDcM~f1edlVC@#d!_7+=kjT`Ne#Gq?#KmA&MqmfUTmb>+A>@PprtdiNs(w6hto?rC~ zZJq&C%)#FrF`1#0bUoJd&_G$w%|StmMO_u>(7lQU-J6XNNna-;+V(y0i*)^(xh{9I zecrjZu%MTgUB^BHTi!){lx9q|mlGfP_W5BV_@^e$Gp`8fA0{@^n1{evX1sW_33c&L ztX^TSyH`!D01(x>Sz;4$5<`NxQgw3(1`GI5CwbhSq}@Hj5f*1 zotm1>VlBS}zF>GG{|)6efuMHkA^BPGgvG1Np83F3^75d^`?QYe+P*#<6xGb@fZx>v z%XWeM1`c}+2QJD{R#$d63582I5{!l>yt~?+IbS!@uAq|?OivI+v;l@ObKyuO+%erq z>U>{QM}eq%=o@W29e#Vn6NTs)CGR46Cul6SkEU6$xAHdEa=-BgAXoVbkAka2c)h~K z-!N9oDO$5xS&5K}?V||ovlS!pyCWU2llOX9Cr}B|?LS#eX9>QmrTusgU@75J4s#om z4gIbDSkz6%#MJd`xb+_ha@y%&fei_cy(r$%?k`?Ge1k!KQ;*v{21F-nqt{yivI{An zQ#}}t+{0>Q*~NCWT<8Ev<~M+Aa8Rc=OXAKYW(~Y5Q@)lqN+A+5CCY?7f$;<~mEnGN zf3;e9G>6kB7+DYK!wE5Cd#C=jU(&GQ8=ZQ)YM0w8A=xTNaANj$T?eI`DsXH2%Tzbf zmtw0R4pr*N(K&}!%ZH3aiXDk+j5HT@SI>xAKuqj;K|toChWE;HdY$oi+L~7Xp3mDQ zGfbXF84jmk>sp%)xPT#m|01jN5);r5mHh|h3VdR*7*DXbkK869wEfy>i38~!bM^EL zIa~_+2j^l%*X$@6^ z5Q=vK&P8%yaIRCaM#NG5n|*2x+=K!fD}P8?^#wFXyElf8c}mICk-@ezadq{#brb_^ zXz|#nu3-n#&FLKfZ)WZ~)v}buub^0P(B$oJ4OqvzQIOz)@f$L|7L2imxY+&}F#QNn z>JLu}WeUE<$#~o)Gp=G@rs&FGMXDg6jwdQz`CW00e?3%YjfcbE)Z0A;4W|wd~vRw zI-vRJ0;e*4XTJYOGwOBpGb1g}J{d~y!P*j-;t0z{h~+z(sI&+Sb-WZXN$m&wDx%9P zDLTa!?<^*BUS)3(d5|?iMqX%D7kH7?I~Sw*Xf6Kv`#&kPr9uV<2wT>7KftKyU0m)p zGzYmSD!A6Edt+4R|4g7(pr2?TC*8fC?pw|y{}YeD!>_Tkb+G#51jWV9zHzi8C8ZPq z7&m(S9SGILSlBCc+o3Uz7vHl@Mf5|v%#IGx&B%H@-&_D88Ye#t zCXfl0lNZ|A`m_zMUxs3omtU;@HlbI@mH2Q%{5M=piPN8C6von{$}B;}SQaa~y_{eXaTHOE6fk zP~}Irtnw&dt6HkDO3M+g3z{da7eGou8!Xx==N&8jPIH>W{8b75X=W8O2cy8|BP=~z z((C}kmmNlcyLduSxpm-9&F7O^i#@%wwF-}!b3pX+7>E|<@>;WoTf-k2_K`{{9T@_4 z>OQ8b5GF^Ajxf_;x_%&d2@;f}qH}p4$k$i=tUGJ8N{sKhc{wrA#w?~bN?Le=hcjm7 z=n+WJgBXz$Ukz=!pP9;HI)s;foH=nC^nG;o0lEqI+6}(Z-JCL63JlQMd78h|YQh`c zQCizZ_kIimwgf}v>{ZEq=eg+dty{&;PcehlKV;G$e5V%z6WL%r)!KZ}?&oHI{Wz%3 zL}NL}N41QQx93aw0Rt`*7b$TAx&{eKRGnY{jR3JwQ@FBw>Q{NJ3ZB93qO_JYG#$h4 zo_%JlgHhsWW;fOX))%a_=XO~37qawciux;azYIf}E9<9F-wT#|qr2?}4d^8sCKs<` z*e4_}!&80G5cMJbWI)PN%h^80Xm8Fl=yP?CRI0Gd9pT-L9|xLNYt9>K3byh8)5G< zvq{2@{b56_BA$m;N$A-Ps7j;@iW&vPZdh(|(Js0xivh|^^1V|%j1UrM^Bf}`aGooZ za^{+ak_HM*udO&GJH}Rj!U#=L+IGi|rbROF?AyR*$}bDX*873mM^!R;i;Ox4e-I~< zF|m$0U6G@?wkve+QKpn+AHz1vvw1`wETJAPgc;mQ2wmLT4CTdG##3#og(fKf40jlh z4GxE#bX1-yRcM!0+rr{@Ti@b29*bm@03sy5t&!>K-N9DmX#;FPhbKS|)8^YhVvzD| z5?8tsGSygyR)R3NI!eB{WtbR-Poj6y?P6++DwsdIKG{qn`l{N^&{$hK8>A|6I#Ust z%=m3>b_N?}2p$9~>YTw|&A<}6z7~DE*N<*nZzpaG?Q)v5d_~=UoM%E?QK& zd1-i;PPXZ330J%sM&g?{Q%~f6p%GC0L8Z=<&w!1e=L{|lrI?lJ%Ryh{e#A!?3lu_2 z)+a**oq--}k(co2cz*mFtTCaE!5o?z@47V_%s)dpL=A^u(p*c(1s~%o_G?;P@AO(0 zR+FAz?*qCo7))8e1VeRxeCc8z;kYsLgDsyQZTFO`OMznkmT*tbM~AgdU6Z=X^yT45 zx!DEeN$!zP}i-R$w{6;d*@VfTjiG`zZa*Z(q(bM| z(={ExE9O-`kJ+m8;aP(Dzz5eGB;l#X?Z@V+Fl=QfS5f~_hw#G)(U#tHhx-iS>0n_H zc1x~=DHzc5fy^vjlG3TEwld1f4zy+Fkbj$t-Yx&5KV%UA?AZ&oy@v_DL{DUvHizN0 zkuA#L*C7BMQGeV(e;QdIIz{Mi^eOU?W~G?G zOque*srXq#W2hZ>fBfpvS3>;$Bf3Ulb>dRCe22l407$FYQ=WZe1GhSiqwka9Lw&{V z@uZ<iuda%`zZw^H95&=ix3y@CdZxEXQ z!JINUhTJ)AEk$cCtFDk$9HrjvSgb$mv>GKt1~p<7akJ+8nhZ$9@y+9GzV z6uyz#KsrFeD^z4zH_jp`|10lpcd1mhc^L7y1=SZ*<7r%P{OQlfDxjObAC${L`Gh)W zinoTjFno%Q6{c~8FaEfFCOeFP=HBqDUD#=D>Ov(=;)#ooZeM6Jr{6hy`&uR}0R-4Q z*)0D7bx^1ug8wFFxuV?=k{V^{)<@Q1jo+0E z)AnHCXTUeOrAub#+tWa^bJ-8{%r}O~@cmkqJy+MAI0aJ6ARwb;oW!po{3k2W!*d|G zV-QM~_jo32I%MnlOvbQG@+TEOcx9-ld=)i1p~s$3(g|pW!8&Q!@cx`VhTX>xR&xu| zq%>%Nl#c5o!q*+k9YSBj=#}q|kxam`G2t@%cxki8koYEGDy}9V&-;e4Sbv%iOSBCs zx{dr@DrxAldgnsH8VWO{BXdZFEMVFoIv#cGrzu;2bbQ~}R5q6k=77Dh&vmEbf~yAL z<-9zByGH#C!alDIMOGNt6uFjeT=xo!ug3 zvt`dPvrHz^Z*}u06fW*^wcYQzceM7tof8x>*@J%+C9KroPD+?s%BQzw$7Z*oke2ko zJQjfjWjohyfMS9VeKtF9=i=#z*Uh48zCr~h*eW!R!+)4;XH%g^fsNGZ1p7HA>X z6ku9~?m|kNwKn^Vm~q!{BUKgMn;x%E;@3pOs2O-3e7Uc&}W&Q|Y>D ziF^v=5`__7zIb_VoJBq3tz4s*3l1d?zlw>)=TBdT@pV?bFnxD?_bMAkyE~}&S|8l{ z=xDcb=ZSo(*1IUr9mZ66dL(v}GW!+As4+zS)|Pa_iyhu3&fF(4tYx zNQB27exR3odjEkT%SgqkUDv)heYROdxAM|GRRdS|X=dVZDjMbUdzP@q>ij>e1Vjme zn|VwnTK87I(iGpazP@_hobb>TnR=|j zIB**lwlgvv8*aNXjm3H!l3T|+ z)-7M+blf_-!sLcD+B~Z^xvPjSAfbj8*51+}fX2@EgqGcM+6PfLGST)J#WPw)$39!7 z;OfC1n%F%Y$g|z+P;JxIJq^d@(rB&hV1GOiQz)On>0s(2cU9g6>W%yoM=XQA-K8l?y>sBM1b5;$Lv=VVzL#JIAq2=%419TC`9|iw4y};^+ftd4-i=O zm1n7(1lV#=55z;UJiaV_BoR6+j>V2EQlE?|ulZ^+4{Z@;b5TZDDg(4Bir51W4KA>B z48N&sw(-sc)Wuo17Pto)<+{e9$tuVhyca=>yQ!gMS!rj^n&9z3uY^I`gt^Xs?OL?X zm@MdHt*Mo_?f&w+#%BSVXVgyfQuVf1i4Npy&*@ zaWnh>(Djy4ZFFJVDAGdlwm_kHi@UoOcXxMp*8rt>aR@Gj;O_2P++BmaYp^eU-uFA_ z$605s{77c)$(roUo_)(DOvW|yrz?~54cH54%&anroDsu?v=yqA1va{;uK-+5{ttJ6 z0Sb_qAW$ggI?zIX*9vIkWiVT;rXeo)>8Duy)J;Fo^W$ZsVTFTEzaOAS-BRCwFHA|i zZ7iD8JNE93H&M>#Qz2Xxt!}?eFs9}qAnU7%x)Dnc_I0d|n{xdKSx zOx;^S8)5yYXj5?jQxaatec=MeROQ2?;LFF z!PhZbrFFPUR3J(>S1`K-myn8QemB3qdy6XU9l{P8*VP!j^P^jOfCcv-dxC-31nk5^ z@tph2&mkSBqb4uaafY*dgBq`WWfl9Lf+xhjW?X37pFo&NYGs+$T=)MjlIi(sx$p$x zhX|yhH-mk-L*Oj<6KMDwYw9-0e2({cDl(;iR6ttZL3d!_#z^$778A*#Vn-7wU-pTQ zH1#$G8}UoOqbyM=V~nJEv_{qLsY9B-s!+yGsbF7B%9u49;gl980(VB! zTUf8M4y*?WMiM4}G};b9N{o31t~8BaBcEE``_2YPBrY)KV|z;0IMh93^XN!i(sI$j zxbL976zN`xpIeYGv`GH2%-e!L^~z9X74yED5~QCIIOiaDI1(h7j?1hK?vRfZ;2q7Z zDfp5tT@Ie- zu1qtA55GY)^Q!hn^neSrcBtpwvNyJ&z;?9Y!A@hnHYKEE{O~BE>yjf-`_V7VSx2M4?-23@Y%c@Xe=O+A96uJv`p;7GA&T{xMqRf zq-jAzR4E%_&1Bo_Y#$Y?mW$_iyM}&eJ4Y(3Rzuy&YhPdTSjz;o8y^<&lDlQ==$M~! zkYZzEz`f4vp=%lr#a9J09rhv3oPrc$hqnjVU&E0F!|bzbTmgSMOASow9!ACWy|?tD z=bK*)1y-wLUpdjQAJfpDYtEP!o<%O0N6eo(uICy`$NMbZR1SwL6in7o!j&MR%nm_; zdcTdSpsr_A3`KuyeROM;0PAh^tGG%Bsi{{5* za38Ct`*~FDkQ|GG??j2+zT!JEW|x3K__rwVBKylFMAk^1>EziIt+3RmBVQ=5ujuWwZ2^-Tap?k~kO-x4c6wN62|IE=T(Eqo zx(8-JPoG<#Z303>k+;}q?!PD&KsY@k1Vr}=aM#zDjhpVNHoOWFa1}bIAWj(-#r3SZ zx`TWPL@`dWbon!gom`Z$KqmY48@$N5wy|qh%HdA&j8yanDaKs!<7ZIFeaWlFn3VdP z1^&^M=f^k~z$C#?fAo?!RXm?gpA z=;i)W{2uAg$fjmIdq?|;Qqr#DLn!cse_y%ssYuB*geEg-Y7yjN2Jtl z^R>F*_y7s@5F7?XvSjbqT_heEAbqdA=txYgws{%LUqW+V&8|nB)pJoXUnKiOCuQvX ziKFT|$j=9BF<8z_IRHgndh&~TmB);Av|eI^J1%pARMI}T;a2}<^!y)M5;G6F;~1<1 zgVv9$mQl*Z0X52XkV>W*f4u=nFGS?SeQ*JVcq=5>=y-GBl~^z?G;M87K5~6Q zUo{R3S`@;l=DnnOd*N2or~B7Y?JlK5B0zFw$yan1oHpb%!-ZaNcGZ?K6dW07$F znjNtsSZ=*LBcngRdyUF>puEeHih#?jdKo-DQKLOD^n6*a&p#NI7uZ;x&7=AukX*~s zl=xZVgmaEk47Osh99wD%QTt0jUX{ynl%5vR5VaSU&eCLIdxT9ed*|84xL=|lFg`bG z?!Ajkmsi-7Yn#Y{qVfRR_PyzVfWDXAr^>d5u$X)EnE@X@7lZ54!Mj$?yCsI{(Pz)2 z{6;MoTS5FJei2UCA@^b%tE=24)def$$ zVhIKf1mcyem~9)9XZe0#ex5+PJ(2v}?tA}ij1(r?Ocf%tbtG{~-5i}kUF@zwmc)V` z87L5jg{w^4 zQikaZ4y{(^pZ;zKx=-r^PC3;(Dtr2}3j3vtj1M$-zpwHR*4z!R3!|hXI2e0<*ptSR zU8#WPR5bnARKVzmrv~tW2XooO85iD4hp7ByWpDrvhzL!{jX}L-LU38AK;%OL>Py%Z zpyMbrnC|K7T}y>YINr9F(@Eyn1J$gDA)tH)}kZSJRyHgpt-o>PHjbsW%=kHUcGONZf-lnY7=Fw#1ZgYu?@&Cpkz- z?BYn72Dk!&U`($(7g#|+o$B&yWZ=39O@ju4*e60`2}r}e=GC5$?Qj*Tu`^Z{U! zHTMejPpCH{t)Nc-o(yseTDGJCO4r3#Y@Z-m64v#}(7_u1oHnN(15xK^qC1CIRefKB zmP*Y37mev(R3jT7eF6ntJ1kWl>ih?~CZ%jvUUTh;a$-*fNuWDj`F8M4^PSzjp1XMs zW$-m+%7rXAjJTtdlc}hRe6BT;{tJ2eD3N~s0!1ru!~;?GGFPjpM)y|gmv+*8rpJ(- zSv`hmH{N?C0%37DrWwg}->%6K87y3uJOif7jT;tj1*Q0CU;WCuP-Wz0StoaJy$#|%W$9+^m1(j4Ly=5vYLaK z&eFge(6Q9i!Hyg&X%O)gn0WCDz!FY3Tz80fjo8pxQf9>E+dpzknzH=wCvA1 zBnJuJp@m$_{1pf9ffTee3`ZzwHS6io2>*;oe^-HbN6(tRZHd=g_gh1YdR;bCDT-T1 z85hv2nkK9E@ZjK)9$Ev$zKbGc(2I)68Uvq4|53L{c%6}AVeT&px5;{)TSGyb^7##J zPaC_nRt^VNWU~*AIx~mc~?UI1Y62!M6Oo^x7}2{+6piH++6wwss=N zFi=v9gC1gH(E2mlc${c8R6BFPp-^RN-9D;@=>8K7GPXK;G=$hEaUfMM(+J;+Snc46 z`*794u|4N0W}_#>_G}7Y2%`u--wynK{Xyq5`=6QEtCvsHa=Hiey_WZBWYE%Pu43lE z;l?zzXs{bEbRFa;i9a1z;6pBK+2s676hFSQeP{l^N_fz%CU8_*K~Ab%Ub~q(IUQAk8;9Rxwy1b5>?G8- zqn#8g3x~o!}Vvc(Nx|~29NJ6gMZaG zmyv~^64RNOtIEXxs^TVo)he(x2={DObGY2$|i`Rwk|L)H{-0A5ZF^?rI;!^^R%Cu+3$ z{PX+Kblw>57*C+WijL5)0$+P$!c|ypsdh>51pZ=H@t^xeC`n4Muar@YwIn$Q(N%cv zizBY))ZjQ;kC0A;eAyBCb8z^-EcxFZ^WRID_3_S{2&OrCL5+-XDz+9a|Lz&WJBX`DX?fka>F@8U1+#l$ z%n$mEGhpClIa_-nSZ+)oo46zK(4ZUfV_Vjx&?|@C{~?f}yKqPQ;Mwa%7t%3y*|bIw zu`&_bZO7HhofHEb2ENwAp}xkQNHcljem(!;tYH4DnRuciwb0;~Do?i7D~%(**6lNF z$LeVJ2*r&TSSOlzWnvGz_Nxs+XJuXv0}L&tgz5saVxexn;%-Q2e)K8<$aHBfms%(^ zI#F*#^EK(!;#!KO5=%AfNY_I}dQL6t?Q_K!4Y_ehq~SEF&@|3{#K(cCOA#-3@{uG4 z^G6*hVwiN}&lk6biABcbO0*2u|L(WL4U-=!*%*+w+BYRb2aR%=DgPrx};g}3QeLccP`OK2nCN=rU;bU%I z{DRom1Zq{P8jo7YX+NFUwVis|6Hqo72Q+||STNg>BO(DCYC;PMPkA{s)u%687Gzf+ z_Ex9U{u?ibqE%zby|Rm;!=^_6=KX(_CnLG0`%cS0_($rwbEfhbn6K|ubq4n`KP}KPULi3<- zfOk2~ZOp*+SNJer5r~`94vhDG*06Cv{Ts=wKTLb+^TD?GipFEOuDTOy&7<`l=;qo# z9!K9Noca}>3d6#7C;zXbcg!1NW!vRv5PWZVMq}IvSr^2G4*beWWHx+~{IHT}*2VIDcb{C) zKOoYrjfo;-U7@Xywl+J@V^ejVC3?eR_ObsDi-g7>gxMYIQKjODq4NDJms@B2E@VUn zfVoP3ymQ8pC<7MF#Rbw-%(Lp|>KH!8$UKCd=XSbQ>A1`>j&iml^BXyFMr-hKW8;9} zA5MEaSLLlK`Cw$iv_QDx4vTjkr?jU+(~Ec9o}1>d%d0MAl$qbS*>N(5iGW5wXE7}# zscQ=)V*SQcf+;&PN2kmvl-}CJ5uOyxKnDb(2Bo&~(dsQM-a?||nG&v#X`yJ_lg)za z$a%*LW1nWNGTv9I=LpLvMJH~mzQ516+3h#@neK?M%E17!F^y0r*Bea4V&?(kcZ3`t zEhTr^(^wuvKK@imVf8S-j*0hyF*7%<@!#;fJ6}B)AexBqvSsnHa+kJx`k6i9as@tS|wy#K*bIE|v%^swoW zBD_+|%de|eXRm0i8j4Ljv z*U@1i>7lOGAL`Bhx}1f6-q%NB;rIFy!8dDZw?cJbX)%z$s%*c}Op1X5ZCT~4o~*>{ z{n^#FlAFOtx`)T~(b2B6{8=HHHedCP&bM8OM|#85dWL0Ee>7g>pEW3BN(+z45kpj9 za~uF@)5+;8KRU1e#SwAX881z*NC?GC$YqFf+t>USv{GS~kd!9JEU9wFUWYcYqT)u} zjH4pesteW9Db^ET+0!q}iEanptUUAZ#hJX z&v+ANU2nx=eB2a~AGI>08}FQ_9j%w}wB2vXNfbK)e=xb9KK}*d6(4aly+eVdpV=lc(t9>EpXLNA);M%|i8Ew>EUioU+x1>lx9cAA zxqYQ~LcZ$TniiNREIxDiBmaPQCkph}b_X17MJN>UAJuW7;<3j6`6=Y8qHp@7f#0l( ztP3^M!2I!J69x`~*0&))_GokGdU7Dws`JE^0IL`VaYyv^effZdLQ2Ml>tKc_F^=7{ ztCH7vOYXYpC=7@Aadjr5G<4x|TM94$0ug^Zi_P_Clsa_^Wdq%#2k00ec$GAH^F$|_ zAKekLsk;*%Ec0fZ{FWkEeSu1RN#R^f_sayK5=cV-4;UOqbwZ(|qrH$)9m?^HPSNf; z)M5L?zol%;4__UOju*otuPJwOc&Nul57{q_BCZ$`e0>}FR$;P4+aBN&LPEeR9cU?a zgPd%Uv5NkqHCvKHm5>gXE0o?}Y< zBog`+_5q#Da1(~lp0d{I1^r4|B&Fp-YT%(xLJB{_7Zp$hUJT)wcP5_epHv7PhJ5zt zRmR8!Ydf-4zS@l?EKacI}s^*nj^_G6@Sq zX3ik`@@9CMVU82WM>x?y0OIF6J7JyaKcieQLZr|~{sCzQe}fE>wfkO@dSa4p;feU% z<;+-4aJ}A>`3uMT>NGO`)19e>>dQw<`*S|9$Pw_^w}DrV=rdHoQC%U_qo*C9qyMbz zjaSOOs8g5Yl+F(%?6BYOhXK+d-L)5w2W{N8kDsBru9870j*L!ISP;IudyduE}S z@!)HuB8b8Lau0(w2s|RXpD>v*p>_rBxBzJfSzaUT!}Ry<91_2PtI8adry%zKo+f zGaAH(p36zwl#SM5?P*+dLU$QjWUATO3z7^8GS)%b#j88cxyliZl{@GdtbRxeTz=GN zJ3sdDYlMag>pnVjcj>K{w-Y8j@N&YX3#pZlAVoMlu}$fJHy zr_Ml9T2}st^lsT}!m@|z>J5>!+!|OpwEc?d-Gdc<75?|bck{8yzZ!;R3?qhx!qxRA z|L%%nKS)gU*e^IYXgqD9ROmf1?WOvEdOJoyw|mM}w(Salo%R>Wdplcc4Fzbhk+3sY%-;s%TR%rO{vDGiwLgA?E(@(?Zm49-oUW|EKuq>{a_HMTP@Xf8U;><(kBh6u&3)B09yV0;qPD;pFgkoi5v);i=o2UIrM;x=(x#ub zyzs%lfO*Pc>j=B6gnWkTuhl~xHuSdX8?VO1*BA$ere%VDL-o5fs2EBHdr_iD3Z}9D z#XLeEH^=Q<%`hef?b2?i1pghe%x(GXRS#>dN?Ul_MnN&|mz% ze=aXY*Db%~;0iy1?$Sikl}&{hqlU;j$O=_T$-1~u z#3#SNRGvDmfQ*Ju{PW@->Y%7{ozmU;2IC8xbA(>^kI*~m(|f1wGZ&NkH3@$TLTzZ7 zvbeJBj?achVxx0D>1Ni-I%Yuz^9fI=Nh0k3CK5k#c(PBn(yz9jk{>>+{hNVzIX_)Z z{W=5ZeDDPA99?G4qRIF6McLFd4|Zr?xZ(!J>)(ugUT{8@i*Z=tOmq3WV$z!2tLb~x z$Gh0R8NT-+p0y%X@iXlAZ(z3{w#m_jE_sc{fpR%0=$~X2>bdzpiSI?SN9bw~%b~yC zGcKDk)2DBG!=N=`1We}dK?2jxcGv@=wL4Lk1H%#LB?yu4POtlHlSb1eiM{?%+%7eO zA-H}BPhhf2m2?OJh-1p3<~5zuOIqJg;nJ|HN#g6Imj0`R)z;FC*EnC_5!PHuKk~PSWm3*z0ofKBIXx^rbOxK14X+SS zGEYOjjQAYTUJ*A~vVg7TGlSB-Qy!tLCgoG+*RO*)pQ@;Lq}EwzUPWY|1KL?PudDno zfP4k_2dx`d3NgRiIQPgqfQa}bj)2G!SG;k1&A!A&4E*Hz3Zm1>d7Dyh0R)am1+!{C z_lYuQyskcrr*1HYCJti^`;X9WS>jy7k&;uk6WcUTUm(7~dUPEuK?}lW-s?{V_w^V5 z6*l}%2`}!C?to|dC~iQ0Ix+uBJ?t`g+hL-gp82GOtn$6U_Uq4ud~yA4?S^wEvNUQ! z7Ok1hg?8>HW-e>G%v!vsw3Qj|E_nPObC=&BtKs___|bo9)vorl&VL6)lMs63M| zKv7HJiXywHsTNdRrhqT>I`Mk063}^vPQQ2=v$hbK_^o%7=GDE?*+Y{$U#ZWrd#yem zG+=}&*H$0Re0b{r{@VHiSIAnaRx*QoFJkPvD1zR`)B`V88eu~MVHWbN!o7EbVQ!sy zI8$^P-6xlc?AQ!#-pUnt%Y(p4J#5^wB`?kkci9IvxlIqH^-M3CJx zJAW1@i%dIw2-pbnpWN3rNig+~r=Sq0eaCdEMSN8v7uoXZ6Z@BugsAobc89lp3gq-O z-112yNj4Qa`G~%QY*8}t_waH5&|J-I zI53x1VzzQk!__EaSi1fJ^GlzDIzQN`*+4I;Z9-+05yRJkQwGqv8W&CX1QeR+Ab2mae4G3`&u=uioKvrXzL*a9b2x*>_fQvj)L6Z*2nyR9&J#8i5B?BUC;oso47gLfm4jNhe zWF_B6rtG(C?2h5d7`WTU_n4TnSc>Yo*>Fuv`hqO3e8jI@2?FYnhQwriX8R30G^@de ztMzMIu0=RQ4x798%59wZg|L3EO!S}!Yrbs3`_Beus>!&)0wWz}WDaM?S6KW``w z*AvWst~D$_SSNl~p$MS^P$`oF&JZbYH)m2d93MXWyL_R9IAbtbCE*J8X2=4cjO(ddI7o+Xl#N zJDIVC9DWR`5{e2}k|7?5Lh5d(hYpU_G*lqwjl;FGhi*c+|LuTdV$iiB)U*-aXBqtD z9o^4ukZJ3z%vSYV4vJ^_l|7wuyZhBWYZO~bNvKHy+tU8Z#Tfl9jDIY2C7+QQN7Vb% zujf7Ynav`6Yl)-`5HUmvW>%p z>FhnP*3DlS@y;q>nfo_Bb`L%n#Q6%tsha#UUeM1il`@?IhmlYN7np!v-`gD328Mi8 zOc%^$RqHiT^$w1>&|&o2lOb5yM-n2;xA3;RbxFW#5Qlm64H64@b6BkHYRIUf9%W}~ zp+If)KqxwV{0dAKN9cF(1O?Vb@-3bi88~8H<$uC-uLaO`|85^Oz*^M zci06%Al~V6Jw9Sat-e90e%BA)fyto!znRD6t8wI(uXKbwn$D)z{*U*`yv>sZyqFlHX%U=-Qo^hbFwLImzINH+7cjBlurG_kDIO|yA2#g)H}hQH z{BZpCa8LPWQo5YhO-i{5igzzZB^dJDw|Q>c>zsZxP@JSKyL!~&X&~jahtGCcAIw*` z7Y#!ElZY2G0qwvysMg+=Vn07UiRjiN87_xG^!Oc3yNZ}rWuh^kI^V7i!T6r;wzQRG z{}q`P9nu9-7mD)!fwM&V?OurS1yd;*I~dm%!l(9_Y)o-r)*+X0_|ZkXMo@B^$+7>) z-sxB|%qZR|HHP3vh-<^%6bTN4F168}hE(1@G!y;+CB(LPpCY)ZQ^q-7;{Eb&pv+DF zc0A2jOWQww_FqG8(H{3ybI5S@^bZ|p)ah54&_{Aoj56B;1jjw5 zmB!PpWqOhqCPv@t-=}X&BSRx*lLHuBqx&OYnD@s#ma<@zW7(d1@2UYS z3|GT2!+%$x_kD#Y1AOe*l%8^`szkODN6M#R5p_JjH@{DU89!@}^}($PcqiUyyG|f$ zEhmSI`4&Cfw=L{w^T6AAza?_oj{B-*iD6bDTzgW9I+oIF?CObfXz%}<}o>5>hn-$FJ zyi+Unuh4WQeQSdKYkMIn7HD<+&%vZzwlB}vokOSbScj}@z?n6&7P+BCC2@#uyn>QF zIyon-y<&X22=uL(7i5B~(cV=}Z+~Y>WC!^3?C2YMp-q)bQO3h4Pit^q89~JKU3?0P z-Iy&+KYpnT1_qssGM6J4A|Yp!GgCAxd6MUo1NGJ@cm`{&CHU@HK3d-{*k;gP z{IXyed=QIs6(+v#e|#!>}OXe+nm#1dLgSuxxTrAfnZ3>p~i59#XK z%6T<8AbLG?;P5wL!VsTI7VZr2vxrK53=*7uTFF~t%|bmE{XHxb%OM5XJR# zHTA0FXw~$%`0gn<5bYpfxNEU-{yF*oY$~9k@Va?o*pN_y!sno3ADEV?X+AXtHJ$MH#G!A+jEHXS-!JN;*o1C4*Zre zp^HO3|BWev+yHO+4HYnYn?5(|w|&>VWL#u_@fhiJ<8!44if6HB^PNu@%n(mdx%@4k z)~ELEC2$%5qe7HDbW&(>A$&vSi^rllJplKJ+xnYV<1*V0SlGI7cs9PDn{S0rHD1Yf zT8v&*XQ@%yloxco%UZM+zm8v?S31MzsyK7F!2)uD#{Fl9siV_fTBu-Uxa=4oQSx)Ku09zU+*t zE!Bw)xm`I&F;vJSY8t9!3$g<36dj{V#aUVR^&FCr5}{bp|H<2fydg4_;=O=8M#>Ie z-4z@IXtH_U@g&zW*z1Gm|IO>|cvU|*9iS7fPxW0uu7e>3`d?!gYsfrsGb75>&ObDB z+AUv5$6r%5oC<4{^3}`_{i?q-iESh`9dNnP6=+@xeLUKw{CX$M@3Hk=PRu60DiKKt ze1qZ@4j;0=fS3NjE-z_441;|~8aWIj9Wj)EJ*U87(4Ps?1WOjecI|la@%8{$qgW4c zu;e>qD<9XGQaTIn@Egyqo${@I!wz)Z8O$?Q@&O%At%ti_s}qMymgEV441N=Z@W$Um z5yekS(gKl7Ltc+k3wsc>x|;7gZ*j7?ld>wi^Eq`VU#P}P$S~{kr!AcHoTTuV4+`C1 zKyJ8f@!fatp9xw6JxtWEz#Wk^3)io1nfRPS9$v)617)*sCtITS0$zB!+0 z9=nKbt>*CfKD=8^V4~ze-Es1qrW@mWVqT%~zM+YA&O<_|ga8Z2SA%wh?1pA{E*CdcMW2WBz+0C0rSh9DZ|N zl1q2WMvSjkD)Wd3A}zSQdmu`IdS=sX8b)*X@8oAA{&};>@l+ebP`CF-0o^(9%UP#J z*nh)5#V4AFzfKz1#yH0~)$=x4Lhkx3c<&yY-pvj3yP-}@M(Gs|W*NdSit{L3AM(@F z&epPWf{AAI28V?_N``Y~y1CRZ(t(x}a&ysD(ocLHtEjBa!6SnKX*%4OcRLohsMXPx zP7YCDz)oJ=e2n$8HW=~qToO{Brv`CdOlsFN`_p>qKD>A^iPUf#@7TzbQ8nH&= zP%!4#9oX_?RzptCR2Q*w$8#DxgV%M2(alJ|xBB50*|YKn*Sq=bMl%=z^xM>58XtoY zGu0-^DEL1I0r@8B+CD@@P|WJ+`{B4ANNrSQEe?&O2UF7^N|f2k?*2qR_4&-loF=6- zC7XupoV@%Eq-eCesd+i`is5%P>B8JPtE`90RLdAs#B0K z5Eq*r$pF#oPioN#?Z%-0&5LsO6RzUo?A!!YdF@NI1nb*V&hV|1W8ZK$(JrxX>bo57 zf6+%LhMU99hbs3Z<$Iszi*B!fh8Aghu7^j`O?hb#o}VgobCm97=yH8GgS z&b2>g0WL-H^4YbXA9dfk1PVGq`|1PDu=b?+F9a(Sm$%n>nGP#mKNDG6|@y8BX<&mdi#Hu7@o9R0oU$Uj|Zd&KRAku}{VDz;7PL>_%fakCX1jCtuB z9{MqWtgr2rVk5x1M9`QnvN5)kv(@YSJRETaSdi-%QJ~uyxUoCe z+vUN_m1ZIH_LfiptH0ivk~d;__JnGqU%V(1by`^g@OpZ9Zr2kZ>?GFR*X>%{L-XRor3SgRY^AagT9uXM@ zpD@f7KIHOul_%IPKk{lfR=1U9M=aEGWyv8%6`a{cM1PX?SLfN=yfa)fqx&~pKRV(M zd{?nt8Lu1s`6t#?LjHtg3C}Vl>AQ@QOj>mrCY>{0U1ca8+u=#xq7sJmB)> zKWtVa>E`3<(kxBFI=&%tU5E!vWZFC%7EfSloo8{W1!Ks^ z?MiFE{flDXW!#Ffo5HV$W>^&}q>Y?y96zH2NPpc{&Wub5L}YhJufGzWOf(u@U>&~P zItSqgBk?Y}h(%DIu1J*71nd{cTrS1%2VE%>bk5ogU)bI`EQyqO30gfJDxs>IRe2rj z=hgYbVBXrLP3P0sooh4$8s%&F83@HJ4%>O3lpG>zj#+1+dP9hr!`)wFqUC3A)I4Su zw@;1&U9Rcs^>}fJ5#BvWW-p+9>B|eAdKmeQ=1XKgaPxK4*V-0s2ww1eb|l%NyvSZZ z+ca`L5R=i~B=ymlECd@bElTH|O5ucI0oY8=~NY>kg9xBGtai82ai-nsC;QtDD>O*exrEa z_EWn$Z1Qh2nWM83-zDNKlvg}0QUv_P!C^jsw9GAp%54tFu-8>$cEAEI+~Sfs<$%Ym z00h=T1=2dmxTa^o5|4YX&UrVqc&v5N4meW^ST@v zK2By2ZWqIcc+~E8E%CWF+K=?QML?Upq%Rp>R&t*^^TU)tffpL{OF8rgviC*{L$qx# z+uUAFL}!!L`OyjeS6AXGFM`oAzbXs(hMsnh4I!6zF~pPmSIEK^vo6wl=RXiq^>&g? zmI6ot!URLhI1^@C?38aGTb%R_D!+J?$eXRun=&;tw{mGx8_n@$Jc}Ht)+ltIGu&YO z{PmQlly~LLUf-#7YGd{@H5A?~8DF4h)=Qf#$&VkFYLQT)yJKTV4KD}HjvnkC$Hr6l znH7Ke=&D&?-Gg5yf{LcN#j>DhI*TKp8v=NqTArMjABtMd#hcG7`*wCu#faEb5z|TD zG0h%p6Kt@vQN@kbTM^Qk;bs{;tscyvu)%>Tv#hKt!fUP4%RR57(pV`*%Su=Nhe{#I zk%OtaZ~EDM?6PI{s-sK?e|foK%iAP|es+i_k8vI>1cTL28UM9aUq4^Ipn|Z;*VK&u9P{gbvy3OvfE98 zVszBw2-WSIL)@wM39}%pC92)(%EgTyG6@(oxlS=HBwrAh+VdD${Qmazmoz?AWmWRc zn_H*k$%V@np%Svo@y0$+%xN~}+)xOUvW3qcs&uOIxL3}e_xE@qulf`vdT-8gQNP!X zRWW43Fv*2)#5m@!h%kS%`)O~A#jDtJB|cZ(DT-HWz`jUP%L}VhNLCMDOKx;~(jiPt>UV z3HNsJ{@q@sQUCp%n{a*6w!mGKb|2LWnflv4{&ao7^lzmJx!gfuIh_vR{``<;L88ea zD%=lKqY_4)K2=;u{t<`c_6ysU)CmD^wbQPySJ4u8OH*97OYl~1>vg6FtqSYh@l{WO zB}mnHSv`1fdCjV-+6be*a1$c8HG}6z2aQ5yKe$q`8jOth?L9Gleg|G>n;pzT=a8N! z9?xy60~@s8x;GAJoApd|9*5$y{c5noa;;}I9QSsfOZJbr6!IY`XQ&=UVxu{q-`7);;T{ZM4Gvdvaim}^Gw5Ldmv!byC4KEZu zT#11I`w33f5GAAf`cCFk8zVajd>Kqvu{pi+WvvHqkZaAI$D!u)B(Kjd8 z+QjtToaN)SCastKol|^r@uvsS+LrkcnzdhDaY&Pi2ZUGTXllk(qXmJ5u=nc>b+W-Xze-2=-c_(QyZs!3QlQJHq<#e0qiMy&B{wVi|GYwxfmwDp-cjNg~8PE73M*EB5 zDxnRE%DN;NoB?pHRg|7r)l;Zw#9`Snn9IK{lnj!+f1p&7_?F8wPcdgOZ--#wo&4@S zMadYN6#S9n*!^}%naXF(^$P*LYp|WYxNwGw+4WDzLlGDi8L4mOkb`>YtEB^c(-xG3 zY30N1h17-jX2t5oHkpyQjYE}#nKLogwULW@&aA*)@kvZ$ zDfOWg_TagI8PSaRJ4`GBS;Z<^F+|Mb2h0NKsbVuD!&nRgm97 z88nG;@7&G6s+03k53vkz|aQu?Eoa2mToPudY~0NevgE zIx^W36G-~V(83MM8LMi3JloTd;>t$kSbz7R8JCSKAuSLPE$l-asB7=!=MTpnP_m#& zl=(nK`n?AU{i;v`kva{oGy??;+`gJOMIm)jJTe{5y&Y1|b`Mv;>-mHx0GPu^uaT}CGqoFR zgWY^^kSoj;-9e8;(N|Z*!r=w6|37F9=u{BOgnn@k;$sO{`h~c)zZM-h%PN_WA@w4= zLjn*q@_qYyaNheD-2375Tt?#h`*_~L#^BYHELf^c}PZ z+QCIt%a!0a_V9r{97f=CHdiLTplkc=?`o&OgVM@izwns7~1cR(=Bj&C# zo;y9Cou~1q=VD#o=zMwr$)Uhq!{~SY&J(%qa}$Xcw2Q*+vFm5&12{97mVe=`Qkq9x z8EsmGADd=NhkPTt;{eyM;t$`i@~I;y2Y&_rLOQbQZZ_GETMr*ta3F#001CpF=w{PI z3T*cvY~9DAw9raHPHzFOj!Z)QwpL4aTnU3{lua6l$f()yJ;Vj223+cQgp!MmNOn@U zmF9N3d_G3z_G3#X9{%FAE;6*zG8HQB+h@M}00V{m|4j|qFj#eH2}X5Be$(Poez6&O z-`Ccio06HA+B7{@i?YVoIypp>4R64(8_c=ATVKku?qN4*Rl*y!>TkEAn&8_`yJN=$ zS*vURiZSWDj8QPUvE*{%!7x0s%W%4tQ14?k3PS(aW^EVyiY4Y6MtG>RZZed9UQIrd zs-!Zgmc^mAD<9fJSO)RMPG_V(;4p;)(8H)c$VYwep-7M;riYSK(j^;_zIj{6`ju#c z9uR+PU3IIoWQH)i$-2#O(>VG)P=C)r=Dt`%&ohE6SE7+S{DS-AENPd{hd1N}59j-L zw9|ysH~&t@e7z4-32?*rY5zDg)zy7uSh=cc%3j!$^Hjh+*ZXtfy=YwZzB3X%@X&)q zPyQXHZ9+rT1VRn>$I=ar6OLgE?t8xv7_|h-(G<=!<+!Et0^$p2?^aAq!k6MALX6`* z4jK(QG1&iy_M*vA_~jP`a4h8IczaW-ude!Gov*vZ<4euS2=z z=~r?z>&jLD(La_+FvGVHv)@UdT#y3?6}mTaw-z_U(#NFc{j1~WkhBS&2R?r@P*KNG zfMDD3d9UX;O9ppxy_e4C2Rv!JSl%ipwKaxg&16Lq6X%{6LWwo{-VP2Oc_PJZpsTAy z?0cip6(}s#?E8Z_3w2Y=eJT6v48@j1d{4poR=$=*8)+(D?$+$lE}d3g%Dun(_D7cL zX@2cSj=H(Z?8A>tUksj%KSv+Dz2LF^lz3Jp^KZ={|D9BOlo5T_OG#1=XUx_3za*ZG z(Eb{~Y(G)`gtKki=*%I6Ruc197m95OL@>Ba*oPr{L;i@EhvNoRP5Di%+?f#eZ0$kQ`0_+@DsY4=lTH>>v3ByzPfM>`wkb@xM zM0CF!s`c*SiFucje9G;?172aF2h9HmZvjKEoy$~w#aMJSm*xp2P7a&3v29Czr<5ai zo>naw^$hN(^-xPiXw)8fs<`Eno6U`Tav0?plUqjsZRiY}^tA6=vq;2{{SGSpM^Ot2 zBwA;nVu`6DSAyq3qi&Utt#It>Hq5N=^2%n))cj})D}?rzWQ*;HLim$&l`K>|PZ4vjD9F zLa@rwM^WvV3XiL-L>1D-ZxZ8N+X2y_PW%EYzV0Li^(t!}Jf0uPK+yt(6M~!YW1i*x zlbQFla-V%XEI^>d)ip$^$wlz=oTF`kExNzQyLz7Vn{4^`aB6k6d7{}r3^e`26$yMK zAI2y+4{_ov51xvHY_Z0(lm&y)Pq7c%>Dw2~_y7kKtr!?&I41kwCoYbO&n%P2?iO!N z@p!@;q!X!}4t7F-$aUd@RYwV$cO0hc8bDr9p&Y5VHk(rK+PB+xNXa*dVgFx73KI`! zarWfeIq|mUd8-ReQ+i+ZZSjLokNhg>sA5zuOvYXvaV+W&DIVvgkS6B&$)0Qr4a$ zVr5Rx_$A03)NEd7J1lo7r&TH8u%e}QI5^$upMBq1(dJjs+odIX%SOow4jsnuAb9y1 z6|PNyAaDF%UJdkTax+X0Azx#Y^1YsCN$RXmF>|OOg>Dl{n75?cZZv!d609~*mdse@ zV1`6nRlI}2?}Cj%9%M|9DyQ8iI-R!2`-T3vrRl8AshU*8BSTK(cduW#2_F0!r`QJM zi*C#VmWd(Y(WN2pj|O*qq$>k zMjjT0e2~&2cniTZ(nL87rMZ*NVrklp&ff-e2yM~HDBKrT0Oy%f9FvAvQUha{1G$b| zPK#FKqje!$`EujzNc}X;eY)OXG77YN78>Tk=FE^Fl>Wu{vZT4bvs{0a9nH@7+urts zliF$n=Q>N2T=55*en#2GoDmcTpLTL#s!ZSBH137P;2Zowf0cUV}k;O_43 z?jC{#cXtc!?(Xiv-Q91Kz0Z60x%b>#Uw!Mxs)Fk3HG5jmF~%b!`5H5ucu(GtF$M_a zf~qhL!5owz-IaT9T}7Mzl3A4KiEGNr!U7$0>*nnc6hChNe=H@jzx#bOd+jg>!s@g~ zlbbrzgvS$tgS_6Du`O5d-(-b+nZL;j6W`NJ4Tbw*cP!TK>zJOwo2^Z=I*ky>_dC-n zJ=75nMwh#Ee`lZ9V34P{?-qVtXbs0!AW29Tj|lT*UcJ4YxplPlQ!_SMG_ZYTMRZR6 zr2GK)bKw(qL6sh`N5(K{Aohjq+j&hO!9F=t$=&<)2Qg$H6z{#)!|*l<4Rl0b8lu!} zZHO+V+y5o4DjX7FsDec4DFc-cuNZOS{IJ`^~PmAD*0S69Sl1lI@sc~~q% zj6e0`-qL-YA_LYH=@Lv#XmCp@qdAEa!blcc-9AyR!Bqs-~iCU$fYc zCYNft+pxDC%1M2(X^4|<*m|~<&0zk)lHEH#hK??Eg+}V7AW)JlI;CYP+(l0UjtmM_ zPYzs1b8bvQ!oIz;wrrI&C7pQ@pKS@DCx}sE{{ter?Kko|jm684)0K>-!+t!5^cU5K zVvMP3Ux5taxD>uC`y9ElCu{Kf1lCS&zVzBv5{?K3fu<}eGLy<8GaUOuf;B~L^rGW& zU#I=KZEbF2Ff<3MZU2B}U!@69jx_p@$lhX$kL96858bGK<~vBH&M|&&aUxRyJ^&hK zeRr%yrnt5gSyD)(fBbe;QsD=<(w~sLW;|Cvy*Zww^b_D-qe*LT-!rug?S(zBSbyz$ z=N5N0voHX0%%d#P+)|BpES+mkT~=jJ{4zk|oUgIudf5r!X!DTFGqUT>h>pIEkVVbX z{T%P?eESARIQ_s}!d$hNxBBvg^-rJ-FT!dUU<&r{An=(2e9z*!+E5Dt45gC|Z9WHF|jp5P3j;o0;^%sL8$l#8v05NMf=-~yKlrN!3hLWp02F&fWvSm6qQ~;SrOwg3(3ZwV6Cc~3 z$P&~vR)@(nvgn|GjI?WVW8bK7j|VB_ZyuoltWzT@D@$FH8v<7NOv_Sst;-^onKfSyGRz=EC&(*|L)W|R_8NhD689(7bz+W7kxSMlA{k_50`|BC%_5t%t^x#vox>u^o?fEUC1>PQ{r}Hs`7549RnsBk`BWVFG~;8V zQ%(X~GYm)!5V?;GKZy;*=2;yzLW4rWlNf6wXrQ2}cq$52ql7*sUhhVaXJkZJy*6*l z=4yHE>wDeU4?^E8{z(R*ZG&xG>MVW&c}Fe=fW{%N$|18iP3xmwlES>bp&aYQJA#7M zL!%@~gH&#H?XKW^2U;NCJ}_rc7aUI)s{?t8;C`W`OBQMO1XUGj6`Pa_Y{wmr zlaDhFN7&EiZq!56C1vH7k`96moKuG?RbDeCSeLs(Z0M@8Gb8(hYZuT>P1nVopO?ZR zNKe;kC5I?ToqOT*Cdc4Ctq1d7A#6WHPM!o6THN4*@9UipkQ}ih+KPkznR?$pi%S%< zb0Wjr+Zhryjl8AKS@Q$~r0sw29*)J*O{D#Ap=!=A?#r@u7PsS zkNJXQshu2WxKA&xA_A%I%%)9rbQI%Wjyb7OL|aqA&6vU0SHa=5R@#~O*&wreE18+oRo}V z|GONSd)vA7M|bkzK-h+}qkaqX${{jcilhft z=uy99Q1U}oOYyqfWG#~Zxy(H2aGUj`TP$cCU-ID`8G?LxXQ`!nNb`|sz9$AcBLM>? ziyFt>cv`oK^Yfc4dfNwsZ^;YZuctq&Gysd;&O3z`n z0#NkcM%;0T`UGS={9I#&LloWxjRWE}klU5d%e2W}71@YQb-Soxh7cN2cSg==&D7pa zM4xdUC^@zk;f6(t1^rdNcNvNIoQ^n-jy0ydB`wL0te8M!Ms|*MEF>0OJ0FP)Qq#Lk z7rJ~n3_Z{N1Z@21&{n|pwwf3U*J-=hYwAq%cl%^pw#H#N8)vFc+BOPBu{%83Abj_IBD+k%{Aq9BsIzS+# z6VF6)FqV2Q#YrBoP_~jpFti0~bn%8V>!noZ*E1na>*#7j5sz&EO{%M68iu`AhtKSH zxbwCdTD~IIG+XN)qM*wlII6x-7OcI?_1?qKU2+n<0M0>gvWy{PVQM%UzvDX4dYH9g;#%21bJ6viXJ}cB$$0$2TOgiBF)0L`FX8q7nJ7N zPiExC6f}OH9QC~L@#PuX)G}1xB^{EqB>nTlo3zIZ`IcXdd=ho|OiXn9E5E2479EA9 zSyoZ2ZUlAY&gTMGMTu6aLXV3RF-UBYM-%g^!K|z}Nu<3gA)(tp9Bkv!^jTV48i!ghteS_8A@xmx zl|4Pg7nDjnK79apz?d$2W@kSg8od|Ac=SFFhvhJildC!k;67JeqbH2{4kc`zSlwjUx zY8LjyQYwkh&rLITccLto`f+&kJR7Kq9rre|4neA3(4T+N`1GU$(%)uwLP`=jY(%}R zR?*F*Z||Cdf|t!A^AO9XN0mj*8dnZzz(YvlhLN14AFu!~k#iCRb-EdPy@~Z;jLOhp z&bQc#@>oOJ^G~-y{9XR?z!J`O;5hBcY>Fl}Gg~GIl{>8e!>ucf7{sX6OzM=?M z*a@e1I4A;_yA&a&8J6GhDQWgW)YZmqO*d8oscm?|ucAqG>q_i12ff{lX%n85^;GXl zO0TYmj7-(%9h5y1W(3?){2cw_mYwX@ni>yJT^CPSV~}237oAsYvze2+Uuy-RZFl=1 z<>-8-=@)PreD7CQi5?N)I_^x`d=*E0;sz%#)3cGPJ??Y&*~OVY2-Defr@RtYLfKqO?K8A3~?e1ET!A0w*?3By2 z{=&4D&8+SVXHxo*@$gZ-aaH`vn?{nYr+Gia9fo$%eF}Ll(lVoVw^V%Cv9$cS9qcEh z-)%q9xwm(iUJUk%`$O;;6423Kk`TF@MlbH4-}an7zP8Ms`NZv-t7s*0lp#Xb$>6N@ z&SirSk3o)s);9g%9qbd03FzrJ;>Rx<10C_2R#S28NOZ})VwpW(-!YjI;+Nb7NT!zT z9%h@;r730(VDlMG1y9(!JA@Q#P|qU3;<+A(wme%A(tz;VcY6&hE<0)FAJg_O84j~w z>_B&GhEMhWblUfBD|cq_2k7AxmX)vWOt*efk56ed7dM_QYN>$Q*xwnL=gDDD+M2MG z&DUhw@UJ3G3`jw4qQCLgu6)~#W-D>wpwIwwL(KpXD9RD^kWiP_qCM?H=Y+KnGi1o; z72VL-w1X%p8DwxCEi$x96gnpj9a0kgXi6T{ zao{5&Q{iw3hqMJA+Y;iTc7F~sIVV^W^O|5NcDa8`Dj7_AZv50T)Qou_U3&OEX4>g7 zujiXAoTB5S0+OFZDguub?1$)B=~(BVEWLsUQ3J5`2wsq>YXL;!hDw%u?6D~-gSs18 z36Fi1ZfKe$)RU*S^xm8DlJD5{&|cU-mk5_u*q5jY0g4mnVX1`3V6ZNnlq8EdM9pVr z(COP#?S(g*^4d+OUpexJAGh+(iszYWfIRhuU9dd3YsC_KHaBy=Ce^AL}`Hh&l+ zApJdfFdE)Ii8Ju!cukXrlXq$YpH1;IS062}rzF|&B7f$T*1#9MwH;j7In_zWm++D8 zoHM+96=PwwRv-Uh#WPa_%sH2fxnlnhLHuC0$oKYNh^&FzgEDgwXvAUh*6R;q^e`>yX+5-nUgqG)45zdQehcL}PA=^Rnbo{ps>z!{WjFQGi zSi#c?Im&egmidEBo`k7jIT1Sxv3R50fAE7Pua^&b; zmlbZc3R+jyXoGTTJm}4c^_pDWzPJb0@-vI7*tcvQ_{EWc&?5PbGp?&%8tP8HpB}C7 zE*X&({qag7XNa~retNsvOhy6zbb50bsR>J|%pU@OaxP$!J=rzI(fpbq!<=$+R|4sj z(CuWCatVHuvt)}Q4pnJZvM&_jy&bG~^Rm7Fnq?lRCAh{ThmTx9zz}pbOF8|^n=C9= z*!_0_mmcIihh&60f|k#KjKb3+Qs&3c`p=WF!mv^WRuGZl$FD=H)aXE=x;~w}*x;kL z#};tz>M}?sDl+T%M&Fst$^8sCrmB>i>W(7MhBI@w`tF^)^o`(}CcGR=?`0tL2FBwk z*FSG@RmQhyQIw88dw1y+42|$&H77i+VY^jr$aEWe*CSdWL(|@$%@^NqiLN9RpCD_# z$*X0vqVgYRP$<*x#Z!1%ZNwQ6Ow@4X>oyvLDY#Dbr>plA67y=x(xiPU;yMJT+bnVs zIOFj<>Wp4k{>0A@t5@C;7Rm$anyw}qx*_X|ea8aswcD(a=n4JE$V5uAg;@{^C znAL41J8pa3)j`1@hlou=SZV|t^J645w6f`Ts!SpnO|D*M28eu4(W_F946o*eR4oGp zUP4#y+o$59JSa77s~otk9{DiSGjBHNUhi_!B@uz*CqU7KIRia2iH1!|=^5AG&i@wc z_A`D&+w1|7?R*ti#`tQb zEK*mq#pq%8VO4cynXRK5k5Ih8&!?2bEty}+Ml6n5wvTq<-X_#RYSK^9E<4@b0R{@( z@81HH$-lR1z#nJet4>uAY138CW%pwb$(HF*3P)Y6a&UfxJ zJw2!WgW55W39Z6JELYs4P0vV{UpxcDXXHa{%%j(#Ky(Wyy0&rW+vko>w$l!6-aF{r zons@E=TR z%p9JRJjqp>qVy=tUC2R99CCD`)$5TmEwjLvUY&q@385`2i0g{CS6@f;^j3owk+ij>-Bt zgzzaes?5Ygy-7|5d3tweSbl2~5y%>*-F|kc7XJQsW$4dL?E^v&RP#QnPT(+v>Z%#Q zW%{^f1E#(xvSCS(Hn<9LX7YsPPC1lvIY|K#CPDcmoZ?ooF&GR^OGh{Rk_I_ir(ibu zu9yBQt7%A%6;tr#w^w)^i$w~Dj;RJ14)HSWn?iM=OoZt-zb$RID}jT7M#4fJv=nOP zRWFNUy6N2)obQi~RjUG{A>j8Ld+BM{%pBf#nz(B?ZNk_His?(W`{(Z!wreI-0s$j7 zyD3c}-$q9XpVK(IO<0LN)Ssl=_l-wx5Jme*(KyUe`P#U26kMHoz6i*aoMM}{qVqqA zOVy%yFAvQifB<@I!-D^ID~n6AuE{*lD6IUytnJA zLI8{R{vh(4;tkUocR8F^FC6pBmr$&8oEJLNxY3=fExN9-IFv&DotrW@*f!|%OIe!4 z%kO#@i5N2$A%um2AnWgtT&pU5!QuO!C7mPT*TUvzTkX3q!jn^9o<<)+jv=De*KfC> zc!oK+NrWd~5?xTQmLEQ=)22D{bcUa+&kn{~hX2%n`Q0`qFmah6`f>q6 zZMy6a#WnSYH$s3kQ}@j=t-N)gX~mEN#r#Uy;oBAp1UH>`{q~K14hAIafs5|Ho0t39 z@n%Hi9UhLZ(@rC`n5S@>8N-eK{Ybd8v<9LV6f>E8s^Im~$-&jZ<4ors!udekmHU_lClrlVe$_p)KC>ps2a?}rg|=8y5LH|C%*_xEYAx1Y zAab5XqUzEl!F6hEIg!(eI%8jw(DQ`O?hU0cD`W1r>JTTQYxkUGdW|M@5e|R@Fh@B) zLpyDCskqR#kYsx8^K9ag_EEcNGA-eKj%d$#wXW!ush=8kM22tQ!f}p0YMC*VjP@unQ7*&>9k?An9zczcmARFC8+jH zDbM#+mb<>`HyHkK@-efD-V@s(ZkpFU&B<#Cz|g08wsstM6}^T(+hkv<=h`xY?~7A| zf*5_OK1d~w*5gve-vL4YI?r(4DD}pH6O(?v6+D8jk(Q8L$v_=WtM{Gf!&glj?^*5z z2^R{lw>7_Ttt*Fi#(LAN$R#Ol4_4DwEIth9%WsnnY2Fj<{AlZ%0b_qK}WygH0=Q325O>gTeSmgw1bgPn4`5 zWte(rziix9n-F*-v*4}SU~1Jd6XK4WsSgMVWt$f5Po&XW+U~J`xr{8jOBPi)=D5op+N;44dt$=9Zo$L(RTV37^Rm)=%k-f47a; zR_JaxwgiC#`Q3#xa~ouPP}GEB>o=G~$_gDqxPL~=Y0hh_jf~%XZoxGx8H1y8N`G=# zM~pi=0S@dG))kCEB4Wa^SZn(b@jh|U*?NT%Nk2||{czB)myBU*5atKf&7fxKvveAd z{MzPhn#a-ko%1x_o;IqjrC|7psCb_K;ccttRu*pDn3nHO88Bg-Fj6TIcbHo`4~oE0 zl6UTo5Y z70RW@H3X#8Aw8NJYC#Y?foVF?j?;iz@e1m9mpNIDCAZHV3GBKs42EJkrT&~wtz|u) zDroIU?8@~>jbr4wFsg$C&>4Fpc#8ryO_x^H0B&VDIxa=1hHmnl(9)QR@z@$82b!c{O#t&r zP0BhYnB#J?MjYH@czV2W1+<7cmd z7Khw!y@>o=tq5gk{s{wM%t8O7?r`I4{tEva8XVaBYo<(WFG4 z9UNA5th3F#YY=k{X1+Wz81$}m-nCec4Lm@@T?rDpXSlN{+VedpLC=J;m|@n!CLd+L)*JRdS!0Ht7_7#rW3B$ z1PNu4Vwwy(f12#hj(3Y@af#*mA$|ar;8SiT4P;}jz~pa-eQwu*!aYgE%h_0!1)?Qa zc8;p8;62bGyR(0-XYq6CkA}Z9@gYq;R}me}yj#oufH;Pf3!NxlVbn#H@aHEN2&poT zFdz5Uet?xru!LUBf3Jg!2gM`p_9>zKU`u{qZ-2p?qxK}7%~k^s<(O#Z8rN!Xz4tEa zVJpMsZ5nHmg@^C$eG2BLMhgM*Y!nfIT_uCRfWP%KVdzZyXHk`)>n(EAmtkQ{ zoZ;eQi>mW<0aoeJdaKJL`Q+FgasLM&(^*i6^R{N!OM^H6V$09KgQ>stE{K@u zki;XjBg%43$1|a+RjXssXM=N6P$j=84R>$qKHl!Ry^68G4#n@XSOW>r=}2R=O})O( zE1UnCKL0;N=p6Ifdw^N0K&{#re9%2n%!CS&(mym?!BAcg3SrdC%ZqX$Oc`Q#Opx66 zx09jK+ncp73iI&XGcjgY%Bl#S4}E}?j!NO(_iv~zwQPnN9R760$6kPWZ!c7of1FH6 zlVt+T$T;3>Cg=3RMhA%wckTReE?l$dV(w%CYvYaQ4>pYn50C%AfMcI|ukTohJEFro zkZwoDXl|vc-{72RqrTnO;=|S_TWWfIi_7XY<_(F#{7Bfq9zRjY7_8a4AmQWIxvwT` zE4?ziZ6H4>DeP|Ue)7;1a_MqIVNZ9-X77Qw`_R3x5D&1lSxO7)=kj0YUZENp(Y>jv zs;i%Cc<6wi_4AQPvrS6fX5vdlekee*tbnglGu6YVSi%-NG&cqYpyAWgyC;~k(<{Nf zIXp0P6_^Dzg6~3?rV#zxM|Hzy>m*4?i9cD z^Ixt1Q!5Yp-0Df`($K07-oMp-Ufp&wd2|F zI+6qp3e+0-(*h`M3o_qgk)e>;G@~X#^xv^0}By-=NKC7CGLV z@r}LgmF`iI=_jrP(I9@>_v&^ytn*8rMESd1dT)+OEw-!@SUeO|0R1+l_YR1LGE-j# z5pns9JUyG##YH5)Wp$mp0i;cA?oYYm+Dm2H2yoZE%_*_b*=E@SiTma5>egUtZ?N%u zj#gWVpJkYkL92Z{2*fTC z2>w`Qq--DU=uZBh%5#9*9OQpMdwpBwpni{}n1D8pA^wT#U9MY;eSJ-g@nAC_63`)w z0g(aw=i*1h-~7z+XqxM@;c+4dbb!p_UyM*r@`Z#fC_9!=e9&ZshaNu1#w3{3-}2$H zH%b+DB-~rpk1NiQq*^2-LqzAANiBddh>B)WA?bB zQy869ZT9AxifW@7WGN*!!raP}`~mC|>I3b+vcfML{ygJp-uqWY9jLW%>r>R+aNKWF zR!rQTSqZJuVvGMepm%8UANMamI?!YPQ!aXt@U9V6FVQYjF0zVR8UlfB-O%Xd1KN>{a__dye+tF)~gTVXWkE{rToz_AX!iznS7BIwMNyr+r~P zm`dh&n=D3zirx^D*7$#FW$VO4l!etv;=3rJMV%g4*i@$vE)1`1>J+oPu1TdG9fTaz zb7<#DM`;X#;T*+4RK+^54nqDJ#>b4~gD#_;YjbPMS%tqm5&kv2Z*D$wq*>kHvqAt^ zG(K(bao@~_q+PHghvS~4slr_AA6L0X)JHirxaw2_!6@<^&gm3#oSSRkB~o}}eTftd zF!i>4RTYg>v{*Z?&4Y{<5cT#$;+*W)APgihzpLKOdfRtV|Ap=Y1fHp$#P&0JR2?ck z7H2VMi$+-J%A#%~@SK8yd?{~Lo;TW{FFb%Cu2h1&c${%C&0sK?R87H)@Py0c zG8EeG&Q3|HKgG8yqhAA%;_0VZ!I0$1{D8dkD+?~86!F949_9Z{#W%Zz%7Z^J3<9b5 zRWW!%a_HHPzVu^e$>_CxH{yQfW}-57^nwiB)|Ss0RC`uhMv-QGlM>M8dFW8L$g{~f zfWs$xWB(tTyuf~`Jc|^W4+EQ4Zf9Qla|YpvEvyZRT_$qF+e|{8oM&y_w>7jXh;j(q z&_-Bf3Jh9p)~8bFXLM!NTf(tnIftJvb%O{$tmvL%_ z6Qn4b;oov)-z}L;sBS18PSBZ{IpgHy*gh;qMnZnD(DJK}cVFK|9;t4cez>fnnQ0(X zjeZOKPiEy=>R)beE%hdSO5(d0Nl9R|v;RAAC~Bztk$T z2l6 zLIbD^pL$`UK9T;?054Xd5A{h#0LJ9Mt4%g=tU3q3|F`kQ>#k%~YYcB9?6SYdSluN$ z7?lau&_gURUYpSXxW){SGOL#nu@0%=@iOTVSfJNTC!-1YTmHn}pS+KVAC>?_ABRQy z8#dAhOQ|(xfyu_}ss-6-r-8sMrl=<2-P@PCd;LNX`6rufh5YW%8|&XW&D+7rh`T2^ zg%gUOuw#d6{Xb4jlM=r2cNA_F&q^Xc&dRlwj{+`S$&XmO!gu%X3i$ah>VLq)w;TVd zME^gVD{|8VJDtngkASJ7*?;c2e=YA*p26niZyFAXulpQtez$uH#?#4Y!K<|v%|}3L z(;@9hLU%+l=v?RDZ3Hg7mBpERqcb=y4D5=X%XQ4hwi!G!p?{ z4@w1S=fCm4Y3qhIK~Kj3c3pp+NB;iiK^6DHmo?6q+|5 zJ?S0kuzFiECPVyLEx^jQ2CvM8N{^g-TgRxovHE(#u;BXQZCn3E7@@%`0eJ8%mG4FF+WC?GYot>EIfO z80+mgBNx@XQiWHpu_GL85&gPD7uk2p&Cawh(q)7^C3Kw;BTxm4lzGhQL%6>8{y+Kz zcuz#ABqSDS#Gx|_VCSGr)=)Ce!fj<3%w|GYnauNPsL{k4lq9^{6*FjwO$n+$x9Xg` zeNSf&7ttB`eql8(Y+E2%q<^ahRJ_j=D?|nw`2O9>$MoVRy=QaKv@^=SN0*5CK^ z-&#Iy)WTdYc!=ryANn^bP`_hfNtAga%Pm}F=OY?JLOWd&%ws@okG zaB{Cr_}%!h*q_cCxS5lJxa_;q1}M3E8wyU5u!n!Qa2Yu>rCv7OTNsPa24*?CEz!aR z4ld$<-CP3Hz^>cpi%AqlyrC|UNy%9;@>F42qu!FMoq9fskcMN809?htPu-i$?^ylW zWCc;sa6H<*jx}X5ap`g(Vk6S~{md!f!FU-VJ&!gT~ z#^X+>9AOzW$B9uPX_QO>%&2D6K}>oP@o9jq5BjS!JG;o|S?b zXCqqgp0~zv^AALDTpr3neufi$djBGn*c~R zf!TnSu)T$~c4S@dp6N*<bBu8eyOo1Fy1IfmZ86Hj8ZWel79n~#b^>+Qqh@h zuko)9g?D$o3>epHgmEZ=TAvzrS*_?r@zDHQIk98jZagkVxoojp_d5f)ue$kKl184v zYQbwD<68n)_C7p^gHgI2W?I3iboC|EB3ZV*Jz!UIuf9ju_#sr{YU>16sTMH+K zoAUcpe}j%&IEj~p#1>Vwj1PpLV?lKau+j|2cme0iP{YY|){6_k3{99u%e?cy&I6 zL3C;&Qr_T`_`L6@V<#)xR6G!nkpj=oQdug|*0+RZJ%I*rhCv=l< z>|y$TpWZ9!I2feMIDy?(4eAiMbAHfE0cA=? z&%_1z&3!N%l6ucc2={V*h0Ruokk&_(0)M-gl>-p$lI0E2F7X(6xv{SQ&<=yR_(*?Z zmbN;SmzzlfUN;;}Ki(vXF}s(=le|3mQyuYrPemg*;`|el;e282u8%_(;LuSz<41AA z&`6W`%8BE1QO&CGMZGUF+=JH~y{qp1hb-R4aEu`CpsuFspCp#Qe&-Fcl*{-Cl5so> zy%H};C(MDnV)<3LAy-Sq>+EO?Z8uEg&|DB_nt~t9;_#Yp@oUlSkAwazzfh)i13t#@ z#;kWWuwgLx#Pij)rXi`vu?pr$yln!@kO#1+x_pzV`HT$kKj?MBjI7`jlva=_d^5iz zl@qxyx&jUyT+I4&@9Eg!G~p?J0imcf%ou;OZ%kT=&IJkda)4xZDac}Mc@lbgOOKuk zJ#J8}UvM@9KHymL?-v2y8$?e*nb$wS-rzkUs6Tc5S5U{ukvyPGLq+qf>wG^poEh(@ z=@?Z6tv+Lc^hzCDFDbS1p-wB-nr1fc%K1VSB!)Q0wmUm8xha)D0_Z+@VXG1AI!m3Fp|<7Lk4e(ANWv0p};&*dVVy`;jtFo zlZXeN>VUY9Uc8_pRaTDbB`l4G%EThgoHJA@lv zwZOO_v|gFx(Knr$Zs~kTMj_!*Xe!seeqF1s6&3HA-3qikFZej7^oky&jg8W{uu>7d z!Epq%Un_PD?7b?+wjz>}L==tj9YqxRlao@WAIKM_!mmYaqC5!iemvj4ohMJ`I`DMM zNCpsJb`~#WH!Pl{>M(&G->6OkC1P&)DIeX{r1bE#MKfXSuw+`e`5J=Ymt8Q-OaC`A zrJ~EP9Mmt@a5mE-J^nE{J2LrIJ-o3}e}j7b9#l3+I{=mWcTk7POXc=xqAox!`38_m z=@{8-$6&S7BojP&dsO_hv?K2M>7nW6>iv@?i?ji zP_T&Y4xQmK=k}0$Jl+o(iUe4zPhoz1u6${!{kMTc376s4c9sb}6z=O214C$Vr3zTy zIgrK0xFO&4kV>`hU=KuK34%He;mh_91F^%PTsR_KuhT&aadOwcH2Qx?B5>2W#VL7( z=}7dF4Nxf-o$er-jmnL=nxR#01JP1kCgyZeYz?-lVOXS&&9orVn#DdC)!|YI6_%*( ziDzuNOE}C}h83LF(j`EuYxQCS8avltXTjVnq36}hYVcJ6h0dCk0pf^)(@~~jcNk=YpRKyM`qf5<-zR%46pH|* zmc36j*{Z_nPCSZ#U06p3RH$Wvv;=6d{4b(dq{kUXZ}vNM8&BntiX4D3I71q_g*}yd z8Gr;Aj`|mSFJ#NIyMl@d-9E#E%tFH{{)5%M97-HK9BZTDhrv}10UwVayA#;#w6far z#--`h1uqL2^<)!$Ob0$MWnc~SdbaDc2+lEByF%H8l@xM2>F3ykx2fBD$Ir>3Wc!Y8 z{{CUnh*U~7PK1LXm2Q^2DpuGKk`7veTINI)yZFqCSlHL}!(d^?8|@_w?r1~B3bQ>O zfK=!M$=35h&dG_jV=F4TPKl>|n#k;alwQFMMCQ*#vO5-0F?hNF@F2IN^fV}bHvqj| zG;%Exor~Dl1@FnF7(A}voR%+DlY7qp;RiF5Si4$C4T2rIqg>PI>RUGbDrnB92o+1$vwmpP?5lpm1CGDIE7KxNTf4+?kDy!q| z&_I3D?4Zb`d|(7{*l&8j@pj)8Oi*7y@uoQ1`s`NiwDAzVWs(b zcI~xs1(z?WAgBCqBHXS>ecN&_;Be^pk5^CVsmpt7g_-v5$CWn5dYijtwqx9KGPo=p z#Z}Px*j1YU3KkMAKH(PL-58KW?VLqBGIKX;U!J^a&aylt9(uKAN`ojtNkVz1^ zBwjLX>?seqTKH9sY>iJBg9>v3JsuSCFZf*Ta1S=X0e@$!_oWhhK_9knCFzQx*#qB% z|ETc}Se3{Y$-RMGb{d)n=L^nhMvB%pw^a#=)@`OF`G^~3IIn+x5Gehb|}L1p9e z4Q$wU`XQTu$qN+?-3Yod;s}$fNI6_wji>&{_pt?Lk1p8$UKgFVKj8 z8hg%|GtiOibuDxhP>q~>rB((Ksj-Xv0NCk&1-r;)LaR@){F-0%j^ntkcTYXSdY`)P zS$eQTTM+bS1}x{XYtuFW(>J486ObF>m=SUq=Y5aK_3@%Qo+gTMe6Ay)g#Jf<6ldWU z-0~pY+GzGA9$N>;GXS(bM!~1)xN0#o#J~;}HiWweYj}NLZ*5(#be(L>{}ROTcUBLG z*!io=YD`y)i@wFHs%Y!3rXGgem+;M_{nLo0Fe+}vnL&H@8&jQUqlxQH* zjB#2&MeaJ^j_p34f)aadQ#4C>>#CSMU5ZH4a1<@Z7p0_uM`aNiOo%Tn&AYe$CTLze z+Uhc)Bl#@%P#vwa{Jdkorj&acs3M*Z>t8LdPT9JT8{}5?{ILl!!8TPMWSenPKi~zj zOPNZoC4W3qT3}x7s>j5cTPg!K&zB3|q@JO)D9)CxyFNuXFrMAo1Ee88T6=BQLyf-u zPRioX7Sc+~uE#VwcXBik?5cEo&8vMTUG`4wO{m95i|6ZcF%{?HI+o)B&&y_PR?NuD zB{0-|Z+ZU7p%UQ!)=XhiH=2x}$}g8^r_fx9R~^dr!<6mA0S>loepyA%!1dLRNLMg& z)~&I!^F*}|dX@%O!1lq-{9>D55y9gJI|rjd^hx(Q`*S$U7o*F&NifLfqaaCY0;1A2 z_iU>ZDQCwFfx(Z_fT303{^ox&uKqYw+&~}vY*7jnmHdSMe`Z?k9P48??YMXHMbX!ZvW<$rx z($%zQI8`uhspK-XiuVU0>-#TAu`kP@D5k((TU~ip$zN0Gm@Ve? zV)GmutvHDYn*NU|;a38VN!9MKY7Mv6&zl|qTeX91BsXc5pyk=~`j`t8;OhB!3vX&Z z_mfJ5*ycSU?^obMEVGUkUGqbskZo%*>*-3)QsN+=A6do4-ev5PQAJL&;G=x)R8F?F zs4TCV&re~WrM}1yR34r~W)%rbM8+hb(OzAAh zTwkiDc|ZCT?AMHaYkz<~U{wu}p@LLLpOYnj@H;kZG>C!GTJ-2tFwnfa+3<|XFsZJz z)>=rOR^xQ{6Q?qy>I6LJ>E279{65^19P%mHR4>}I3(A2IlIZMNi3uIlR1DZpX^w)g zpZz6e)r@!StjKeTQ6dzu^er+Ij9pg-Exl|#n_Q;(N0g!e?f}Un9QkJm?C|$c{sH5X z9*Af5e6|}{%KG~Xm(N(IxSi2`m|yReBkvEcxydNt(npJUD{_9EDD63vfHxcSO%jmd zdMCMDVv*e=(JWs-bqI|QPw+S>1ni+4pIj8l&fh1{@8a!e=S%T3v5p|r#}Erd+*F#Z zXjU_rEhhV@ztB40?#V0=`dYEo$`cC!c{G#0*sb8f@`P3R zDm!=_STB21ru^!D+}0s0s$FLV6j^*lM-d(aghe3jw-7&QdzZwY z8#Q`qPyuKd>(Odc7V5uyXIna$sL!@s5ePy2SeLVJWB zfLA+3HNJ>se%oCU3!CHCnwjOL;W{gZr{K`5OVHSxClZ-I(g;|-3MRHut!8vAkO%@f z0@PFSGqlsg)m~|FVp*PTJfy9Upfr4nbltwmrjO7+`K8}mSr9$>c5cqb9gg-9XMxh( zn&8uOW1{G_KDFU6L8D=$u`nHpzmD*fh(nLF4ZT3=8{T(V8WP*k6%2`jy&1re|BIWE z;qr+WxbigpKA@9AAivSSj35(!%)d7dYEoRj>#i#_XIjOT_ILLeFjh`nVHQ9H3_oI< z7?;`o%SrPc4y2dt1M~UbedQ9E6!_t%YdfSm7+NwVmFUjNmP`BDi9#eut9e#VZ%D5w z8uw-~h}Gn+igv8P2vYq4ybO@S+{(ZjUsI;q-_gx8&%euL$ZsPEfHpdvx;>A^;PllP z-f}+=juX9XUA;5?a-!O=I!+p6v84sFVQW1O*1rgDSqh^G^{mq&%KMDBw&7%)zm#2Qr%c*_GyU9g?L4nJ1g8%#}7 z_>=pzi++qr0-NKe#u=9#*(fEyXI)_p^4nx1j+!N(SWSKbVegVQ3}r zBUca5RH>3l!_jiqJ@PsZ-&YpySRYe$m(gmTra?%jzK7B!Vqi@BLW82m_eD5)`B5kI zf{I(nS0^Rjd>UjE9lNEFZaqT{g5%y7O8#jSppNELw`8fqXtl{`cBnpI_h_%5zXEWS zKm6gZCca+qE|FeWhw=KB33OX-!EyxSvhdT1Fs&+ip*Et!m;Yc-qH=rK>Bc|d@|fi6 zxp=^`mF#S_v}N09{bWuOlg#C{iNuV+E#-qmGJ5{p_WlzF+r4Ru-s?J;@lCmsPSjD`(2g6eyNd6;G zYZI1JAT?KlEunw>y*n4>k-=3?A!`&qsw*7=S4AVM;%H!0G8FBfUyM~|KD$6ULUgV4 zow-qG9f$1A&2ap?r`FbHK|Y}&Ub6tFBW~O_ls;jyl`8{`u zW*Eh)z!`zX&AWcaDGYPK<(l2W{oIxh@1&so=+G3Zw^#a}gKFee%x8;J=3_EM)W?68 zhQEClH03Ve8O7?L#$ZDSw>M_8_+p{z8nzP!sHm~Q(t2-0XK~eqIU4<9KH<#Uikf;E z7T}fJTxjQuMge5V%|Hf{OO>P+TZ8o{1F=B}v&J)WPuTG^+CA@$+`Q}vMi#~wd(Y=7 zi4SfG2uuFJ2CxViGd#DD(j>AcSRLkd^i-dZQCA%7EgZ zY8I!*6q8 z*3fsz`l6!YzJ>CY;hlAzu}m!VT(EQwC+I1VGqdrec75?~tecOCP8E-8g|#|oNhsXWRQx9o?E~zwd+xD|yu=o>I zdZ>PkeG*X_D;@HSYFapX-4pLCRbl0OuVgdXP~!t)y>}1yx&=+6Lo+zO)I>gMT|c_~ zZ5zyGEG+1sAR*twf9Nh*H9{FzAHE=jfJ~LsK-53kCVv0^z>kXCZwA*)xUP+y!R*pC z00Dt`umVQ?SxvtB>T-a4^LB>?pMW)0R1jusec6A{ZJ;q$0sbA+W(DNI4r}oH96*db zY06*)?XzIpl(saXu(o$tV<@4yJq8nEMxJuYK%-YW?$~7Impgav%nsM!(iD)1OU4tl zLR(0XWHdRmGuD@xE88V)zOUxqgs9pXqrWl94=X z|J4vkI!~WYRxo*YA#=ye8YZ3-z|1j-L~p_@A46|jf(e^AN*yDLf{FGc&L|tkO1BvHO)WopusF zd96?kbIJ5OZ8I8+&x_``np<*2{6OelAMj@r#^A8w2`}N1DdtCPmi6{8BK{5w|KT&k zVA3&6bfa?h&0HglyMfcPKtWdoDx>2^@V2zZxMWRx+<07a;s9pXNrBc?b^Dheo~g0l zro65=G;5rzh&cyX2uJqfwLbPVV=nldR5Pj455FLa_({2Xk)+Mgz2jv|m(%nAeGQ6E zca6vC%fw^y#~5(heb^)^$Lfffe`hT6pb(oVHkWoZ{zZv?-MUxGn1?G-z(i6J+oBmt z%*r!UR%0AzS<>wuH6Rd`r4t5YHBnCN*orMs#~%2%UCuupL39aYU{L$8lJfn_L&TWg9>lNV!|&2aCq zNiL90-qp6ys?2Pz6pJl+?rVj*^i}b!q4Dg=%Iw?#Tv*!-y^aqN!gF7M**Oq-!%j}B zAvU&_0b3bw9S&QUr7BS@$rILSgb|*%k0Z)LX0(`&cb>c1(PnJ(lXtM@eK0UMAHL9O zuq`2!{p(!n>rF?Mdvy=CCQ3alel==TA4_TItoBEBQ<@nX%b)){V<{iSGYeMs(UOKs=Kj@LoC9QmSt9yn8dow(IH&VmFf= zGCD_x=yFTdy6e%vm`^_Eqt0p##QTWGxc2z#`J{wSgn50f2qInQ3JsElF|gA|7-35T zhe+aJ#|>H6|Fz}_&J|9?G>ox2-c%X_`5b4%5` z>lHsGBi5THFQRg}!H;9{f2`qFaL+iyzURty9q9TRe&ai%I%g%@AnDFxI>Pls=H5elq-$-O79VS(~WXf=#1cQ(8Yf;0!249A{H=rw7(6Zdj!i z^LjH8lwG(-mFHiZQIziv8){6iiFO-`QXk(Lhx?r;lviZ3%kQ%I$+fcd3O;y6ck${& zqiQJxMI~(Cp6~=9k9?oF6}O?N=}63~s}Y#BfMD+sdo?(0jKT>Mb!aeqW2$!z*)#drgHMd{^w(&YQtdimo(bMyRI*E``FAQtQd1OynTJQ3)&Gu za?AB)n3H44&jt1jD`kYs3Sor|D)`6egC!QxoQd*}FkeH5PYb}l$hCgT*PDPD;Q-}) ze{QvCU+MQF1JvK|(0`x$v~Sjyld=irqkJ;eU2Zi+DdCkV=0@E5rnZ=)1liWlz$~DC z78U*JT6dvz=+3OX%4c=|q{)(~E@sJq=Jf!kI&R>7dg#N*JNF%{{hmaQm>W>WdmF^P zvpr$U(o&(gTSILu#C3XYpoP4|~keOnheE^qJ@m4SWgP0?ulAN8=%VFl**XtDGQpK+1LSj)f~ zJ!)R8wfPIvQ(m&G3x_IePk$+3=#1k(3S$*VRg!Dxr=07yYjgRZggOjY0ulwt`{iCi zu4g(VC##(m8XeyPN@lZ#C0E>U_C4isn~!g`QNZD2W7=%QlWzF-wAbK}6|kExCI zO^Ytn+SdaPj36p1@-^ak`b#fOqQn|&6R`@d<26EFV`HL%s59U}jjs*;T(M9Abn*T^ zOJ;-)&p4d(WW=W)Ov2Z9+*(;(@NLGt{0q;ZZ#;2?wh%g7=y{jWBOBS{EqO8N2CDZR z!O_!rs~|Vn>q}KdM(5x#!FqNLK~A~`8}6D0|X9#P+=IIk(cw4VP!5VBaX7o zF#vQIwv0)!G2`cDXr#5V5%lLEL?at~<_pBbu^!NTT{Q~5kz|v z_=m@8bwm|c_q2)hSfOW+43oU(gNtns<{MttM040bV7re_>V_v_v+T)maH(g5OsaiW z?c=SYGr%mk!T3!+6$N&X3!C^-zqgbc{W`md!~ZFBf|#EgwGCWMR(HQh>M{av*VUP* zSXq5{7)+}0dn+NNN zisc5!%mzo%83&k9`AYdpX)8v`<`&USDs}AAkWTcJnq${Q-xY@%Jsx>8Njq`>{iURXcFt2#<8dw>457CywhpLOn)u_paO`WAWAE(kv>Rsc5Xe3w-(Lw>S1GZk=7`ygsM6Ep=Q z3Qo>OZ?YJ?OQUg3Onz1WBNU5zeJ2h@lW%){BW^x#zL$>bO1S?5{5x;EJhdf3a-)ba+gygXZ0Ry#wjRGJk~G$W53^^NTG+N~N|Y4t}Zs$LmZn@j_7 zNjGM@Bgs}qV7l>e2jKTxig-A%phcz`U5>0Xf}t-o#-hhL`SHGL_a3bxnM6=!)jJv% zaAjISE+9~u^=P~Ce&qHdf^LejcnL?alvJ5Aks_gQs$fkmS+R z461IE^T;gzo_6$Hl21fU-gWzgZj^C!9oSZ5laaFZ)o^8V*-~3|r5tngFOgREC|uV9 zOpEF+o~@mq3(3RphkB_|W)PgiQSs@qwA)xavAY}Q?BMtG{7>u*5-px8tvO>tURPE2ccTl?$hd&NKw!0$eVW2;9PMz_~ zk7Jop!-C$n(E6ty`Jc^OK|#&zViS9S-wQpKisHhD6;-vt0$rIQ{G7|0<~&FJoPWErP= zV)1Mt2SB+{QXZi{!S;x}n^w7RmH&zc_kz`jDQxFoEG#?OF55BsojpCX0;u3frde+E z3at?JwEH$!lxLDqYWy5sH2hB_@^3J8lMe`~2n`M=Si_e1uFFK5SLTXlvarypZk^3S zCCot=h3BEux|i$3?EL{jU+0vF@6lYF<1ct@nvl{?GETz9u$TqSl(?BLw-I-1nv7~TKDG}I{lY1+Q+$LLp#u5%n zY$=w1a5>j?Yth)6AH%PnaA>`8AcsTZj>AHCzENFRG}z})s^T4+O5-1X&%~LY95_7G zV#q&`!_AvoR~vg4I>RaIY-BSj@d2Y`EAEK5SgwS;-)%% zx9dt)JnnZ@Ge~l8dyjvL;1rH(Zn~lPaxu}=`X?fZ2eFew`(Z$H?S9`f9j~zP_>-8b znC@a$F-~$7`2i|HmC3CrPabTF{q=i7n#l-kq-M<+?fm{z15etAUt@q=!rm;H(HORa ztzG|YpSIdtRndR`RtnIK%j#^X;m$>=!=P)})E(K)9Y*S`S5B97%dCXg#;S!N=Dz-S zL}j?0vQBsRXGa56lek^(&ae?Q+{%ON=GqPV`=CIvsl_Q`TZW$R5WfjnKHhFkTWWTs zr3!8w(blfL+`+*2cDxckSypp+^u)`gwVL|45QyIFfmWz}yZm7`2{9s?$7H zKwKrRJfOW2QXBG=5m$^*2!E{_UJ0!0P2#m9L$V>RYxt0U2h?f;i@fRo`g-oT1btUl*1N!Wn|V#0Se8fg47zDJdydx!ET^KVy?^2Yz zYJ{Yis5NO{)OU=R)s2}7@-F6&1T`ZkkEi;J^a`I!K({K1e+qlcY6^x9(K-({^?;^- z?rcij*67$2h@ZirX3yb7^zwwoM3b-;17pLGja=;lgk0$ks#lVJcca2PS-{lx@b)%( ztu)oK`@z#W9M+@y`HP}35&HW2WxaFyp-}Z|@M^?s>pv5#v2cxKGPB`#2Ll6=j_dBP zXeTcFh)dx06N3P0ppABNPFmZUqU)@$ZkbKG_nVCdm%5oMflt!+?jz`3nu#(m9GazC z5*3x){~KPpH3!ia(foIO<#_FNqKh5P1%~|T`RVuVpPuWAfZN(k$;4k^0? zFnlbOV4qGx=-X<-<9VAqJ~=M+^1S0JILyzdfqwq5XX{a04lQ@q^G1iyN_gaFsEKO3 zc3|EU;|A(qDY}BW($3B{7f3BuAloSUe2^#PB2al4&^m4!I4-r|MeT$Aa8f7Ce}=pg z2KwXs4EjCPP3mgGc)Ef^T%sK>1+95MB1hI#}I z*;dUBs0a@fH#h~f;*YxqHg^34v8}%~h%WX%RilAxW43{KskEnbnRZMKLEyqjzJv>A z*7m;H0($B8ic~kV_SC<1l*8dSOY_!9YU>jAxnBe|4$3{8Jr_)xu zZ~poIF5=vZ56>h4wR;@z+$V29eC?c$O}Q(DS>txnL2=4ssvU@_9riJe)1S_JX6>r1 zgpAYEafaEd_iaBvE=?CZAum#Q7!Vi)rEdBeg?wi6*6Kl@rqsKA?cCRS>I9F;4qp88 zkeuOaV#~*w@aotoIEpp*1r25 zM}2E>KWG4d=^$|37cpZ9{~_;7glrCuiQ0X~^YFL2nff^vi#hn7YX~5$V&&C3x!?jO z@Het^IgkTvmTdsYIQ&XVVJYo89W7b8Yby@gp`HU})2{euTlBvku=D1KA`NtPrN>bN zu2wR94V#sjiIwcxrx+ZHx{bLUezu!(+*c>Kh7Y|5B;S8C)Kngd&GYFQYDcmJGj;%-xhHnk_#Q&41lHrJ-nbipgGf>e$zcGmdA&GxuR{{{XyPS$!&C ztUe^*jbMB@z4mq=CiE@Vg?fjULhI}+ zDXR`5&8(u2=_1(l!sEu~O6=J;apAJfiban((JZ4gdNwdND_IW6H<$&{QC9@Jttpyv zsS1wQfj9*%M&ZlSpuK*MunR3`K{Zof-h$M#r}k$znXw}!LVmLD;LOW?F)|7{3TV}0 zi8mc8*=nGXH8P5*Kfu0kmT0e?Lx;NrBxz}qwt{?2;K;mfBS%<|kTvIYx;5>NCj-=U zl$~4JZY-DU^##~1ZZd|uY#XvIWoQ3SLhh8_*P<->88?g+{q#%`zvArqR{CeZgg;+H z^sb6aYNM>~9>*t*H4POmG^_WrB5l~=XU9m?891{B5X_Lws_3W%p*$apd0=K0KP~@m zkGJ_GGXvK)^x%xFLDg!r*a`Ujng34q{q^~$v5Xo;pxqXuZTTVqAVJ$4Bzkfm!n&v$ z)8@&+tnS&HK5){sX(-eG*ppO+uKmgsUq$*yLuco4!;FSaOXRmyd7YSUL4908TMJG&?sy`Y z3-`^?e&I)_mXTDipkuLE3+?72VZ|$u$xO461-sPW9$a?$_LkMklB|`KQ&BF8PjV1F`?N0Pa#_kWV_2$r!RYmHl^>B|5Bn>>=l@@Fv+Nm?6eua_Ei>^M zC-2K@@g_)fMC%_@m2%#L+VXy?U{Wk|Wh+UPjtKV=|egDD6b7;>>0seq(&ctL7Ew;rqWWr zR^Aw@pVZiKWu zt_PJ=Fz7d6sYBZjj!)@tBOqFPXf z?md5crivU-NIrt$rRe*@=kyH%g3#D)bM?CV0-MZa$dUbvy#SEYC<&S^AhJ4WHLbj{Vv3t29GR z0-bb%1%Ks)@Oa7K|CU07RYC=Hn@L;JQzllq`RT59#esN5waa!Qu z{i5pOP|eIV(kpYi4A4w13&tm9YW>8Givn^tQ1yA>y?-3k5mi;i9iq}hO_t}hRvO`Q z`~PfVv@G15^Wu(Bd{=H4;K`s*9-uyNov3FSU5Omp)RIq9gkiv=Hxhwuqj7wCS>^HOcwf-&7Go1}Sb4}>JL;L2u>CT$W#e3R z8p6(*ifAr5%_&fKmCTXSWHV}fw%)&K#xe4;dbzLC@KgYKkUwtt$~}cE!##rZy3l!^ zkMuRaZDK-Pc;PDc7$&rTx#-tUqiKx9f7#x!g3|x&pyd-SCGR|7&p;RCY1FOYx{+(V zqiek#)&Oakp)68T9b$v1`L2R|vHgrV-sgsY4X%oun=-s2R!P!wr|%a7*%LAN;Mdbn zMx!yCnP869%8(uBTr z>0KeYrphXuV!bphww@_IUwR}JWf8_qljbZ~?o{jNCQh)EM_R45t~8>2|}YcG^H)gEVFDi`+M z<|d&IF~;|`H-0B+Rx@wkAJv)t z=uqwX$mk8t;tQgctn><~Y^R%!;mO)r=M32`tO-ayV@SbN*Il+bwd>B3CC*0r zz-8h)|5++9 zyG&5lV$|L1{Gr8(c5__0DsTbV4a(XQ?44+=;INd zP8_Js(9E4&F$Nkav7e5F_1w)W7Ut5VA4h;m%|e%N1sF?YuyszP(#bM8F@YIOBml6} zKb6A{HV*=@ny|T5ZedMnZhv79a++oCzeErbIvZZq(Z*=q$bO?ju1Wv_*MBIknNG9n zKYk}6mI`0@N?(8Gh!k@}!HNxBU*03H2-;R8Da6r=Gol#VqLmWZ`J2(J$J$%=f7!vI^Z1f z!C|-RCA_tnEXkiT`28UO;1xR7Tl8Q|w=FuId^op?c4wQJBc#_r8Q+hJ2iU)A3HsNB z9D~i0q3OFyLwsqSwdE^hFXnh;rkntwBX=l<#%p13zUcQ0)9efBBabM9-xM8w!{y_c z?7Ay8s(4eZS$9R~rnQAYl8?mCi(4g}?~LfqLwK*L451!$omU<{gHN^Zo(GI-N@=KO6{x`WH*Hvvcuq=I@+8`FGg4o!%N9 z&BXp=8dXO-H@#PF+A?^!I+P~p@EiUXqi42^TtL^J%6k*Sq-f9v{q-Z-1@3m4F452T$aavg;?wyj&e-h6D(WN&J!4Wj_{W`>h; zDe}ffTWufEc9=R9yzwLtGiE+HY_L5iosj+vD;P7m$o*?mVIQn=zs1N&CKw`BW?_sz zMTIH^Cr@P6pjM69bTQkyNoxvFzQ?D#T{b}57+>{RX;JZ!&OdmQ!1ubJ_6Sv;1AzrZ@ylN962j*ZS%f-uGC8c&zCTN`|+Y^~E(YE!#jR8CrS0aqZQnIRR zGwV`N$23yng0vny@t_8xM`C9K4j7kMW<=l4htufB_Xr9g2Z~~OKJGv#0PuqXudjZwEJQwud%b{r)R6&-y&`YsHZ1+ zMTGSg0gi7o>ur+}LKY(*cYQ@y-5F7Rv`niBYYr(aQMfP>VEOvunz=fAJfzwdg(H4@ z`|1g3E@K~2Gk(aFPub+(5qF2-SC?0g^nmJFnbgk-QuX9ZmpfLU}6;Z%iDRLc$3H42_Hm|4hBhACDTmXQL;EUaN z;O`FC^I~4wPzcolTeL_Ed69{Pl`gqti@t7hG(YOcuB03bChTsNpVNAP9yK!stVQ`6W2Y8+QlMuK*V>jM4wO*CC>hm7YK&Q^`PT!1 z8C?3%=2J$3k`s4FjG7~vakKXnm*R=;@s`_G5DzuyfjFFu-uvMHiavvjB?cXOeXS* zvKOoo$Ts?LnaHGQk{>h`QdGpz(kMWSl6xJGS@*$sOQ+j*-Zu+Y(R@-T)W~g=?`2Ri zMRteBJy&s-t_@U`VedgBR>;tJtH4UctJ2%hIa2yR?x)2;Z=q{xS=4k!&^EyA zBc91T3ZY_#$>JVk@T-c;`JNb5fzDl+aN3wCcogaKwRO8%n@*N%QaU#njN7gepZ#H| zOIX1?1k>Sr+9-l%L-wmpH#O0FNirP}4FnUb*dGYGAVn>VlDm zCQ6kT{no}CY;gRhJ5#Mc>hrOC@3bMz7$zWOU^)YXx}?+du2smYhlg`m(6 z2)$!ZR6!;Vxf06l@F`qLRhygeh?hNOZRxQ)Qo*5cc8CW0JE)=9Tt6bgy^C2@HK63!gL+tHnfEoJ}4s4XZ<_aF;W^^eA;s3_vb#I<|%j74Nrvcsa^NOdw^$K79k;+&E=8zJ#mIi<&xJeZa@D`x z$r<`yHiG;!PiiDMxT}kmkO+v=8ZD(3d6gT~EX=J50Rap99~#oQ`3W1}zuG1vGV%EnfFGN>wH8?u$kb`KI`+MWtvidZzbi&WW z#RBfvyDW5d_1d2v-yF+!_!p$d52kwTY%5K6hv{Rp$a+epHI~$tV?2c0!HIx!7jm<; zR;YNq)WMcyk*QOSVcRsf2ZYUdx{TVI4oblChP@sh%3&`XqvdomOaSQo(Pg!7uE%7; z4acq%E!f!u2VTQVw+K@u&bH&aGPh@^;!kYJ2X~ey4{n z9eM{`0fVqeZUD&T&VzZHq(qa%p8%C%Nxn*#7(~32E^=n@EA)47B<5JQo206jgmA{M zq8}W|rC;;PFCKq-*5bEcF^|WTU6OZBM?YN|U{bU^k^VtPPAeQB-%<-`8o6#GV<1%> zxtNtyWw7BRI2-JyyL*4%GApo7DRo#vWHjPIbEk_ibdw&dc1_ih@gYnawN&{PR)MDAGONgklw>pM20(qpmd!ejUes1-~*+Os4_;O zoCE1Rkm;oR-88WU_XvN8-XGFWs{z1tXe*BGy@0npVf2k2+r$jWV9qTN>%mlR@m7-0 zk}Iu|$oZ0E7N(fnXCgt;Qwq9&iMgk_JMfS%`yP~x@=GNA*25Bh^@F$Z;#L!lDz>yf zvkiU-ER9y?@6M$LmkEMt|JpWA>`vPO7G!q$OmGO;05X10Y5*QU416L`#fUNWn(`GvC9SM{@fCPE!F?08z7 zDyg?uZ+iL=!F<4{PvT92+~!ILx2D!Fi{(-|>B}NrqW}Ez$gHvigUUM9F}gpnpR&2I z9CN&!qjn?2)EHr?NQjA;Mk2ZUW1W!ve!BMR&$U3DQz)v@3xUzbU7p2)<0YNiUj_63 zS|+k+qA=TZ02{*OkmB9tL|;^5vbgezxcn8gDT3>5p6D_J%58si8BEVEP^e7TnO#$m zipwmT11(y8d<4E{&A;DUAKOB!4|c zAh;`i?aPh4Uj+u8=EiUBBGD-O`^Bn9v6-apGg5MdCDX~wuJKtYINF04Z$jffs++{C z=F(tU7Zt?>9|Or~#H%FSP~HHuSMA;XgjesMhFXmR%J%5@%{21z^pVLLzl!4=kNJ=H zm1rKHa8N2RJ2SVxnzPl`(Mq=;{TZAXCgCsnCh<Clde)<$oa;^mb7qPAL* zei2Ocko785M>}YptxcU@)xa| zE+fgd2+BOC8?Hc*Ua4G8Hq;bl+`QGg{MTRBdz7FlXyk#1K7#5~lF2!EguS~^AAc3? z1Z~uo!buFQ0e+s|`ZDo@B_#dfP!%_eGhAaC{2!7yl;%@Q@Rt?($$>0j=<%$_4!?-} zavdSrw4SQINOr81A1EN&3y!0x*RuJ~rAx)mpT^A-_nN@;MClqvTy{gV z{eug=PiKs;b$l*CM4qATUVtWlQ$fadC&=qK4xH`a`lpzO3YiDnDREsKJ8D6DFOI|h zMSqRKfn1ODvSOHr3^2#ObY-@T~f9hdyC+q#5T1!Eho8gYB_4JQg*s;ZV zGh7rIwzgGYIZ=zfU05XJZQ>$?AgN${`eibDa4zlce)(@9C>3?tIqx*E*bsJ^@I zo!Z$9VX|HosbK68RHOS3lJTztU$M{^aBCcfNYSTaVH!1>wrn49(r0r%cd$5gC1aYB z^^}S4E^7vh>>lDOZcOGUge(G^R*}qZIwjQ|Ln#8$$PthvciaI>U_*(XiJ!e+g@6&$S?0=E z7V-J<OSI0}GKZ#wXr{BHIV_C%%WL!D#Xcw#j3##W z$@(;-B-b=eS^*@}QK<;2PjoL`Kis*&p$fLu@P_x4OjkrgKOhT_d0)H>Zp$im2h zC_2&#AFMUv#5>i@nexK#SU8&@do%Xvw5b!(eIpAxDLs`-+fpGdExd-f+YKRZJ6gQG zw*WL)J$4z{@zxq4jk5w!PD?RO$yaxX{mZd~O&pnZy5K;1sXFVQB08#jo2dUiH{T4E zH*J{Of2b@wQ&q{8w&o4c65(oV{d?oBD-*1}EC~PbCj~naE!rQ(pw(B@f;VIkAgW9t zd0Oq}vBh|`yfOPE(s(+90Hh55$Y1&e%A5Hx-EAvW|6W=9&UGP&RV)cIWMgR5)0iZj z%4}g{=FY(I@QYFpFZsVgFR09J#z{&1HFqW{Vo2NU&V8OQ8;f8>Zv`kZ3_nTvhNziwP>Uax9pTZCrzmka|LJbilFXg4lnvnJ z_78#t_RW=0h`D2BMM`d%PQ_gtbaAYJmuo^{YjIL7meIE{v#*;?+Krm zthVIXSO7SK|6vn~0TVQt)jyurJTXhb_J68!S@+&*CzKenG`vCB@m}QDn4p{V!C2(< z1vHFo8|#205hk4Z(rrgNkn{~QbK=50dR+TQAqQrbzWId?J5G;rOf)-4Dg;Bkmw@=;#MjkRCp+6O+wd7dq@VcxF8ps@E4 z(`NjBIk9t;@%J<6`u73#S9N^-uKmYJXkkX;(f3MGEc|*8_cSNXP1!>-y4M{(guE2N z4c%0j?QFd9P#Nbj^%)jajFR(`L+@_ikEWgo4kwdZO3JktF;O+krj2nVaFckq{Gufj z(o}Bnc8)vYFRB9FFt+PJeoGyQLUn&~U()Bm7|EWs+rS-<4f1O&gg*#TG7X+ysG$pZ z#>gWOJ?2j;v_)bG5QMM&I^7znz|C*+oRxevXr&%6bobudneMZH?2>y+iuYw9fk?^~!^$iu<6d0+ z!2PDT`~YGZy+2YbXS=kPmSImwp?<_nS<*8-ZHR(< z5@&}9l+y4uCqO|1qxz*X>*OIzFo(aVpMK8vbHgRu%7RGTw|nM`>!`-u*THP1Kq(7s z>W$kITu-_-(W~)b)B?fjxy5n>9jRO6`6}wx264O4naTg85gLGMG6vJO=E+hw<`4AR z%5MRjALpiOC~4*p2&i$71Vo71fdlm&K2X}EemqoRN>F5a??vop@EebfPOIlI zTA%g{vv%LU`67%Wm8Xxi+E3JLL7QqcB%N4O9FZNJG>#|jQ1J?$doRq<}rNt5V!-`Yrxth!@W`x|5j4&3$=N0kvLfP4gm$Ce8hju4L6qLaqZlib^pDz8y z{x7faolp_D?|=#dJ{UlJ`e+N*PBBEjtTHK|d;11_=W9w&Jim!r0wJoHsAvKjDCRK( zQM4m!7%JkIb;?Xlwr3C~NInynN{>5k(sA&)p3g)jxBLRj|6_n}f&&59xAX~Y)JLmO z_#N`Ts@E3Q(vToyEh(1cj%68r5UWui*I$hgnC<96jJ7ZT)@As|R5+8&NjJEy5ItB}#DT+sNiOMK@%}y;yTXbV<-(d)TN9MZRgL;w24NJ(Wmd>30`p&qFD{2; zEU|rj$V!K^;H3KK!%!eV59UOf#d(5fO3ED!HWVX<{3J5&tF-BkDc4OxdW!OPbRKT8E{&xVwZta3V5?JvIO~T57&D1M4;r zsQ>Vm<`9(rs}Lfc^zUqk)p8AvjGQclE%ol_c_iG;y`UOQNA@@%Vf67uk;)2Z$?;sU z1>C#V*(Jau^5%YiMXb5wbwA{-6*@sv@nxf~J3$F{$7=+@4e@?8BHj-_B_==4t}Z5o zmG-p~b>VRj+(Lb#7}F@^@N#5td1+8oD^te5>NUOf?!n|bWCf#3=j$Hw=eIkQf)M`6 zWuyd530oj>!{v6>RkF^I+Hwu0yFV|B)^r|%Q&kpEuO7l^VoCUI=ket!MSyB1Z4=y;oR|xCXM~BU?JQnD( z+mqoHUx8Yr7O&8jojDFnG7iPb=xJ5ZxJ;IQc}flSWN?N6XQzPGenW_I}16L~>`fAd~<8548hKd68335BCJHZ=r#7u#+4%|)zW|;;_U&2yZ^s{O z>G}*&0NBXf=ncD?8jM?JFi0VTb=p1Il%!^+9TIhpb|IDiTTE?t|HKH~`%ohfa!b6` zxJ{8Vi}Nn9r*@khfc6kXjlLUk0dqVh#`}ZB{T6TnmOkejQu;CxwaqVCeT;uB6HUkgVn|6%oC=;Tu&Y5e!;!!M%dz{@s}^j7 zp|`hJ3jbcPtk&fWx-(B9@C~%VXG)O4gUa;J31pfvWwj~qUaR81Qp_A z?9!hs5~>_-a(_yc!Sg#}JLzBXbB`mZI4Y?<4g6bsgR(>0-3#BQ?fwJm6Kx#pt6_cP z?O{kp3(z)dn3XiVN=glN_W#x4P!7I)$Xs|~E|}2Wi`Cldz<9$C_|vK{`Scri$c*iGYtoFa@n`Ut*PVL&Q2DnK0m4WrGn;X zV4xfKA-=?#(v${}BzL}FJOnePMY~mgHGE{R*a76MjO@ZpX>+-|+3l;};^WDgI8C4` zYiqqS_}Rh3AFEH*Zh6&>a^}Dj*g?hG{^dy8=prN2m)K&jT6Ycb{I$pX|4drlDGK5j zo~-;0SWIcqQ$SvS4OwseQYd33)GWo|6u?`c$iuXxgVpFmd920;XoTob`?A!Jr2sWx z$JDBU{4p@$P8JqN5&uezd3VTQed%}wQ%e(N*zsG^A>i4sSld~NgUM<$BRs5<$9cN~ zdIj;+MsK8?-cwcD1}`StQd+6d<0Q4v?MRQG1XtmlU+|g2J_B>N64poTJZK)#_kzUJ zfaQ2d(0BXR&L91v)*}*2Y+wE#-rh1O&UIVYhLAuKAP`7`ySuxE;O_2Da0}2_6Wpx{ zF2UVhJHegc4gnf>ryKh^bMAH4JZG<6``=f^531-YDBfp0?-=7A_ce+GEk_HMEN_m( zUGqo<506)vkU{S2qVbu|Ss5d(D<@oj$s6929@x)wKDTN4;P_jP?qB~l{y&hc|7L?Z z|CZx5SB_U-ya{w!Y;YL7MUa|wy*iX1(hRX3`y(Ksx_~yM2VbD^S7x&R#@vmTi;N`X z&&2qTV%)IPe-z{RWR|?2fv97xrhlP9wiQ|+RP#5Z>^ z2=FE2j4N%!gI#0vtn=+e<6?@MkuSxOY6Alqs{IKUU#QKk=6>r23>OQuQlYy56UV>l-zaq9DKbfm`{v#zPl@Rt}g<{uGO4VL2K-C_uO z6N#n%2j%T;H=dw1>8b-0A27Ebop~w*1*1pAnpE+HbNL;!XsKDNpJ)Vk`)bVQ9ck55 z(E551+G!35`bDvDz`lL>Wo}{U)b%-;>Xw?=&7dReOj}BGKIft2h%9$yODJU{F5^(ICYi zc+Mzu(;>2m)B|F9y-gqN%WwGDK~HAoQlsmCxm&*2Q3Rs|F3zS|j;5J38-7RADAS4j zF#iU9`xu=F3JxPff{)HDGorxqY|5y-cRCZ&7aAu1%UV*1Q?4-lp9Q|H1a@_be#a0v%pc*>Qf;PNn=y4voHuz(EE#(jwwNhrSDKi+UM|AF**|)T_};^hAt*3Xq)o zrlEIJOKeOvQV%~x+yD8S_A&Q9dUes~YROdDpI@mBo;UgY2nZgA(k*3~fC_&Zyr6WT z@|vOFxEOUhJQ}<@nqa&m!&{bOr^l3*`JJ(fmEBp;k)e<5x#{`9cxKT$0^8X03>`lA zD3$h-r4ETPu-2XGNBuHO7WaK@8uJpDXBWmTU4ICs8Nu`U zD0sh0k>4*|rzcm*bfV4M*aQZsJtkJfWmjbI^07Tj#TsNSbL)Jv7Dg3@#k++%6(-i_ zRHAuv;`S3bGaB*2h1z@hBn48lX!~LxLJ1o4XQ_wDu|Ib;^!O_DA89UhPA6{ibzfj} zQB~(R+1)V+z6Yg~g6CR>vk?0WFBmCp~|MyoHi5#UQ zL0n-2)&C$Ca(eR|(*v_H^l7#2L5&3c*7oNBNH8XRx^5gu3q$?L7{cqc0qC+~j1 ze0%lnv2pJxLcx#yj3(YI*6WXU*ONp^gzJ{?;}NFbfW6{1Z4M3Itz{n^r|&_)b+8-) zOHlLsN(_Vh_8fr}21l|d2=90QNR8r-2$R_J0D929nG|1^0bH59{7By)0VO8xB+)fw~FOk1_oCodU6m;L3z+s)9FY8p*jFu1i0|2 zRCCW1I;G6#J%NtkZO5dsTV*9ZOcU#h$Nx)LHXDw9Iw`m@k})Z%>_l`r$1F1x#DGXLnS%dQ8i zx81|yWbT_n2Zn|yuZqm_%|awvpyE$Gu8IFRW`EjoDR96j(f-QU>MWO#1H}B{;ebX< zYu0|NdxG}!*2}bciQvxN82tmxrAF!;Tahq24#TaA7S|L$04B42esN|&%O3ex^G4tL zuxO2FhY#QoDv}Bs4IM)|qSgeyMsz0(YqF~kwbu5?6+)@BRd)eCiQ)(dPnR;{qUsD) zBC;47Ic}h}pwbtr2|2If7z#R_d(yuYMz*(yG9P*5CS&`_(JK%A8_|tYH4s3+j(>Tb>W>wZ;f5AOHzgQ*#50N=dp?W%gNb8 zUS)v^k7YN*R3WJdj6_fZg0*r9>0PNz>{b~xk;JZpiW8-qs5fTcG`tog`(IL{+P|g7 z9g&Uio?;qsK@sr}YhvPW)`YlC0SDi?9)5GIg(T#8h!7hH53v!F%}0G@XYUvj@~pN_ ze)cW~vHC~hBysE)-k5{Z4vChdWY|im&+QeN#Ao}S8o#qpI{}%79o9X<6eV}r1*-22 z#inCgQ(JSPHj5$)NPw8XR(<@{oeH9NJGgjh;4AhwId=?>u_Oo`I{y$Yx2YI)z3;Dg zhr+6oQ`;?x>g9|0AK(%tJaB&_Jx9_ETrvD9a8h%a-jQD>h{xrQiBI~JRGVD{AGH#H ztJ*y#%mp7AIi&N{YK_$QTta|Kg(uDV4sK{1@@Fe(`Ym+IhqNiLBzB}}(qA?a*Yoj3 zbu{?w;pEFpzx&Seth@J!3cQlgE8+x^O`hl${h8J}W=|gl>h1YkHE;RSb@7 z^FWV5`ftLgX$B+qZ{`rw>T34AaH|rgop1I7k)yr|bA8-iknOtSlwUjMwIzYN3Ewr)2 zb}++d>vN5*m89SG3yY6f@YK>UTSmG-s2o3LT5mY(*H^_jze!56_`$PjL3PE|Tf9(% z%UwkbD<&q}Z@63o9Ur%R;T+FAQQgOu+Tpnnuk`^g)dZVGy^$aA9WfI+P^=A?&_v2! z)_jD&@D0T-5P2K|#!GKTn6{&ZC%+fEu&X1u-xEmvSifw{nQBOWb|Br)>Eu z!4v`YFv>nvlc7RPGhy0xSwP#c^S@vc;coA|)>l=64L!1^25I;*GbB3~pP^Q?XOvSa zZl!guVSyd4<9xF=xrDQ9oCvYuGr4P|rRK+D`22vRX>!-)j_9LU6hvR48f)qzs;Up) z`w4M2Pj|oBgZ~_b%_dO(yVMLM4dUD8G&5->$%XKGS#-ac;_d$@M}QCJca=9>+gq$Z zSQktf&q_ufOLpJ9cy%)KwLRTKx>@;d+-9zoY7>c2W_+fOLk2&tMzOs8^u|3`O-tl2 z3c$hFymfMc*@xuYa#Pw)`O>i%;;(^tAKv*tegE2wlWn;PJA%A>){1K0a7ISy!}}8v zzGE(QVpuTyLBv;1R*}t|kfwoBIPQDa2mR=$5QTmRggQF+vZbubPnutPI87?xcLK6)Go5H3N{t>j%go5qX^o8=H< z(O&h67-I%{>%?6h@c3lYS_s+o>ZU!rzARvbZVG#=YU-R5J2@*ME3Ms_6lIDem z!r>QTq{-VE3yl#KMOVDSu-7cY#4+A_-z(DZmqM@(4M#fHn3F0Y4F@TAw;RPu8+S}a zEgJao205uqr0oqlgtB8Q$cRpan2NFK&USo;Q_K$0IfL z{1&0eUD1+gAbr;14M#Wo{G$fexJE7KLs?T^Zc(-b+Z;nem3H26jz{f4;!9(Yh!$)- ze0(2XzCj~3_NS3Zo>W=l-JHlHDhUPgB^MTTIAX@45r;h?01qoOXx92N2rOc-n+u&O znx*tqqx-QDc%&6tf(?FPN}>~o0pmoDZ^mlR1k4E1&?_Hr@6dbT|GKXRSNZz0#yh(q zT5XIm8Sytzd7{xa*l!Y25-G$t`b)_JTn@Pm`~r8@5?J|tsa^(S`E>b zKOL=ydK5W$df)WS^`jWAbv}PXD1E2roG~(xJX-~9uJ`--)lIFS`mF3knjxP!)=bNX zu#6a9{u&_>TT zjmtJ<-RmpdkZsANYb?N-(#GQkc~YHeM~7eZ7{bF}ABz`l23K2Ry*(IUN;Wh98jt11 zA&LGUs^{LzjjC;UJG~#ibYT?7!I%Fb!b~_0!DapazPyZX8VbrR* zyzJjfkuL@YjV;G)=XC6C9iYAOI+1;uR9*Ah+QA~X?6gTeXU{lt-^;?8quw3=QU}15 zN?+md7eirl$M&C`1|qJ1aT>mj+7@-O>{HpfGnwdhpYS&nO`4jZL$!v0O=)GO1gmh# zPYCa(=~ca)XH}OyhRZWRMg-GIMVaxj&cC<_3!usUXzjO4}Cs|(tEVd~$bry14ohz0JdkX|{ zxerv-MVL(GBrfD^>}yD4eGFcp{?azhp^r18=hVc%Yvhhsh(fRHwk}F+H|NxvuPyg0O6v@6sIXW|9m&+4xU(&#?swirvrKceS}0u1z9>E>%%3!!xUOtB zD4qBFR(edRIl7}&W-|VECOePVC-Z3WwAaniov5wLJ6-;ISQi&!A=d>JoOGe*6%LXV zJD3TETcuPy&uZaEa(h3?v-9Rd(^OsaX%D(3{3$Wz4g=Ht(nZ0**@Yo2WUjsq9ydA!HH(b7Tzj`i!Dsl@_&=;AF*ycA=D0n53|TI9cuDY=SRPiH8>H(ysg?Au z06DvAbD*ySw-p@}I?S1~KX^)vd~H{;;Shf({Y*C#B8{|sT8V>fKsES%vDlPg_A|8d z(wjXZd`fC^C9bD3p+$uu?p$asyN-;uepx)sIuxyIYk8JB^WFP>nQwQWO+Kr>`>4fD z_u6=%f*!ssD#Sirt@^-`kXuXCqvAY2ZHrd!toW*k6Ov-L65XRII0(2uBSgXAZiTnBHlWwz_I_WgBIN2ue*u z;HkqWCW-3HC8om#9hj~_zm(1^!lJs3qS`A{OT0kCprBB8PQk=e|1pcTo6w8(;F~<1 zyOU%*Q{6+No?oZRAt4yl)jvaENqiyaT?y9i_w9kr59bFvC-e6Uc70mh_HO8cd^c*d zm1%PM?7xb?tv$&a-GE1JwjZz83ZZTA1sa$z`Lw|$SIdj*OOv5r*#n_8R-@ZTe02ba zeD>>pd@cjsSP{!Rw9OViAmql+sj-e!9q@29#XrlOstDZD3u?W{9-}GC)j88GW`~;+ z(c$``GSKCw2DWw}qqY44O~&q?HjP3f`eWib((q%)H<-8KzKK)xFb=cRI%Hh@sVLyc zFy!my34B4I2Q&xhCXk$bbVdCVSjce@{wS!KwRVB%%RVQ!RIX(A3Q@>mSa+}ohcrbHDX>h+s!Mscv3g=N=vRN z(j>^ES*Y{n5okQI^!7pb+P#gn(iB^{bt9AqJ}EXgFPNC=`q_#6`EmbscDQXA3QrVa z|HWX$f*{_nR~628%P*#N%Z0>J8z2T*ZtCqtl6}~?zhmUAXb!*S+|9&+r5c7#fCW`| zE0*s$sHtTg<>y?UnDP*TIB-3hauW)n0ioz!pO88LIaSxnlXc1%S>}WlOnb*7R#Ax& z0}Ddk0y^HEK=Y(A_$rJDO;l%7e@NwrihXKVh&cEr!s2sJC2*Z)m)xN@Ut`0)snBRN zZv0v3G37`NrYzs>>!@~9mno*x?11VmoL++V>k0az{6VD#{D?a`x$`ir>GLS&et4fy zfIaj96{C#45Kxs#@nP2yhgJU`luCss2gRjOD zP6}+ph}W1i)P~P~bbs_yCA)F*d%nq z_q<{7@-vFE`2KZp6b8QR@9#wbGHMmCxZ+F|%PdUhd<@7d1R*Ap*wc9*^-h_eP%G7tHYP_E<1L8H;^w~H z|IA=|ckIZrw~fF*wdZiTD=Qk+(`EA6EP?mJp92FrguGZ_^BXI&d;Bc7kVU=LIgBb6 z^~Lypg9Yl9;KxWOhVfRpOZ%BO`OyjWWj#5$|(=B zf>o6;Kqn1A6mP$%?W0%6F6hRyMTe>D6c`3;;is8;^K}<$vXbn~3&PYL5`MZ;BN<_} z9k{la-qSvSO&lN{dda;mZJqF4rCYiWH|R0SYESZW_I! zJ-R?ZP4=3rVNZd*|K;kVok41rlmNHi(9)*t%?pG(vMDHJsTO>c1(f)mGaKJ36asu0 z*RI=mXj*nOJ6tip?`iCQ z^0d#|mY=<+tTi3DVARkA@p-;snJU{4Ml?)RFq#Pxc<83^JZ?x84+&*|Yyh&Fe5BE5 z)J;J5M0;r=-7G@v-*JLKUL?it=RXK|_G2PcbRCq!wPTT$jwuW5WrYwUeM+4fH_+v2 zKo{#8o}=rBe*&94^+A_%Q_T0B(B{e>3GomnKu8cd2n#wXPyX$SoaXIuIdTMV-8JBo zr0{6Ib1c^R4U##6_8rH%#1zi}p)c;`w7jJdm%s3q*VeQ72^!9T9YgQb9qMJJ%-Vc= z`{mVhk3=x2-~VNncS(S~qx#l$sec|uV3!yk#8#`dGGc4#?v3&N{1`7nJ*$*mK)gfFqK3@L&>8%K` z9~N80(c@;M#axza>q523RkGJ`rDSwI7@2c^M)GsgF%R#EQDg4ft8n?svSqwtS70Qe zl7NQ=ZyXc}W-MugpJ@w84LXk_*oB9wHY8O4FKHV-i$GA&mx(Ln9(|OSC*7c`4wcDtN&7;Hm0a_J{4JZANrmoho5i8-x}?jbQ($VtMgP>A zP9#iIbI&F70pQ}tStlo%1Dcyy4c)7w*j;!N0jU(z^#*2n%Msg*tl`KoTef1}3;WZ{5%oAiT7 zfb@>px%)U_aU?Eb5ipRbWdw%70Luy*U#ofGWZT&gy&Mg5HqmvQ-4VCE6D>)VI zs{WI=);n;hu2%s_psqz0kP>tq5Ywd8s!m*hpWtDr;MVm z+WMm{$(4x=OGZW{^_{*Iz}l$v?ysmV7m_K}jb0pyAfy+)<@{VNcsgb@QF9AN+fI8RY}2gc0+05o~7hr_i3`| zQ2T-zbw`|C*atVBWz=%dN>3Z?=_BLfaDP#UcS6^oyhyy6%45aN67>T2BgD2&;Ssso zY=9Av%A#QE)5%C)kGV zJcF3q{gi10rkVP15NiK)hPH41HuQ2p{N8|L3-(D4o}$1N@nz2TX^9D@7MeTkAGJ6d zV}Y7K%9B%syoFBc3+vRyyf5tT=Z~@4YnsrQQ>o6rWFG!7Zn>JFBe;ob6}5JDlnl6w z{PVj(1Ft3<*Xs5yK;~i!jao03_?)mD$5KbT<_wt)v;a^6;dDh3Kz8bTMw$RZ6jsME zyVWRl^R&e;r0tZ0zG1OAV`j(j?BHhf6ODD&Q> z$=ZSBszHt~=UmV>3ooPObE|cfbFMx6y$dy#?SK{{U*hcE$vGd{N;bKA<$GXB;gSMV z$N1t6ULi&`12i^laW&h@5EQfhSQjlc5dCB{^l9!>q}&hB?FWHc7z$Sq_1^!rMGvmK z|J`xMs9oc=@u?hGj*OG`SEdFG^FUk_YdiYXSQT@DID3*mQnPbYu8!{*GcE*;D0dOK zm^+Xe_RH2sbo(%fB?j6;2M}4Cx?b$s6ciL!zg#&(NAqQjkWtG^IsGQcS)oG%8{lE+ z^gB}#mZ3dbzmWN*{45mQH%FaqO7F67tUvv6UB>kByuN-JTuxPjsq8`SvrpM8FkM`A zY&rW|B?!$gtqQay|2q33`=?!a$r#kl^(P0{*{U_dB4pr+GoIa$|vXWQBkp1Xc4Al{!biHH{TqLEDj%O@@S;kKA(#x^~iMq zi2|qx!#D9Ci67ySVB`m=^N@SxT=A4L8B`I^pWFdVZX85z(JGIAKxjNvjQ8vRaC`}n zT@95D$r|0*MV4?3zkVz?c+b=^TW)4j{f(3IEOyvakNB~9yv_`g6HzxWAhl1#@p**Z zZxm5?{w=%A<0$o2uNeBWg2}~nllJZL-N@rGe#{4Hu)We|V}*EF(==)kEugL*y-<>E zon?tvO} z_(}74IJY?Xy->ENq9Wxj>s_YRu47dp{E@@cfAc*UXjfNz!u1m+i+WJs?MI^QYz^&UJ>ZYBu^w#K^2z^{+^jk56E!hZlhi4#I zZ2-6DBG0cR#|GJ#80ZX$_qvxOcZ8RIW|4p?w>Mkl_uWC8OCEI>A0q@~8=Z}q$hrLf z6nr$V{Lk(mwq6R3Fw7@`ENiRf9~%-LFm^?N)JoU20lAFVmD&T#7(DTAMAI=yDa78boaIi&Mz$l1V70}P`%J06hMgUt$~2@v)X!sFZFMog`~EQ z_9qYUUeS49DT0^&Y4Msk7x< zO3<%X)oHhC3C~7hkll)ikyzztEqKq-<50kdo~g}5tRUifUy?~BO{C0?00X_3=Cfik67}`1S%4S9MF_8Fg_Bzy&3Gf6YiFtJ^Pk4(@hF6GVtM+QkbzKL!+}R%i>0Tj&JK2kA!gkj2(D&5;SXI`sH*zWEZUY&l@`4iYEK<7_9%owZeU?x%QWT!OoZLl+s&hBO2l@z| z3k5&DdhNEK6Ai<>2^85=ZplM-RZLaj@wrzzR68DD1Ox_w^eU$2at7-{2{CdWtko5Uh6`TW3jmm~n8gH0oPA zJ4e#Uy!+OcghkByULUBE*t@EP^ER(o_l!${sj}q2$yf8tDyAql{XVtn$2uM~?G(>; zx+#C7agz7PE+n@8jp2_N$2^@HbM87OjI$~nITB5X*V`t<>0`kbS0Y;x@8n=t>Nx0@ zo*IZEZ zmg<}~$0p9dcwKGPdjIrw?`Wo=n|yfZnW%O7zh}^cLa=?qrh+F$W{4l#iS}a>aYnaB z#_TN)G(W#!Xf()uS$&;77?KSyws&5jeR&?iT*K!Te2z{%%kFIhzX`@b4g+W@V>D|H zWDYBBj2Rs109SH664LQ94J)Ryr!@`NAz#V(oZOrsi@tRFZ{ZmXLeWnr$pw6pL=>Lh zGafxhPl*Mtf08SBqSp>NGSIWp_d&tLVgw78=bLz=Mnc3r(194BoG=CriOkHC&Yt`d z6?vcsPuxI+CtzM&N=fogDHj}CFELJNsCQh*v4@f(03>r2(I2)ntD@*vwuA@SDz|0k zuGxL%Po!NNGT@xv`ASu+*}PBX%j(>5#?Q>tLQk)Ib6&j5T6f(|y_r_s?! z*$b?%3;5??lk41`CENLd-K>WeRSxIa-~l5xDE87fWLQjdC&S!(@V*Fc254@qIZ9kM z8()k(rQPgCkBh$tnetu@-T#c_~Dya~e*>$NiNhMYuo(+4d$C&gF|o!`j{-Td#XYUgIc{ zAP38d1v=iYudrWuCh7i+h^E^;e#TVK>vo+Byk#Xf()~CxJ~5&hFvodDmz9REUwFUS3}u=DRcSdex0`~VLc zlah)fh6G8sOdTv|YH88NehhM}6GNXA)o}USsQ?O0Fb9;x1L3^NOi7RJBphV)$ zr7toV(p$wQY)$(W&Y0d4mToMHr?^@C)p$ee!Er9LW;i)@%j{m?8-ueg(*J8r7qHvX z6o!T`Yy6A5rAXR-kGEBSUx$F*{2keNd+#!p9KcEjEj@=A7`r%le;UV0i0giND~D+? z3dvazF8^)0wDQc|VQqBmijC0+%EgTvO5b=QALMdd6(FvO3qFWb5ro1%t?JY!dYD2W z5B{WH{3Yv^F;Vckeqexn^OlUepAj-x@_y}-nZg&Tj+mhL7H6l%<-y^aIzNtVO}*|q zdt|Nb=s^v8N!wd~@r_L^so`m&1GyDtd6mSt9lY5oFTg`N+?{Q^$Jbf4y!nVFC8Bld zU;e3&3To-YAOL%g@jVvF{Jt~2ph{i%oe$n4OP$*_CGyq2c-a9B^vjNyFf`d&`?__2 zA(f+581hcNv-%awNQLLvNE$8g;M@g{EXH(7yD)S$Un?zu?d=lSTv#!@F;mbhU@>!| zYGz{{N9--nKEQisneszrgym5wMk-NOSCtWy*>$8!6!G~7fheKzL;f5>$WL_#p(@Y5 z<8yHTvd3Jll+GEM)#mSzhK{n@+i{8;@g&2OI+q8sG1`OaFb5WsJX?mZ zX|x(t6^2R@J41f3lZRqZonKk>*e#A$fsH=x2UYZ0KwL+VNh7O_fN62zRczk9)b59h z5c$G;5p-LUEI`W!@hRBvR!|Hc^5qzFwSYUa1m)4F(>rtM_h7B zhdJE|qt4GoPYVT6YyCfdn5PhT>T)Y@iyFAsrsGa7g+VTBzvNFZ7VaKGHhohx&_O6` zLHLvt(qUvCt3+B~hktBndbb8?@ha@>Dfcj&|J-h(MPWEUY(V>E$IoPMTbv!y8^t~| zq>8EGD@WgQ#%zQN)mC>k8#C{L{-}d&^6l5Woh_!F`DCzE+GRcNDSMu`G%B;Kf;ke> zX*DPtaa`W~TI*b)I9?N1pksQVh7qvJ%58uq2Q?5|prkA4VrrFW}iK*S(p=r|w|gcrG78FBz$M3`A category's saved progress: where a restart carries on from, and what the status and verify commands report. +public sealed record MigrationCheckpoint( + string CategoryId, + bool Selected, + MigrationCategoryState State, + string? Cursor, + long CopiedCount, + long SkippedCount, + long? SourceTotal, + IReadOnlyDictionary? SkipReasons, + DateTime? StartedAt, + DateTime? LastProgressAt, + DateTime? CompletedAt, + DateTime? AbandonedAt, + string? LastError, + long AlreadyPresentCount = 0); + +public interface IMigrationCheckpointStore +{ + Task> ReadAll(CancellationToken cancellationToken = default); + Task Read(string categoryId, CancellationToken cancellationToken = default); + Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs index 9ba0554964..bc5bd1c3bf 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -7,10 +7,25 @@ namespace ServiceControl.Persistence.DataMigration; public interface IMigrationSource : IAsyncDisposable { + /// Connects to the source read-only. Every other member throws until this has run. Task Open(CancellationToken cancellationToken = default); + /// What the source report and dry run print about the source. Task Describe(CancellationToken cancellationToken = default); - // Collection names as the source reports them, not migration categories - Task> CountCollections(MigrationSourceDatabase database, CancellationToken cancellationToken = default); + /// A count of everything the source holds, including data no category copies. + Task> Inventory(CancellationToken cancellationToken = default); + + /// How many rows the source holds for one category, for progress and verify. + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// Reads a category in batches, after the checkpoint cursor if provided, or from the start when it is null. Throws on a cursor it never issued. + IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default); + + /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs index f015402f2c..1f4de16640 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs @@ -1,6 +1,10 @@ namespace ServiceControl.Persistence.DataMigration; +using ServiceControl.Configuration; + +/// Implemented by a persister that can be read as the old database a migration copies from. public interface IMigrationSourceFactory { - IMigrationSource CreateSource(PersistenceSettings settings); + /// Builds the source from the instance's own settings, without connecting to it. + IMigrationSource CreateSource(SettingsRootNamespace settingsRoot); } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs new file mode 100644 index 0000000000..6ab337d815 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// Implemented by a persister that can be the new database a migration copies into. +public interface IMigrationTarget +{ + /// How many rows to read per batch for this category. The target picks it because its own database sets the limits. + int BatchSizeFor(MigrationCategory category); + + /// Saves the batch's rows and checkpointAfterBatch in one go, so progress never gets ahead of the data. Save the checkpoint exactly as given, without adding counts to it. + Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointAfterBatch, + CancellationToken cancellationToken = default); + + /// How many rows the target holds for one category, for progress and verify. Counts only that category, even where two categories share a table. + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); +} + +/// What the target did with one batch. Every skipped row must have a reason in SkipReasons. +public sealed record MigrationWriteResult(int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs new file mode 100644 index 0000000000..431730ba82 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; + +public enum MigrationCategoryKind { Required, Optional } + +/// One kind of data, such as endpoint settings, copied as a unit and resumed from its own cursor. +public sealed record MigrationCategory( + string Id, + MigrationCategoryKind Kind, + bool CarriesBodies, + int Order, + string? MustFollow = null); + +/// A message body read from the source, as bytes plus its content type. +public sealed record MigrationBody(ReadOnlyMemory Content, string ContentType); + +/// One item read from the source. Body is null unless the source attached it or the engine fetched it. +public sealed record MigrationRow( + string SourceId, + object Document, + IReadOnlyDictionary Metadata, + MigrationBody? Body = null); + +/// Rows read from the source together, plus the cursor to resume after them. +public sealed record MigrationBatch(IReadOnlyList Rows, string Cursor); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs new file mode 100644 index 0000000000..6f15bcdf24 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class MigrationCategoryIds +{ + public const string KnownEndpoints = nameof(KnownEndpoints); + public const string EndpointSettings = nameof(EndpointSettings); + public const string MessageRedirects = nameof(MessageRedirects); + public const string Subscriptions = nameof(Subscriptions); + public const string NotificationSettings = nameof(NotificationSettings); + public const string TrialEndDate = nameof(TrialEndDate); + public const string RetryOperations = nameof(RetryOperations); + public const string LicensingEndpoints = nameof(LicensingEndpoints); + public const string LicensingThroughput = nameof(LicensingThroughput); + public const string LicensingReportMasks = nameof(LicensingReportMasks); + public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); + public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + + public const string EventLog = nameof(EventLog); + public const string CustomChecks = nameof(CustomChecks); + public const string FailedErrorImports = nameof(FailedErrorImports); + public const string FailedMessageEdits = nameof(FailedMessageEdits); + public const string ArchivedAndResolvedFailedMessages = nameof(ArchivedAndResolvedFailedMessages); + public const string GroupComments = nameof(GroupComments); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs new file mode 100644 index 0000000000..69665e34e1 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.MessageFailures; + +public static class MigrationCategoryRegistry +{ + public static readonly IReadOnlyList UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + public static readonly IReadOnlyList ArchivedAndResolvedStatuses = [FailedMessageStatus.Archived, FailedMessageStatus.Resolved]; + + public static readonly IReadOnlyList All = + [ + // Required, copied with ServiceControl closed. + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 2, MustFollow: MigrationCategoryIds.KnownEndpoints), + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryKind.Required, CarriesBodies: false, Order: 3), + new(MigrationCategoryIds.Subscriptions, MigrationCategoryKind.Required, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.NotificationSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 5), + new(MigrationCategoryIds.TrialEndDate, MigrationCategoryKind.Required, CarriesBodies: false, Order: 6), + new(MigrationCategoryIds.RetryOperations, MigrationCategoryKind.Required, CarriesBodies: false, Order: 7), + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 8), + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryKind.Required, CarriesBodies: false, Order: 9, MustFollow: MigrationCategoryIds.LicensingEndpoints), + new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), + new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + + // Optional, copied in the background once the host is open. + new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 2), + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 3), + new(MigrationCategoryIds.FailedMessageEdits, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 5), + new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 6, MustFollow: MigrationCategoryIds.ArchivedAndResolvedFailedMessages), + ]; + + public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs new file mode 100644 index 0000000000..c9ce5d9fc7 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -0,0 +1,266 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +public sealed class MigrationEngine( + IMigrationSource source, + IMigrationTarget target, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + MigrationEngineOptions options, + ILogger logger) +{ + public const int MaxBodyReadAttempts = 3; + + public IReadOnlyList SelectCategories(MigrationCategoryKind kind) => + MigrationCategoryRegistry.All + .Where(c => c.Kind == kind) + .Where(c => kind == MigrationCategoryKind.Required || options.SelectedOptionalCategoryIds.Contains(c.Id)) + .OrderBy(c => c.Order) + .ToArray(); + + // Runs in the order given without re-sorting: required and optional orders both start at 1, so + // sorting a mixed list would put an optional category in front of a required one. + public async Task> RunCategories( + IReadOnlyList categories, + CancellationToken cancellationToken = default) + { + var results = new List(categories.Count); + + foreach (var category in categories) + { + results.Add(await RunCategoryAsync(category, cancellationToken)); + } + + return results; + } + + public async Task RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); + + // Halted is deliberately not one of them: a halt says "stopped, and here is why", and a restart after the cause is fixed has to be able to pick it up again. + if (checkpoint.State is MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned) + { + return checkpoint; + } + + if (category.MustFollow is { } mustFollowId) + { + var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); + if (predecessor is not { State: MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned }) + { + var blocked = checkpoint with + { + Selected = true, + LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessor?.State.ToString() ?? "not started"}" + }; + await checkpointStore.Upsert(blocked, cancellationToken); + logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", + category.Id, mustFollowId, predecessor?.State.ToString() ?? "not started"); + return blocked; + } + } + + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Halted) + { + checkpoint = checkpoint with + { + Selected = true, + State = MigrationCategoryState.InProgress, + StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + LastError = null + }; + await checkpointStore.Upsert(checkpoint, cancellationToken); + } + + var batchSize = target.BatchSizeFor(category); + var isFirstBatch = true; + // Per run, not the persisted totals: the skips that tripped a halt stay on the row, so + // counting them again would re-halt a restart whose cause has been fixed. + var skippedThisRun = 0L; + var processedThisRun = 0L; + + try + { + await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) + { + if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) + { + await Pause(options.ThrottlePause, cancellationToken); + } + isFirstBatch = false; + + var batchToWrite = batch; + // Stays off checkpoint until the write commits: the catch persists checkpoint, and a restart + // re-reads an uncommitted batch and would count these skips again. + var bodySkips = 0; + if (category.CarriesBodies) + { + var (withBodies, failedIds) = await FetchBodiesWithRetry(category, batch, cancellationToken); + batchToWrite = withBodies; + bodySkips = failedIds.Count; + + foreach (var id in failedIds) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); + } + } + + // Absolute totals counting every handed-over row as copied, persisted verbatim with the rows. + // The real split comes back in the result and lands on the next checkpoint. + var checkpointAfterBatch = checkpoint with + { + Cursor = batch.Cursor, + CopiedCount = checkpoint.CopiedCount + batchToWrite.Rows.Count, + SkippedCount = checkpoint.SkippedCount + bodySkips, + SkipReasons = AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [nameof(MigrationSkipReason.BodyUnreadable)] = bodySkips }) + }; + + var result = await target.Write(category, batchToWrite, checkpointAfterBatch, cancellationToken); + + checkpoint = checkpointAfterBatch with + { + CopiedCount = checkpoint.CopiedCount + result.Copied, + SkippedCount = checkpointAfterBatch.SkippedCount + result.Skipped, + AlreadyPresentCount = checkpoint.AlreadyPresentCount + result.AlreadyPresent, + SkipReasons = AddSkipReasons(checkpointAfterBatch.SkipReasons, result.SkipReasons) + }; + + var explainedSkips = result.SkipReasons?.Values.Sum() ?? 0; + if (explainedSkips != result.Skipped) + { + throw new InvalidOperationException($"The target reported {result.Skipped} skipped rows in category {category.Id} but gave reasons for {explainedSkips}. Every skipped row needs a reason, or --migration-verify cannot account for it."); + } + + foreach (var id in result.SkippedIds) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); + } + + processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; + skippedThisRun += bodySkips + result.Skipped; + + if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) + { + checkpoint = checkpoint with + { + State = MigrationCategoryState.Halted, + CompletedAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows. Fix the cause and restart to resume from the cursor, or abandon the category to accept the loss." + }; + await checkpointStore.Upsert(checkpoint, cancellationToken); + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, checkpoint.LastError); + return checkpoint; + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The host is stopping. The category stays InProgress and the next start resumes from + // the cursor the last committed batch left behind. + throw; + } + catch (Exception ex) + { + checkpoint = checkpoint with + { + State = MigrationCategoryState.Halted, + CompletedAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = $"{ex.GetType().Name} at cursor {checkpoint.Cursor ?? "the start"}: {ex.Message}" + }; + await checkpointStore.Upsert(checkpoint, cancellationToken); + logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); + return checkpoint; + } + + checkpoint = checkpoint with + { + State = checkpoint.SkippedCount > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete, + CompletedAt = timeProvider.GetUtcNow().UtcDateTime + }; + await checkpointStore.Upsert(checkpoint, cancellationToken); + return checkpoint; + } + + async Task<(MigrationBatch Batch, IReadOnlyList FailedIds)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) + { + var survivors = new List(batch.Rows.Count); + var failed = new List(); + + foreach (var row in batch.Rows) + { + if (row.Body is not null) + { + survivors.Add(row); + continue; + } + + MigrationBody? body = null; + var succeeded = false; + + for (var attempt = 1; attempt <= MaxBodyReadAttempts && !succeeded; attempt++) + { + try + { + body = await source.ReadBody(category, row.SourceId, cancellationToken); + succeeded = true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // A shutdown is not a transient body failure. Retrying it to the attempt limit and then + // recording the message as permanently unreadable would lose a row to a restart. + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Attempt {Attempt} to read the body for {SourceId} failed", attempt, row.SourceId); + if (attempt < MaxBodyReadAttempts) + { + await Pause(options.BodyRetryBackoff, cancellationToken); + } + } + } + + if (succeeded) + { + survivors.Add(row with { Body = body }); + } + else + { + failed.Add(row.SourceId); + } + } + + return (batch with { Rows = survivors }, failed); + } + + static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) + { + if (additions is not { Count: > 0 }) + { + return totals; + } + + Dictionary sum = totals is null ? [] : new(totals); + foreach (var (reason, count) in additions) + { + sum[reason] = sum.GetValueOrDefault(reason) + count; + } + + return sum; + } + + // A configured pause of zero means "do not throttle", and a timer that is never going to be + // waited on is worse than no timer: against a fake clock nobody advances, it never completes. + Task Pause(TimeSpan duration, CancellationToken cancellationToken) => + duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); + + static MigrationCheckpoint NotStarted(MigrationCategory category) => + new(category.Id, Selected: false, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs new file mode 100644 index 0000000000..0b5a6e3ab3 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Configuration; + +/// The engine's tuning, read from settings once at startup. +/// How long to wait between batches of optional data, so normal work isn't slowed down. Zero means don't wait. +/// A category halts when more than this percent of the rows handled in this run were skipped, and the minimum below is also passed. +/// A category never halts until more than this many rows were skipped in this run, so a few bad rows can't stop a small category. +/// The optional categories to copy. Required categories are always copied. +public sealed record MigrationEngineOptions( + TimeSpan ThrottlePause, + int HaltThresholdPercent, + int HaltThresholdMinimum, + IReadOnlyCollection SelectedOptionalCategoryIds) +{ + public static readonly TimeSpan DefaultBodyRetryBackoff = TimeSpan.FromMilliseconds(200); + + /// How long to wait before trying again to read a message body that failed. Not read from settings. + public TimeSpan BodyRetryBackoff { get; init; } = DefaultBodyRetryBackoff; + + /// Reads the options from settings. Throws if the optional categories setting names one that doesn't exist or isn't optional. + public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace) + { + var throttleMilliseconds = SettingsReader.Read(settingsRootNamespace, MigrationSettings.ThrottlePauseMillisecondsKey, MigrationSettings.DefaultThrottlePauseMilliseconds); + var haltPercent = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdPercentKey, MigrationSettings.DefaultHaltThresholdPercent); + var haltMinimum = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdMinimumKey, MigrationSettings.DefaultHaltThresholdMinimum); + var optionalCategories = SettingsReader.Read(settingsRootNamespace, MigrationSettings.OptionalCategoriesKey, string.Empty); + + var selectedIds = optionalCategories + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToArray(); + + var unknown = selectedIds + .Where(id => MigrationCategoryRegistry.Find(id) is not { Kind: MigrationCategoryKind.Optional }) + .ToArray(); + + if (unknown.Length > 0) + { + throw new InvalidOperationException( + $"{MigrationSettings.OptionalCategoriesKey} names categories that do not exist or are not optional: {string.Join(", ", unknown)}"); + } + + return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds); + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs new file mode 100644 index 0000000000..2b4faf5c91 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// The migration's setting names, relative to the instance's settings root, and their defaults. explains the tuning ones. +public static class MigrationSettings +{ + public const string ThrottlePauseMillisecondsKey = "Migration/ThrottlePauseMilliseconds"; + public const string HaltThresholdPercentKey = "Migration/HaltThresholdPercent"; + public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; + /// A comma-separated list of the optional categories to copy, such as "EventLog, CustomChecks". + public const string OptionalCategoriesKey = "Migration/OptionalCategories"; + /// Which persister holds the old data being copied from. + public const string SourcePersistenceTypeKey = "Migration/SourcePersistenceType"; + + public const int DefaultThrottlePauseMilliseconds = 100; + public const int DefaultHaltThresholdPercent = 5; + public const int DefaultHaltThresholdMinimum = 100; + /// RavenDB is the only source supported today. + public const string DefaultSourcePersistenceType = "RavenDB"; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs new file mode 100644 index 0000000000..c4f442305f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +public enum MigrationSkipReason +{ + BodyUnreadable +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs deleted file mode 100644 index de1d54c3a7..0000000000 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDatabase.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace ServiceControl.Persistence.DataMigration; - -public enum MigrationSourceDatabase -{ - Primary, - Throughput -} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs index c8dcd27f02..3d5692a56d 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs @@ -1,8 +1,12 @@ namespace ServiceControl.Persistence.DataMigration; -public sealed record MigrationSourceDescription( - bool Embedded, - string ServerUrl, - string PrimaryDatabase, - string ThroughputDatabase, - string ServerVersion); +using System.Collections.Generic; + +/// What the source report and dry run print about the source: its version and a list of facts. +public sealed record MigrationSourceDescription(string Version, IReadOnlyList Facts); + +/// One line the source report prints, such as which server or database the source read, with the setting to change if it is wrong. +public sealed record MigrationSourceFact(string Label, string Value, string? SettingKey = null); + +/// A count the source report prints, such as the rows in one collection of one database, so an operator sees how much there is to copy. +public sealed record MigrationSourceInventoryEntry(string Scope, string Name, long Count); diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs new file mode 100644 index 0000000000..ea6803524f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs @@ -0,0 +1,26 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationCheckpointStore : IMigrationCheckpointStore +{ + readonly ConcurrentDictionary checkpoints = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => + Task.FromResult>(checkpoints.Values.ToArray()); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => + Task.FromResult(checkpoints.TryGetValue(categoryId, out var checkpoint) ? checkpoint : null); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + checkpoints[checkpoint.CategoryId] = checkpoint; + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs new file mode 100644 index 0000000000..ad411cc65b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -0,0 +1,85 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationSource : IMigrationSource +{ + readonly Dictionary> rowsByCategory = []; + readonly Dictionary>> bodyAttempts = []; + readonly Dictionary bodies = []; + + public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); + + public void Seed(string categoryId, params MigrationRow[] rows) => rowsByCategory[categoryId] = [.. rows]; + + public void SetBody(string sourceId, MigrationBody? body) => bodies[sourceId] = body; + + public void QueueBodyAttempt(string sourceId, Func attempt) + { + if (!bodyAttempts.TryGetValue(sourceId, out var queue)) + { + bodyAttempts[sourceId] = queue = new Queue>(); + } + queue.Enqueue(attempt); + } + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task Describe(CancellationToken cancellationToken = default) => Task.FromResult(Description); + + public Task> Inventory(CancellationToken cancellationToken = default) => + Task.FromResult>( + [.. rowsByCategory.Select(pair => new MigrationSourceInventoryEntry("memory", pair.Key, pair.Value.Count))]); + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + + public async IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var rows = rowsByCategory.GetValueOrDefault(category.Id) ?? []; + var startIndex = 0; + + if (resumeAfter is not null) + { + var cursorIndex = rows.FindIndex(r => r.SourceId == resumeAfter); + if (cursorIndex < 0) + { + throw new InvalidOperationException($"Category {category.Id} was asked to resume after {resumeAfter}, a cursor this source never issued"); + } + startIndex = cursorIndex + 1; + } + + for (var i = startIndex; i < rows.Count; i += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var slice = rows.Skip(i).Take(batchSize).ToArray(); + yield return new MigrationBatch(slice, slice[^1].SourceId); + await Task.Yield(); + } + } + + public async Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + await Task.Yield(); + + if (bodyAttempts.TryGetValue(sourceId, out var queue) && queue.Count > 0) + { + return queue.Dequeue()(); + } + + return bodies.GetValueOrDefault(sourceId); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs new file mode 100644 index 0000000000..6d7b2cda2b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs @@ -0,0 +1,78 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationSourceTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Streams_seeded_rows_in_batches_of_the_requested_size() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c"), Row("d"), Row("e")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: null, batchSize: 2)) + { + batches.Add(batch); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches, Has.Count.EqualTo(3)); + Assert.That(batches[0].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b" })); + Assert.That(batches[2].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "e" })); + } + } + + [Test] + public async Task ResumeAfter_skips_everything_up_to_and_including_that_id() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "a", batchSize: 10)) + { + batches.Add(batch); + } + + Assert.That(batches.Single().Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "b", "c" })); + } + + [Test] + public void A_cursor_the_source_never_issued_throws_rather_than_starting_again() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b")); + + Assert.ThrowsAsync(async () => + { + await foreach (var _ in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "no-such-row", batchSize: 10)) + { + } + }); + } + + [Test] + public async Task ReadBody_returns_a_queued_attempt_then_falls_back_to_the_seeded_body() + { + var source = new InMemoryMigrationSource(); + var finalBody = new MigrationBody(new byte[] { 9 }, "text/plain"); + source.SetBody("msg-1", finalBody); + source.QueueBodyAttempt("msg-1", () => throw new System.Exception("transient")); + + Assert.ThrowsAsync(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); + var secondAttempt = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1"); + + Assert.That(secondAttempt, Is.EqualTo(finalBody)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs new file mode 100644 index 0000000000..68bbe3139f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -0,0 +1,85 @@ +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget +{ + readonly Dictionary> writtenKeysByCategory = []; + readonly Dictionary> writtenRowsByCategory = []; + readonly HashSet preExistingKeys = []; + readonly Dictionary rejectedKeys = []; + + public int DefaultBatchSize { get; set; } = 3; + public int? FailOnCallNumber { get; set; } + public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } + int callCount; + + public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); + + public void RejectKey(string sourceId, string reason) => rejectedKeys[sourceId] = reason; + + public IReadOnlyList WrittenRows(string categoryId) => + writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + public int BatchSizeFor(MigrationCategory category) => DefaultBatchSize; + + public async Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointAfterBatch, + CancellationToken cancellationToken = default) + { + callCount++; + + if (StopOnCall is { } stop && stop.CallNumber == callCount) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (FailOnCallNumber == callCount) + { + throw new InvalidOperationException($"Simulated failure on write {callCount}"); + } + + var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; + var rows = writtenRowsByCategory.TryGetValue(category.Id, out var existingRows) ? existingRows : writtenRowsByCategory[category.Id] = []; + + var copied = 0; + var alreadyPresent = 0; + var skippedIds = new List(); + var skipReasons = new Dictionary(); + + foreach (var row in batch.Rows) + { + if (rejectedKeys.TryGetValue(row.SourceId, out var reason)) + { + skippedIds.Add(row.SourceId); + skipReasons[reason] = skipReasons.GetValueOrDefault(reason) + 1; + continue; + } + + if (preExistingKeys.Contains(row.SourceId) || !keys.Add(row.SourceId)) + { + alreadyPresent++; + continue; + } + + rows.Add(row); + copied++; + } + + // Verbatim and in the same operation as the rows, as the real targets persist it. Adding the + // batch's own counts here would double every number the engine already included. + await checkpointStore.Upsert(checkpointAfterBatch, cancellationToken); + + return new MigrationWriteResult(copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs new file mode 100644 index 0000000000..b10a340257 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs @@ -0,0 +1,80 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationTargetTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationCheckpoint EmptyCheckpoint(string categoryId) => + new(categoryId, Selected: true, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null, null); + + [Test] + public async Task Writes_new_rows_and_persists_the_checkpoint_it_was_handed_exactly_as_given() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("EndpointSettings")!; + var batch = new MigrationBatch([Row("a"), Row("b")], Cursor: "b"); + // Absolute post-batch totals, computed by the caller. The target does no arithmetic on them. + var checkpointAfterBatch = EmptyCheckpoint(category.Id) with { Cursor = "b", CopiedCount = 2 }; + + var result = await target.Write(category, batch, checkpointAfterBatch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(2)); + Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointAfterBatch)); + } + } + + [Test] + public async Task The_persisted_checkpoint_is_not_adjusted_by_what_the_write_actually_did() + { + // The engine's totals already count every row as copied, so a target that corrected them from + // its own result would double the counts. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + target.SeedExistingKey("already-present"); + target.RejectKey("rejected", "Rejected"); + var batch = new MigrationBatch([Row("already-present"), Row("rejected"), Row("new-row")], Cursor: "new-row"); + var checkpointAfterBatch = EmptyCheckpoint(category.Id) with { Cursor = "new-row", CopiedCount = 3 }; + + var result = await target.Write(category, batch, checkpointAfterBatch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "rejected" })); + Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary { ["Rejected"] = 1 })); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(1)); + Assert.That((await checkpointStore.Read(category.Id))!.CopiedCount, Is.EqualTo(3)); + } + } + + [Test] + public async Task Count_answers_for_one_category_rather_than_one_table() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var required = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + await target.Write(required, new MigrationBatch([Row("a"), Row("b")], "b"), EmptyCheckpoint(required.Id)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await target.Count(required), Is.EqualTo(2)); + Assert.That(await target.Count(archive), Is.Zero); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs new file mode 100644 index 0000000000..f0b02dbdfb --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class HaltThresholdTests +{ + [Test] + public void Does_not_halt_a_three_row_category_with_one_bad_row() + { + // 1 of 3 is 33%, well past the 5% proportion, but 1 skip never passes the 100-row floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 1, totalCount: 3, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Halts_a_large_category_with_a_systemic_failure() + { + // 600 of 10,000 is 6%, past both the proportion and the floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 600, totalCount: 10_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void Does_not_halt_a_large_category_with_only_a_small_proportion_skipped() + { + // 10,000 of 5,000,000 is 0.2%: past the floor, nowhere near the proportion. + var exceeded = HaltThreshold.Exceeded(skippedCount: 10_000, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_at_both_boundaries_does_not_halt_because_both_must_be_exceeded() + { + // 100 of 2,000 is exactly 5% and exactly the floor. "Exceed" means strictly past, not "at". + var exceeded = HaltThreshold.Exceeded(skippedCount: 100, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void No_rows_processed_yet_never_halts() + { + Assert.That(HaltThreshold.Exceeded(skippedCount: 0, totalCount: 0, percentThreshold: 5, minimumFloor: 100), Is.False); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs new file mode 100644 index 0000000000..9b5e637ec3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -0,0 +1,110 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryRegistryTests +{ + [Test] + public void Contains_all_eighteen_categories_with_unique_ids() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); + } + } + + [Test] + public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var requiredIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Required) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(requiredIds, Is.EqualTo(new[] + { + "KnownEndpoints", "EndpointSettings", "MessageRedirects", "Subscriptions", + "NotificationSettings", "TrialEndDate", "RetryOperations", + "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", + "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages" + })); + } + + [Test] + public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var optionalIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Optional) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(optionalIds, Is.EqualTo(new[] + { + "EventLog", "CustomChecks", "FailedErrorImports", + "FailedMessageEdits", "ArchivedAndResolvedFailedMessages", "GroupComments" + })); + } + + [Test] + public void Three_categories_declare_a_MustFollow() + { + var declared = MigrationCategoryRegistry.All + .Where(c => c.MustFollow is not null) + .ToDictionary(c => c.Id, c => c.MustFollow); + + Assert.That(declared, Is.EqualTo(new Dictionary + { + // A cascading foreign key: throughput rows cannot exist before their endpoint row. + ["LicensingThroughput"] = "LicensingEndpoints", + // Keeps settings back while known endpoints are unfinished. The heartbeat sync is kept off + // the copied settings because both categories finish before the host opens. + ["EndpointSettings"] = "KnownEndpoints", + // A group comment whose failed messages have not arrived reads as an orphan to the + // retention sweeper, which deletes it. + ["GroupComments"] = "ArchivedAndResolvedFailedMessages" + })); + } + + [Test] + public void The_two_failed_message_categories_and_failed_imports_carry_bodies() + { + var carriesBodies = MigrationCategoryRegistry.All.Where(c => c.CarriesBodies).Select(c => c.Id).ToArray(); + + Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); + } + + [Test] + public void The_two_failed_message_categories_split_every_status_between_them() + { + var split = MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(split, Is.Unique); + Assert.That(split, Is.EquivalentTo(Enum.GetValues()), + "a status neither category claims is a failed message no migration copies"); + } + } + + [Test] + public void Find_returns_null_for_an_unknown_id() + { + Assert.That(MigrationCategoryRegistry.Find("NoSuchCategory"), Is.Null); + } + + [Test] + public void Find_returns_the_category_for_a_known_id() + { + Assert.That(MigrationCategoryRegistry.Find("EventLog")?.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs new file mode 100644 index 0000000000..2c662bffbe --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -0,0 +1,99 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationContractShapeTests +{ + [Test] + public void MigrationCategory_carries_no_fact_about_where_a_store_keeps_its_rows() + { + var properties = typeof(MigrationCategory).GetProperties().Select(p => p.Name); + + Assert.That(properties, Is.EquivalentTo(new[] { "Id", "Kind", "CarriesBodies", "Order", "MustFollow" }), + "a source database or target table belongs in that store's own adapter, where a different store pair can map it differently"); + } + + [Test] + public void Abandoned_is_a_state_of_its_own_and_not_a_kind_of_complete() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.Complete)); + Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.Halted)); + } + } + + [Test] + public void Category_states_keep_the_integers_stored_checkpoints_already_hold() + { + var stored = System.Enum.GetValues().ToDictionary(state => state.ToString(), state => (int)state); + + Assert.That(stored, Is.EqualTo(new Dictionary + { + ["NotStarted"] = 0, + ["InProgress"] = 1, + ["Complete"] = 2, + ["CompleteWithErrors"] = 3, + ["Halted"] = 4, + ["Abandoned"] = 5 + }), "checkpoints store State as an integer, so a reordered or inserted member silently changes what every saved checkpoint means"); + } + + [Test] + public void MigrationRow_body_defaults_to_null() + { + var row = new MigrationRow("id-1", new object(), new Dictionary()); + + Assert.That(row.Body, Is.Null); + } + + [Test] + public void MigrationRow_can_carry_a_body() + { + var body = new MigrationBody(new byte[] { 1, 2, 3 }, "text/plain"); + var row = new MigrationRow("id-1", new object(), new Dictionary(), body); + + Assert.That(row.Body, Is.EqualTo(body)); + } + + [Test] + public void MigrationBatch_groups_rows_under_one_cursor() + { + var rows = new[] { new MigrationRow("id-1", new object(), new Dictionary()) }; + var batch = new MigrationBatch(rows, Cursor: "id-1"); + + Assert.That(batch.Cursor, Is.EqualTo("id-1")); + } + + [Test] + public void MigrationCheckpoint_starts_with_no_cursor_and_zero_counts() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + Selected: false, + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + CompletedAt: null, + AbandonedAt: null, + LastError: null); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.Cursor, Is.Null); + Assert.That(checkpoint.CopiedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs new file mode 100644 index 0000000000..a2680372f8 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -0,0 +1,124 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineBodyRetryTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_written_with_its_body_attached() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + for (var attempt = 1; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("blip")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + // Zeroed because FakeTimeProvider is never advanced here: a 200 ms Task.Delay against a clock + // nobody moves never completes, and the test hangs until the runner kills it. + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + } + } + + [Test] + public async Task A_body_the_source_already_attached_is_written_without_reading_it_again() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var attached = new MigrationBody(new byte[] { 7 }, "text/plain"); + source.Seed(category.Id, Row("msg-1") with { Body = attached }); + for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("a body read nobody needed")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(attached)); + } + } + + [Test] + public async Task Exhausted_attempts_skip_the_whole_message_and_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("down")); + } + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // msg-1 is excluded entirely: not written with a missing body, not written at all. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + } + } + + [Test] + public async Task A_body_store_outage_across_many_messages_halts_the_category() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var ids = Enumerable.Range(1, 1_000).Select(i => $"msg-{i}").ToArray(); + source.Seed(category.Id, [.. ids.Select(Row)]); + // Every body read fails every attempt: a down body store, not a per-message fluke. + foreach (var id in ids) + { + for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt(id, () => throw new InvalidOperationException("body store unreachable")); + } + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs new file mode 100644 index 0000000000..82d806a947 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCategorySelectionTests +{ + static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalIds, out InMemoryMigrationCheckpointStore checkpointStore) + { + checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, selectedOptionalIds); + return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + } + + [Test] + public void All_twelve_required_categories_are_always_selected() + { + var engine = BuildEngine([], out _); + + Assert.That(engine.SelectCategories(MigrationCategoryKind.Required), Has.Count.EqualTo(12)); + } + + [Test] + public void Only_configured_optional_categories_are_selected() + { + var engine = BuildEngine(["EventLog"], out _); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + + Assert.That(selected.Select(c => c.Id), Is.EqualTo(new[] { "EventLog" })); + } + + [Test] + public void A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() + { + var engine = BuildEngine([], out var checkpointStore); + var previousRun = new MigrationCheckpoint("EventLog", Selected: true, MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null); + checkpointStore.Upsert(previousRun).GetAwaiter().GetResult(); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + + using (Assert.EnterMultipleScope()) + { + Assert.That(selected.Select(c => c.Id), Does.Not.Contain("EventLog")); + Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun)); + } + } + + [Test] + public void A_category_added_to_configuration_is_selected_on_the_next_run() + { + var engine = BuildEngine(["CustomChecks"], out _); + + Assert.That(engine.SelectCategories(MigrationCategoryKind.Optional).Select(c => c.Id), Is.EqualTo(new[] { "CustomChecks" })); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs new file mode 100644 index 0000000000..59b53c6a07 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs @@ -0,0 +1,45 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCollisionTests +{ + static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary()); + + [Test] + public async Task Rows_already_present_in_the_target_are_counted_as_already_present_across_batches_not_skipped_or_overwritten() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1", "from-source"), Row("msg-2", "from-source"), Row("msg-3", "from-source"), Row("msg-4", "from-source")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // msg-1 and msg-4 already exist in the target: they failed again after cutover, and real ingestion wrote them. + target.SeedExistingKey("msg-1"); + target.SeedExistingKey("msg-4"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + // Asserted through the fake's recorded rows, never by the engine inspecting a document. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2", "msg-3" })); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs new file mode 100644 index 0000000000..c48c726e82 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -0,0 +1,64 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCopyTests +{ + static MigrationRow Row(string id) => new(id, new { Name = id }, new Dictionary()); + + [Test] + public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skipped() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.Cursor, Is.EqualTo("c")); + Assert.That(checkpoint.StartedAt, Is.Not.Null); + Assert.That(checkpoint.CompletedAt, Is.Not.Null); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(3)); + } + } + + [Test] + public async Task A_category_already_Complete_is_left_alone_on_a_second_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var alreadyDone = new MigrationCheckpoint(category.Id, true, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null); + await checkpointStore.Upsert(alreadyDone); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(alreadyDone)); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs new file mode 100644 index 0000000000..4c42966148 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -0,0 +1,163 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineFailurePathTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task A_failing_write_halts_the_category_and_records_the_error_instead_of_throwing() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Simulated failure")); + // The first batch committed, so the cursor is real and a later run resumes from it. + Assert.That(checkpoint.Cursor, Is.EqualTo("b")); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task The_halt_is_durable_so_the_health_check_and_the_guard_can_read_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + await engine.RunCategoryAsync(category); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(persisted.LastError, Is.Not.Null); + } + } + + [Test] + public void A_cancelled_run_is_a_shutdown_rather_than_a_failure_and_is_not_swallowed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + } + + [Test] + public async Task A_halted_category_is_re_attempted_on_the_next_run_and_resumes_from_its_cursor() + { + // A halt that no restart can clear would leave abandoning the category as the only way out of + // a fault the customer has already repaired. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var firstRun = BuildEngine(source, checkpointStore, target); + var halted = await firstRun.RunCategoryAsync(category); + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + + target.FailOnCallNumber = null; + var secondRun = BuildEngine(source, checkpointStore, target); + var finished = await secondRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "a cleared halt does not leave a stale error on the row"); + Assert.That(finished.CopiedCount, Is.EqualTo(4)); + var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); + Assert.That(writtenIds, Is.EquivalentTo(new[] { "a", "b", "c", "d" })); + Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates across the halt"); + } + } + + [Test] + public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_across_the_restart() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // msg-1's body is unreadable on both runs: every attempt fails on each. + for (var attempt = 0; attempt < 2 * MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("body store unreachable")); + } + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var firstRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var halted = await firstRun.RunCategoryAsync(category); + + target.FailOnCallNumber = null; + var secondRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finished = await secondRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + // The failed batch never committed, so its skip is recorded only when the batch is read again. + Assert.That(halted.SkippedCount, Is.Zero); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(finished.SkippedCount, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + } + } + + [Test] + public async Task An_abandoned_category_is_left_exactly_as_it_is() + { + // Abandoned is the one end a person chooses: this category will never be copied, and the + // guard and the health check both treat it as settled. A later run must not quietly restart it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var abandoned = new MigrationCheckpoint(category.Id, true, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, null, DateTime.UtcNow, "Halted: the body store was unreachable"); + await checkpointStore.Upsert(abandoned); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(abandoned)); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs new file mode 100644 index 0000000000..fb114a7409 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -0,0 +1,84 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineHaltTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_systemic_failure_halts_the_category_partway_through() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // Every 5th of 1,000 rows is rejected: a steady 20% spread evenly rather than clustered at the + // start, past both the 5% proportion and the 100-row floor. + var rows = Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}")).ToArray(); + source.Seed(category.Id, rows); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", "Rejected"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Halted")); + Assert.That(checkpoint.CompletedAt, Is.Not.Null); + // Stopped partway: the 1,000th row was never reached. + Assert.That(target.WrittenRows(category.Id).Count, Is.LessThan(800)); + } + } + + [Test] + public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.SeedExistingKey($"row-{i}"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(800)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(200)); + } + } + + [Test] + public void Halted_is_distinct_from_Complete_and_CompleteWithErrors() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryState.Halted, Is.Not.EqualTo(MigrationCategoryState.Complete)); + Assert.That(MigrationCategoryState.Halted, Is.Not.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(MigrationCategoryState.CompleteWithErrors, Is.Not.EqualTo(MigrationCategoryState.Complete)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs new file mode 100644 index 0000000000..eb296bfd71 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEngineOptionsTests +{ + static readonly SettingsRootNamespace Namespace = new("ServiceControl"); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", null); + } + + [Test] + public void Defaults_match_the_contract_when_nothing_is_configured() + { + var options = MigrationEngineOptions.FromSettings(Namespace); + + using (Assert.EnterMultipleScope()) + { + // At one second a million-row copy in 50-row batches spends five and a half hours paused + // before it does any work at all. + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); + Assert.That(options.SelectedOptionalCategoryIds, Is.Empty); + Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void Reads_configured_values_from_environment_variables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", "2500"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", "10"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", "50"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EventLog, CustomChecks"); + + var options = MigrationEngineOptions.FromSettings(Namespace); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(2500))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(10)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { "EventLog", "CustomChecks" })); + } + } + + [Test] + public void Refuses_an_unknown_optional_category_id() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "NoSuchCategory"); + + var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + + Assert.That(ex.Message, Does.Contain("NoSuchCategory")); + } + + [Test] + public void Refuses_a_required_category_id_named_as_optional() + { + // EndpointSettings is required, not optional: naming it here is a customer mistake, not + // a valid way to force it. Required categories are never a matter of configuration. + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EndpointSettings"); + + var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + + Assert.That(ex.Message, Does.Contain("EndpointSettings")); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs new file mode 100644 index 0000000000..abf41f8ac9 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -0,0 +1,137 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineOrderingTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + // LicensingEndpoints has never run: no checkpoint row for it at all. + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); + } + } + + [Test] + public async Task A_blocked_category_says_so_on_its_checkpoint_row_instead_of_returning_in_silence() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + await engine.RunCategoryAsync(throughputCategory); + + var persisted = await checkpointStore.Read(throughputCategory.Id); + using (Assert.EnterMultipleScope()) + { + // A row exists, so status can print it. Without one, an operator cannot tell a category + // waiting on another from a category nobody asked for. + Assert.That(persisted, Is.Not.Null); + Assert.That(persisted!.Selected, Is.True); + Assert.That(persisted.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(persisted.LastError, Does.Contain("LicensingEndpoints")); + } + } + + [Test] + public async Task EndpointSettings_does_not_start_before_KnownEndpoints_completes() + { + // Not a foreign key: settings wait for known endpoints, and a halted required category keeps + // the host, and with it the heartbeat sync, closed. + var settingsCategory = MigrationCategoryRegistry.Find("EndpointSettings")!; + var source = new InMemoryMigrationSource(); + source.Seed(settingsCategory.Id, Row("EndpointSettings/1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(settingsCategory); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(target.WrittenRows(settingsCategory.Id), Is.Empty); + } + } + + [Test] + public async Task GroupComments_does_not_start_before_the_archive_completes() + { + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", true, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(target.WrittenRows(comments.Id), Is.Empty); + } + } + + [Test] + public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_Complete() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, MigrationCategoryState.Complete, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } + + [Test] + public async Task LicensingThroughput_proceeds_when_LicensingEndpoints_is_CompleteWithErrors() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, MigrationCategoryState.CompleteWithErrors, "e-1", 9, 1, 10, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs new file mode 100644 index 0000000000..e0b5fd13ed --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -0,0 +1,60 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineResumeTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_and_no_gaps() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + var allIds = Enumerable.Range(1, 6).Select(i => $"row-{i}").ToArray(); + source.Seed(category.Id, [.. allIds.Select(Row)]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + // Batches are 2 rows each; the host stops on the 3rd call to Write, so exactly 2 batches (4 rows) land. + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + + var afterStop = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + // Still InProgress, not Halted: a cancelled run is a shutdown, not a failure. + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(afterStop.CopiedCount, Is.EqualTo(4)); + Assert.That(afterStop.Cursor, Is.EqualTo("row-4")); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(4)); + } + + // Restart: same source, same target, same checkpoint store, nothing stopping it this time. + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finalCheckpoint.CopiedCount, Is.EqualTo(6)); + var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); + Assert.That(writtenIds, Is.EquivalentTo(allIds), "no gaps"); + Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates"); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs new file mode 100644 index 0000000000..8503b3c257 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -0,0 +1,67 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineRunCategoriesTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + var ordered = engine.SelectCategories(MigrationCategoryKind.Required).Take(2).ToArray(); + foreach (var category in ordered) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + var results = await engine.RunCategories(ordered); + + using (Assert.EnterMultipleScope()) + { + // KnownEndpoints then EndpointSettings, the order the registry declares, whichever order + // the caller passed them in. + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "KnownEndpoints", "EndpointSettings" })); + Assert.That(results.Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task One_category_halting_does_not_stop_the_ones_after_it() + { + // Everything that can still be copied is copied, and the guard decides what an incomplete migration + // may do. Stopping at the first halt would leave later categories untouched with no reason recorded. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1, FailOnCallNumber = 1 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + var first = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var second = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(first.Id, Row("k-1")); + source.Seed(second.Id, Row("r-1")); + + var results = await engine.RunCategories([first, second]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs new file mode 100644 index 0000000000..1319ae20b3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -0,0 +1,96 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineSkipReasonTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkpoint() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + target.RejectKey("a", "KeyTooLong"); + target.RejectKey("b", "Unparseable"); + target.RejectKey("d", "KeyTooLong"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { ["KeyTooLong"] = 2, ["Unparseable"] = 1 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { ["KeyTooLong"] = 2, ["Unparseable"] = 1 })); + } + } + + [Test] + public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnreadable() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) + { + source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("down")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [nameof(MigrationSkipReason.BodyUnreadable)] = 1 })); + } + + [Test] + public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new UnexplainedSkipTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("reported 1 skipped").And.Contain("reasons for 0")); + Assert.That(checkpoint.Cursor, Is.EqualTo("a"), "the target committed the batch, so a restart must carry on after it"); + } + } + + sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public int BatchSizeFor(MigrationCategory category) => 10; + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointAfterBatch, CancellationToken cancellationToken = default) + { + await checkpointStore.Upsert(checkpointAfterBatch, cancellationToken); + return new MigrationWriteResult(0, batch.Rows.Count, []); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs new file mode 100644 index 0000000000..44bda6a340 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -0,0 +1,62 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineThrottleTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Optional_categories_pause_between_batches_for_the_configured_duration() + { + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var timeProvider = new FakeTimeProvider(); + var pause = TimeSpan.FromSeconds(1); + var options = new MigrationEngineOptions(pause, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, timeProvider, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + await Task.Delay(50); + Assert.That(runTask.IsCompleted, Is.False, "the first inter-batch pause should still be pending"); + + timeProvider.Advance(pause); + await Task.Delay(50); + Assert.That(runTask.IsCompleted, Is.False, "the second inter-batch pause should still be pending"); + + timeProvider.Advance(pause); + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + } + + [Test] + public async Task Required_categories_never_pause() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + // A FakeTimeProvider that is never advanced: if the engine tried to pause, this would hang + // until the test runner's own timeout, which WaitAsync turns into a clear failure instead. + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + } +} diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs index ee2743bb8b..ce0ac0501b 100644 --- a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -1,13 +1,12 @@ namespace ServiceControl.Hosting.Commands { using System; - using System.Collections.Generic; + using System.Linq; using System.Threading; using System.Threading.Tasks; using Particular.ServiceControl.Hosting; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Persistence; - using ServiceControl.Persistence.DataMigration; class MigrationSourceReportCommand : AbstractCommand { @@ -19,37 +18,24 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel Console.Out.WriteLine("ServiceControl migration source report"); Console.Out.WriteLine(); - Console.Out.WriteLine($"Source persistence : {settings.MigrationSourcePersistenceType} ({(description.Embedded ? "embedded" : "external")})"); - Console.Out.WriteLine($"Server URL : {description.ServerUrl}"); - Console.Out.WriteLine($"Server version : {description.ServerVersion}"); - Console.Out.WriteLine($"Primary database : {description.PrimaryDatabase} (from ServiceControl/RavenDB/DatabaseName)"); - Console.Out.WriteLine($"Throughput database : {description.ThroughputDatabase} (from LicensingComponent/RavenDB/ThroughputDatabaseName)"); + Console.Out.WriteLine($"{"Source persistence",-20}: {settings.MigrationSourcePersistenceType}"); + Console.Out.WriteLine($"{"Version",-20}: {description.Version}"); - await PrintCollections(source, MigrationSourceDatabase.Primary, description.PrimaryDatabase, cancellationToken); - await PrintCollections(source, MigrationSourceDatabase.Throughput, description.ThroughputDatabase, cancellationToken); - } - - static async Task PrintCollections(IMigrationSource source, MigrationSourceDatabase database, string databaseName, CancellationToken cancellationToken) - { - var counted = new SortedDictionary(StringComparer.Ordinal); - - foreach (var collection in await source.CountCollections(database, cancellationToken)) + foreach (var fact in description.Facts) { - counted[collection.Key] = collection.Value; + var origin = fact.SettingKey is null ? string.Empty : $" (from {fact.SettingKey})"; + Console.Out.WriteLine($"{fact.Label,-20}: {fact.Value}{origin}"); } - Console.Out.WriteLine(); - Console.Out.WriteLine($"Collections in {databaseName}:"); - - if (counted.Count == 0) + foreach (var scope in (await source.Inventory(cancellationToken)).GroupBy(entry => entry.Scope)) { - Console.Out.WriteLine(" (none)"); - return; - } + Console.Out.WriteLine(); + Console.Out.WriteLine($"{scope.Key}:"); - foreach (var collection in counted) - { - Console.Out.WriteLine($" {collection.Key,-42}{collection.Value,12:N0}"); + foreach (var entry in scope.OrderBy(entry => entry.Name, StringComparer.Ordinal)) + { + Console.Out.WriteLine($" {entry.Name,-42}{entry.Count,12:N0}"); + } } } } diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index 8bceaf1eb6..3c3d0d5774 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -27,11 +27,12 @@ MIGRATION SOURCE REPORT ServiceControl.exe --migration-source-report -Reports what a RavenDB to SQL migration would read: whether the old database is embedded or on its -own server, which server, both database names with the setting each came from, and a row count for -every collection. +Reports what a migration would read from the source persistence named by +ServiceControl/Migration/SourcePersistenceType: the facts the source reports about itself, with the +setting each came from, and a row count for everything it holds. The source reads the instance's own +RavenDB settings, so keep them in place when switching PersistenceType. -An EXTERNAL RavenDB source can be reported on while ServiceControl is running. An EMBEDDED source +For a RavenDB source: an EXTERNAL server can be reported on while ServiceControl is running. An EMBEDDED source cannot: ServiceControl starts its own RavenDB process against that data directory, and a second one cannot attach to it. Stop the ServiceControl service first, run the report, and start it again. An instance that does not ship the RavenDB server, such as the container image, cannot report on an diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index c8c0b694e0..0bab996629 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -66,7 +66,7 @@ public HostArguments(string[] args) { { "migration-source-report", - "Report what a migration would read from the old RavenDB database, without changing it", + "Report what a migration would read from the source persistence, without changing it", s => Command = typeof(MigrationSourceReportCommand) } }; diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index eaa846e5b8..fbdc0e50a4 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,6 +16,7 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -185,7 +186,7 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } - public string MigrationSourcePersistenceType => SettingsReader.Read(SettingsRootNamespace, "Migration/SourcePersistenceType", "RavenDB"); + public string MigrationSourcePersistenceType => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.SourcePersistenceTypeKey, MigrationSettings.DefaultSourcePersistenceType); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 9d4f255e7f..5aac4cdda8 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -35,12 +35,10 @@ public static IMigrationSource CreateMigrationSource(Settings settings) if (persistenceConfiguration is not IMigrationSourceFactory sourceFactory) { - throw new Exception($"The '{persistenceType}' persistence cannot be read as a migration source. Set ServiceControl/Migration/SourcePersistenceType to the persistence that holds the data being migrated away from."); + throw new Exception($"The '{persistenceType}' persistence cannot be read as a migration source. Set {Settings.SettingsRootNamespace}/{MigrationSettings.SourcePersistenceTypeKey} to the persistence that holds the data being migrated away from."); } - // Not PersisterSpecificSettings: it is null here, and a host that has populated it put the - // target's connection string in it. - return sourceFactory.CreateSource(persistenceConfiguration.CreateSettings(Settings.SettingsRootNamespace)); + return sourceFactory.CreateSource(Settings.SettingsRootNamespace); } public static async Task OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) From 0afd1e1a1a18cc876dde9bb5bc646c25c2b23a39 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 15 Sep 2026 14:36:34 +0800 Subject: [PATCH 07/28] Clarify migration overview by detailing additional skip conditions for SQL migration from RavenDB --- docs/migration/ravendb-to-sql-migration-overview.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 277a53c783..d9fa6454f2 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -187,6 +187,10 @@ flowchart TB - A failed message with no processing attempts recorded against it. The SQL model keeps the newest attempt and derives the failure time, the failing endpoint and the exception from it, all of which are required columns, so a message with nothing to derive them from cannot be written at all rather than being written blank. - A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. - A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. +- An archived or resolved failed message, or an event log item, already past its retention period. SQL's retention clean-up would delete it on its first pass, so it is counted rather than copied only to be deleted. +- A group comment whose failure group has no failed messages in SQL once the messages are copied. SQL's clean-up removes such a comment, where RavenDB never expired one. +- Endpoint settings for an endpoint ServiceControl does not know. ServiceControl removes those settings shortly after it starts. +- A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way, because ServiceControl never stores one. **Things that change shape, and are not counted as skips at all.** The dry run counts these before anything moves, so they are a number you see in advance rather than a discovery afterwards. They are also the ones to read twice: @@ -240,6 +244,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. - The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone lets ten thousand failures pass on a five-million-row table as "only 0.2%". +- Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. - Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. ## Dry run From f1166643895d9f95647f25206b08e408535f2afb Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 15 Sep 2026 16:13:15 +0800 Subject: [PATCH 08/28] PR review fixes --- .github/workflows/ci.yml | 4 +- .../ravendb-to-sql-migration-instructions.md | 4 +- .../ravendb-to-sql-migration-overview.md | 6 +- .../ServiceControl.Migration.Tests.csproj | 6 +- .../TwoPersistersInOneProcessTests.cs | 15 +--- .../RavenPersistenceConfiguration.cs | 2 +- .../ReadOnlySourceLifecycleTests.cs | 23 +---- .../DataMigration/HaltThreshold.cs | 3 +- .../DataMigration/MigrationEngine.cs | 75 ++++++++-------- .../DataMigration/MigrationEngineOptions.cs | 6 +- .../Migration/Fakes/CapturingLogger.cs | 19 ++++ .../Fakes/InMemoryMigrationSource.cs | 19 ++-- .../Fakes/InMemoryMigrationSourceTests.cs | 12 ++- .../Fakes/InMemoryMigrationTarget.cs | 4 +- .../Migration/MigrationContractShapeTests.cs | 38 +------- .../MigrationEngineBodyRetryTests.cs | 64 +++++++++---- .../MigrationEngineCategorySelectionTests.cs | 8 -- .../MigrationEngineFailurePathTests.cs | 89 +++++++++++++++++-- .../Migration/MigrationEngineHaltTests.cs | 25 +++++- .../Migration/MigrationEngineOptionsTests.cs | 2 - .../Migration/MigrationEngineOrderingTests.cs | 53 +++-------- .../Migration/MigrationEngineResumeTests.cs | 74 +++++++++++++++ .../MigrationEngineRunCategoriesTests.cs | 11 ++- .../MigrationEngineSkipReasonTests.cs | 5 +- .../Migration/MigrationEngineThrottleTests.cs | 52 +++++++++-- src/ServiceControl/Hosting/HostArguments.cs | 2 +- 26 files changed, 384 insertions(+), 237 deletions(-) create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d01930ff9..0af212e3f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,7 +120,7 @@ jobs: # prevent and cost the critical path about two minutes; the step timeouts turn a hang into a # five minute failure instead of a thirty minute one. - name: Setup WSL - if: contains(fromJSON('["RabbitMQ", "SqlServer", "PostgreSql", "IBMMQ", "Migration"]'), matrix.test-category) + if: contains(fromJSON('["RabbitMQ", "SqlServer", "PostgreSql", "IBMMQ"]'), matrix.test-category) uses: Particular/setup-wsl-action@v1.3.0 timeout-minutes: 5 with: @@ -129,7 +129,7 @@ jobs: memory: 8GB - name: Setup SQL Server uses: Particular/install-sql-server-action@v3.0.0 - if: contains(fromJSON('["SqlServer", "Migration"]'), matrix.test-category) + if: matrix.test-category == 'SqlServer' with: connection-string-env-var: ServiceControl_Persistence_SqlServer_ConnectionString catalog: ServiceControl diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index af849a9e43..7b3fc8c7ae 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -3,7 +3,7 @@ This page covers what you can run today. How the migration works, and what is planned, is in the [migration overview](ravendb-to-sql-migration-overview.md) and the [system design diagram](migration-system-design-diagram.png). > [!NOTE] -> Copying data is not built yet. The one migration command available is the source report, which reads the RavenDB database and changes nothing. +> Copying data is not built yet. The one migration command available is the source report. It sends RavenDB only reads, but loading a database lets RavenDB's own expiration, its automatic deletion of documents past their retention date, run against it. If you are keeping the RavenDB database as a fallback, back it up before you run the report, as [Goals](ravendb-to-sql-migration-overview.md#goals) explains. ## Before you start @@ -38,7 +38,7 @@ From source, build `src/ServiceControl` and run the same command from its output The report prints the RavenDB server version, whether the source is embedded or external and where it is, both database names with the setting each came from, and a row count for every collection in both databases. -- **External server:** run it while ServiceControl is running. It only reads. +- **External server:** run it while ServiceControl is running. It sends only reads, and the note above about expiration applies to a server you are keeping as a fallback. - **Embedded database:** stop the ServiceControl service, run the report, then start the service again. The report starts its own RavenDB process against the data directory, which cannot happen while the instance holds it. - **Container with an embedded database:** not supported, because the container image does not ship the RavenDB server. Point the instance at an external RavenDB server instead. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index d9fa6454f2..3f04217729 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -4,6 +4,8 @@ A customer can already point ServiceControl at SQL Server or PostgreSQL. They cannot bring their existing data with them. +This covers the error instance only. The audit instance has no SQL persister, so a customer who finishes this migration still runs RavenDB for audit. + ## Strategy - Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention is shorter, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. @@ -243,7 +245,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - A body is read up to three times before the message is skipped whole, and the exhausted attempts count toward the halt threshold. - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. -- The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone lets ten thousand failures pass on a five-million-row table as "only 0.2%". +- The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. - Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. - Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. @@ -274,7 +276,7 @@ It reports no duration for the optional categories, and nothing about load on th `--migration-source-report`, `--migration-verify` and `--migration-dry-run` all open the RavenDB source. **On an embedded source that means stopping the ServiceControl service first**, because a second RavenDB process cannot attach to a data directory the first one holds. Plan the dry run as part of the outage rather than as something you run the day before while the instance keeps serving traffic. On an external source, a container or RavenDB Cloud, all three run against a live instance with no interruption. -One deployment shape they cannot help at all: all three need shell access to the host, so a containerised instance has no easy way to run them, and a containerised instance cannot use an embedded source either. +A containerised instance runs all three as a one-off `docker run` of the same image with the command's flag, against an external RavenDB server, as the [instructions](ravendb-to-sql-migration-instructions.md#report-on-the-source) show for the source report. It cannot use an embedded source, because the image does not ship the RavenDB server. `--migration-status` is the exception and is deliberately so: it reads only the checkpoint table in SQL and never opens the source, so it works on every source shape at any time, including during the background copy. It is the command to use for watching progress. diff --git a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj index 8b9f287883..da4cda3938 100644 --- a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj +++ b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj @@ -5,10 +5,8 @@ Migration - + diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs index 61958afec2..ba677f8b42 100644 --- a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -11,7 +11,6 @@ namespace ServiceControl.Migration.Tests; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; using ServiceControl.Persistence; -using ServiceControl.Persistence.DataMigration; [TestFixture] [NonParallelizable] @@ -28,8 +27,7 @@ public void SetUp() // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather // than off the Settings object, so the constructor argument below does not satisfy either of them. SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); - // CI's install-sql-server-action publishes only this variable and creates the catalog - // ServiceControl; a developer machine has neither, so the local default is the fallback. + // Registering the SQL Server persistence never connects, so any connection string satisfies it. SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", Environment.GetEnvironmentVariable("ServiceControl_Persistence_SqlServer_ConnectionString") ?? "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); @@ -76,21 +74,14 @@ public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_ide "The target persister must be isolated too, otherwise only one half of the pairing is being tested."); Assert.That(sourceContext, Is.Not.SameAs(targetContext), "Source and target must land in separate contexts. That separation is the whole feature, and nothing else here asserts it."); - Assert.That(source, Is.InstanceOf(), - "A plugin-context type must still cast to the host's copy of the interface."); Assert.That(settings.PersisterSpecificSettings, Is.SameAs(targetSettings), "Opening a source must leave the target's settings object exactly where it was."); }); var description = await source.Describe(); - Assert.Multiple(() => - { - Assert.That(AssemblyLoadContext.GetLoadContext(description.GetType().Assembly), Is.SameAs(AssemblyLoadContext.Default), - "A shared type produced inside the plugin context must arrive as the host's own type."); - Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), - "The source read its own RavenDB settings rather than the target's."); - }); + Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + "The source read its own RavenDB settings rather than the target's."); } [Test] diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 5c74f8eb3e..234f9924f5 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -1,4 +1,4 @@ -namespace ServiceControl.Persistence.RavenDB +namespace ServiceControl.Persistence.RavenDB { using System; using System.IO; diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs index 85b8fbd919..859e02c505 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -229,23 +229,6 @@ public async Task Opening_the_source_twice_is_refused() Assert.That(exception.Message, Does.Contain("already open"), "A second Open would abandon the first store, and on the embedded path a running server process with it."); } - [Test] - public async Task An_embedded_open_that_cannot_start_leaves_nothing_behind() - { - sourceSettings.ConnectionString = null; - sourceSettings.DatabasePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("n")); - - var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); - - Assert.CatchAsync(async () => await lifecycle.Open()); - - var afterFailure = Assert.Throws(() => _ = lifecycle.DocumentStore); - - Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed embedded open must leave no store, or the caller cannot tell an unopened source from a half-open one."); - - await lifecycle.DisposeAsync(); - } - [Test] public async Task Generating_an_id_is_refused() { @@ -257,7 +240,7 @@ public async Task Generating_an_id_is_refused() Assert.CatchAsync(async () => await session.StoreAsync(new FailedMessage { UniqueMessageId = "hilo", Status = FailedMessageStatus.Unresolved })); - Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo writes an id range to the source before SaveChanges is reached, so a guard that only sees SaveChanges lets it through."); + Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo reserves an id range with a request of its own before SaveChanges, so only the request hook can refuse it."); } [Test] @@ -270,7 +253,7 @@ public async Task A_patch_against_the_source_is_refused() await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( new PatchOperation("FailedMessages/abc", null, new PatchRequest { Script = "this.Status = 1;" }))); - Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch never goes through a session, so it bypasses OnBeforeStore entirely. This is the write style the RavenDB persister itself uses."); + Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch is a request of its own with no session, so only the request hook can refuse it; the RavenDB persister writes this way."); } [Test] @@ -314,7 +297,7 @@ public async Task Deleting_an_untracked_document_is_refused() Assert.CatchAsync(async () => await session.SaveChangesAsync()); - Assert.That(await CountDocuments(), Is.EqualTo(1), "Delete by id on an untracked document is deferred, so it never raises OnBeforeDelete."); + Assert.That(await CountDocuments(), Is.EqualTo(1), "A delete by id of an untracked document only goes out at SaveChanges, as a batch request the hook must refuse."); } [Test] diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs index 3c098fffdc..92d6a73ce7 100644 --- a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -2,8 +2,7 @@ namespace ServiceControl.Persistence.DataMigration; public static class HaltThreshold { - // Both must be exceeded: the floor protects a tiny category from one bad row, and the - // proportion protects a huge one from grinding through thousands of failures as "only a fraction". + // Both must be exceeded: the floor ignores a few bad rows in a small category, the percentage a small share of a large one. public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) { if (skippedCount <= minimumFloor || totalCount == 0) diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index c9ce5d9fc7..fb9c08c696 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -55,14 +55,15 @@ public async Task RunCategoryAsync(MigrationCategory catego var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); if (predecessor is not { State: MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned }) { + var predecessorState = predecessor?.State.ToString() ?? "not started"; var blocked = checkpoint with { Selected = true, - LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessor?.State.ToString() ?? "not started"}" + LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" }; await checkpointStore.Upsert(blocked, cancellationToken); logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", - category.Id, mustFollowId, predecessor?.State.ToString() ?? "not started"); + category.Id, mustFollowId, predecessorState); return blocked; } } @@ -74,12 +75,12 @@ public async Task RunCategoryAsync(MigrationCategory catego Selected = true, State = MigrationCategoryState.InProgress, StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + CompletedAt = null, LastError = null }; await checkpointStore.Upsert(checkpoint, cancellationToken); } - var batchSize = target.BatchSizeFor(category); var isFirstBatch = true; // Per run, not the persisted totals: the skips that tripped a halt stay on the row, so // counting them again would re-halt a restart whose cause has been fixed. @@ -88,6 +89,7 @@ public async Task RunCategoryAsync(MigrationCategory catego try { + var batchSize = target.BatchSizeFor(category); await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) { if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) @@ -102,13 +104,13 @@ public async Task RunCategoryAsync(MigrationCategory catego var bodySkips = 0; if (category.CarriesBodies) { - var (withBodies, failedIds) = await FetchBodiesWithRetry(category, batch, cancellationToken); + var (withBodies, failed) = await FetchBodiesWithRetry(category, batch, cancellationToken); batchToWrite = withBodies; - bodySkips = failedIds.Count; + bodySkips = failed.Count; - foreach (var id in failedIds) + foreach (var (id, lastAttemptError) in failed) { - logger.LogWarning("Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); + logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); } } @@ -128,7 +130,7 @@ public async Task RunCategoryAsync(MigrationCategory catego { CopiedCount = checkpoint.CopiedCount + result.Copied, SkippedCount = checkpointAfterBatch.SkippedCount + result.Skipped, - AlreadyPresentCount = checkpoint.AlreadyPresentCount + result.AlreadyPresent, + AlreadyPresentCount = checkpointAfterBatch.AlreadyPresentCount + result.AlreadyPresent, SkipReasons = AddSkipReasons(checkpointAfterBatch.SkipReasons, result.SkipReasons) }; @@ -148,50 +150,41 @@ public async Task RunCategoryAsync(MigrationCategory catego if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) { - checkpoint = checkpoint with - { - State = MigrationCategoryState.Halted, - CompletedAt = timeProvider.GetUtcNow().UtcDateTime, - LastError = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows. Fix the cause and restart to resume from the cursor, or abandon the category to accept the loss." - }; - await checkpointStore.Upsert(checkpoint, cancellationToken); - logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, checkpoint.LastError); - return checkpoint; + var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows. Fix the cause and restart to resume from the cursor, or abandon the category to accept the loss."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); } } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - // The host is stopping. The category stays InProgress and the next start resumes from - // the cursor the last committed batch left behind. + // The target stored the last committed batch with every row counted as copied, so save its real + // split. The category stays InProgress and the next start resumes from the cursor. + await checkpointStore.Upsert(checkpoint, CancellationToken.None); throw; } catch (Exception ex) { - checkpoint = checkpoint with - { - State = MigrationCategoryState.Halted, - CompletedAt = timeProvider.GetUtcNow().UtcDateTime, - LastError = $"{ex.GetType().Name} at cursor {checkpoint.Cursor ?? "the start"}: {ex.Message}" - }; - await checkpointStore.Upsert(checkpoint, cancellationToken); + var reason = $"{ex.GetType().Name} at cursor {checkpoint.Cursor ?? "the start"}: {ex.Message}"; logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); - return checkpoint; + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); } - checkpoint = checkpoint with - { - State = checkpoint.SkippedCount > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete, - CompletedAt = timeProvider.GetUtcNow().UtcDateTime - }; - await checkpointStore.Upsert(checkpoint, cancellationToken); - return checkpoint; + return await Settle(checkpoint with { State = checkpoint.SkippedCount > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); } - async Task<(MigrationBatch Batch, IReadOnlyList FailedIds)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) + // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. + async Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) + { + settled = settled with { CompletedAt = timeProvider.GetUtcNow().UtcDateTime }; + await checkpointStore.Upsert(settled, cancellationToken); + return settled; + } + + async Task<(MigrationBatch Batch, IReadOnlyList<(string SourceId, Exception LastAttemptError)> Failed)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) { var survivors = new List(batch.Rows.Count); - var failed = new List(); + var failed = new List<(string SourceId, Exception LastAttemptError)>(); foreach (var row in batch.Rows) { @@ -202,6 +195,7 @@ public async Task RunCategoryAsync(MigrationCategory catego } MigrationBody? body = null; + Exception? lastAttemptError = null; var succeeded = false; for (var attempt = 1; attempt <= MaxBodyReadAttempts && !succeeded; attempt++) @@ -217,8 +211,9 @@ public async Task RunCategoryAsync(MigrationCategory catego // recording the message as permanently unreadable would lose a row to a restart. throw; } - catch (Exception ex) + catch (Exception ex) when (!IsDefect(ex)) { + lastAttemptError = ex; logger.LogWarning(ex, "Attempt {Attempt} to read the body for {SourceId} failed", attempt, row.SourceId); if (attempt < MaxBodyReadAttempts) { @@ -233,13 +228,17 @@ public async Task RunCategoryAsync(MigrationCategory catego } else { - failed.Add(row.SourceId); + failed.Add((row.SourceId, lastAttemptError!)); } } return (batch with { Rows = survivors }, failed); } + // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. + static bool IsDefect(Exception exception) => + exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; + static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) { if (additions is not { Count: > 0 }) diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs index 0b5a6e3ab3..8433836893 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -5,11 +5,7 @@ namespace ServiceControl.Persistence.DataMigration; using System.Linq; using ServiceControl.Configuration; -/// The engine's tuning, read from settings once at startup. -/// How long to wait between batches of optional data, so normal work isn't slowed down. Zero means don't wait. -/// A category halts when more than this percent of the rows handled in this run were skipped, and the minimum below is also passed. -/// A category never halts until more than this many rows were skipped in this run, so a few bad rows can't stop a small category. -/// The optional categories to copy. Required categories are always copied. +/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories to copy. public sealed record MigrationEngineOptions( TimeSpan ThrottlePause, int HaltThresholdPercent, diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs new file mode 100644 index 0000000000..24aee91e82 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; + +public sealed class CapturingLogger : ILogger +{ + public List<(LogLevel Level, string Message, Exception? Exception)> Entries { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + Entries.Add((logLevel, formatter(state, exception), exception)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs index ad411cc65b..bf5d522856 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -12,7 +12,8 @@ namespace ServiceControl.UnitTests.Migration.Fakes; public sealed class InMemoryMigrationSource : IMigrationSource { readonly Dictionary> rowsByCategory = []; - readonly Dictionary>> bodyAttempts = []; + readonly Dictionary bodyFailures = []; + readonly Dictionary bodyReadAttempts = []; readonly Dictionary bodies = []; public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); @@ -21,14 +22,9 @@ public sealed class InMemoryMigrationSource : IMigrationSource public void SetBody(string sourceId, MigrationBody? body) => bodies[sourceId] = body; - public void QueueBodyAttempt(string sourceId, Func attempt) - { - if (!bodyAttempts.TryGetValue(sourceId, out var queue)) - { - bodyAttempts[sourceId] = queue = new Queue>(); - } - queue.Enqueue(attempt); - } + public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); + + public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; @@ -73,9 +69,10 @@ public async IAsyncEnumerable Read( { await Task.Yield(); - if (bodyAttempts.TryGetValue(sourceId, out var queue) && queue.Count > 0) + var attempt = bodyReadAttempts[sourceId] = BodyReadAttempts(sourceId) + 1; + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) { - return queue.Dequeue()(); + throw failures.Failure; } return bodies.GetValueOrDefault(sourceId); diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs index 6d7b2cda2b..23492909d4 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs @@ -63,16 +63,20 @@ public void A_cursor_the_source_never_issued_throws_rather_than_starting_again() } [Test] - public async Task ReadBody_returns_a_queued_attempt_then_falls_back_to_the_seeded_body() + public async Task ReadBody_fails_the_configured_number_of_times_then_returns_the_seeded_body() { var source = new InMemoryMigrationSource(); var finalBody = new MigrationBody(new byte[] { 9 }, "text/plain"); source.SetBody("msg-1", finalBody); - source.QueueBodyAttempt("msg-1", () => throw new System.Exception("transient")); + source.FailBodyReads("msg-1", times: 1, new TimeoutException("transient")); - Assert.ThrowsAsync(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); + Assert.ThrowsAsync(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); var secondAttempt = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1"); - Assert.That(secondAttempt, Is.EqualTo(finalBody)); + using (Assert.EnterMultipleScope()) + { + Assert.That(secondAttempt, Is.EqualTo(finalBody)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(2)); + } } } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index 68bbe3139f..436996e84e 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -14,6 +14,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint readonly Dictionary rejectedKeys = []; public int DefaultBatchSize { get; set; } = 3; + public string NoBatchSizeFor { get; set; } public int? FailOnCallNumber { get; set; } public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } int callCount; @@ -25,7 +26,8 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint public IReadOnlyList WrittenRows(string categoryId) => writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; - public int BatchSizeFor(MigrationCategory category) => DefaultBatchSize; + public int BatchSizeFor(MigrationCategory category) => + category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : DefaultBatchSize; public async Task Write( MigrationCategory category, diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs index 2c662bffbe..b6a21b7f85 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -18,17 +18,6 @@ public void MigrationCategory_carries_no_fact_about_where_a_store_keeps_its_rows "a source database or target table belongs in that store's own adapter, where a different store pair can map it differently"); } - [Test] - public void Abandoned_is_a_state_of_its_own_and_not_a_kind_of_complete() - { - using (Assert.EnterMultipleScope()) - { - Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.Complete)); - Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.CompleteWithErrors)); - Assert.That(MigrationCategoryState.Abandoned, Is.Not.EqualTo(MigrationCategoryState.Halted)); - } - } - [Test] public void Category_states_keep_the_integers_stored_checkpoints_already_hold() { @@ -54,25 +43,7 @@ public void MigrationRow_body_defaults_to_null() } [Test] - public void MigrationRow_can_carry_a_body() - { - var body = new MigrationBody(new byte[] { 1, 2, 3 }, "text/plain"); - var row = new MigrationRow("id-1", new object(), new Dictionary(), body); - - Assert.That(row.Body, Is.EqualTo(body)); - } - - [Test] - public void MigrationBatch_groups_rows_under_one_cursor() - { - var rows = new[] { new MigrationRow("id-1", new object(), new Dictionary()) }; - var batch = new MigrationBatch(rows, Cursor: "id-1"); - - Assert.That(batch.Cursor, Is.EqualTo("id-1")); - } - - [Test] - public void MigrationCheckpoint_starts_with_no_cursor_and_zero_counts() + public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() { var checkpoint = new MigrationCheckpoint( CategoryId: "EndpointSettings", @@ -89,11 +60,6 @@ public void MigrationCheckpoint_starts_with_no_cursor_and_zero_counts() AbandonedAt: null, LastError: null); - using (Assert.EnterMultipleScope()) - { - Assert.That(checkpoint.Cursor, Is.Null); - Assert.That(checkpoint.CopiedCount, Is.Zero); - Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); - } + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs index a2680372f8..cfac8aa11a 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -24,10 +24,7 @@ public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_writ source.Seed(category.Id, Row("msg-1")); var body = new MigrationBody(new byte[] { 1 }, "text/plain"); source.SetBody("msg-1", body); - for (var attempt = 1; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("blip")); - } + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); // Zeroed because FakeTimeProvider is never advanced here: a 200 ms Task.Delay against a clock @@ -42,9 +39,53 @@ public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_writ Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); Assert.That(checkpoint.SkippedCount, Is.Zero); Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + } + } + + [Test] + public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new NotSupportedException("this source cannot read bodies")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain(nameof(NotSupportedException))); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons, Is.Null); } } + [Test] + public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var failure = new TimeoutException("body store unreachable"); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, failure); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failure)); + } + [Test] public async Task A_body_the_source_already_attached_is_written_without_reading_it_again() { @@ -52,10 +93,7 @@ public async Task A_body_the_source_already_attached_is_written_without_reading_ var source = new InMemoryMigrationSource(); var attached = new MigrationBody(new byte[] { 7 }, "text/plain"); source.Seed(category.Id, Row("msg-1") with { Body = attached }); - for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("a body read nobody needed")); - } + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("a body read nobody needed")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; @@ -76,10 +114,7 @@ public async Task Exhausted_attempts_skip_the_whole_message_and_count_toward_the var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("msg-1"), Row("msg-2")); - for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("down")); - } + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); @@ -107,10 +142,7 @@ public async Task A_body_store_outage_across_many_messages_halts_the_category() // Every body read fails every attempt: a down body store, not a per-message fluke. foreach (var id in ids) { - for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt(id, () => throw new InvalidOperationException("body store unreachable")); - } + source.FailBodyReads(id, MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); } var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs index 82d806a947..d48c44dc35 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -53,12 +53,4 @@ public void A_category_removed_from_configuration_leaves_its_checkpoint_row_unto Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun)); } } - - [Test] - public void A_category_added_to_configuration_is_selected_on_the_next_run() - { - var engine = BuildEngine(["CustomChecks"], out _); - - Assert.That(engine.SelectCategories(MigrationCategoryKind.Optional).Select(c => c.Id), Is.EqualTo(new[] { "CustomChecks" })); - } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs index 4c42966148..ba354ffeca 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -6,6 +6,7 @@ namespace ServiceControl.UnitTests.Migration; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using NUnit.Framework; @@ -90,12 +91,21 @@ public async Task A_halted_category_is_re_attempted_on_the_next_run_and_resumes_ var halted = await firstRun.RunCategoryAsync(category); Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + // Stopped on its first write, so the saved row is the restarted one rather than the completed one. target.FailOnCallNumber = null; - var secondRun = BuildEngine(source, checkpointStore, target); - var finished = await secondRun.RunCategoryAsync(category); + using var stopping = new CancellationTokenSource(); + target.StopOnCall = (3, stopping); + Assert.ThrowsAsync(() => BuildEngine(source, checkpointStore, target).RunCategoryAsync(category, stopping.Token)); + var restarted = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var lastRun = BuildEngine(source, checkpointStore, target); + var finished = await lastRun.RunCategoryAsync(category); using (Assert.EnterMultipleScope()) { + Assert.That(restarted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(restarted.CompletedAt, Is.Null, "a copy running again does not keep the finish time its halt recorded"); Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); Assert.That(finished.LastError, Is.Null, "a cleared halt does not leave a stale error on the row"); Assert.That(finished.CopiedCount, Is.EqualTo(4)); @@ -112,10 +122,7 @@ public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_acros var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("msg-1"), Row("msg-2")); // msg-1's body is unreadable on both runs: every attempt fails on each. - for (var attempt = 0; attempt < 2 * MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("body store unreachable")); - } + source.FailBodyReads("msg-1", 2 * MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; @@ -160,4 +167,74 @@ public async Task An_abandoned_category_is_left_exactly_as_it_is() Assert.That(target.WrittenRows(category.Id), Is.Empty); } } + + [Test] + public async Task A_target_with_no_batch_size_for_a_category_halts_it_and_the_next_category_still_runs() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var unmapped = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var next = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(unmapped.Id, Row("k-1")); + source.Seed(next.Id, Row("r-1")); + var target = new InMemoryMigrationTarget(checkpointStore) { NoBatchSizeFor = unmapped.Id }; + var engine = BuildEngine(source, checkpointStore, target); + + var results = await engine.RunCategories([unmapped, next]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].LastError, Does.Contain("No batch size")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public void A_halt_whose_save_fails_still_logs_the_exception_that_caused_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Exception?.Message), Does.Contain("Simulated failure on write 1")); + } + + [Test] + public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", "Rejected"); + var logger = new CapturingLogger(); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); + } + + // The store shares the target's database, which has become unreachable by the time the halt is saved. + sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + checkpoint.State == MigrationCategoryState.Halted ? throw new TimeoutException("checkpoint store unreachable") : saved.Upsert(checkpoint, cancellationToken); + } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index fb114a7409..21e0b59be4 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -72,13 +72,30 @@ public async Task Rows_already_present_in_the_target_never_count_toward_the_halt } [Test] - public void Halted_is_distinct_from_Complete_and_CompleteWithErrors() + public async Task A_restart_after_a_threshold_halt_counts_only_its_own_skips_and_keeps_the_earlier_ones() { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var failingTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + failingTarget.RejectKey($"row-{i}", "Rejected"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var halted = await new MigrationEngine(source, failingTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + // The cause is fixed: the remaining rows now write cleanly. + var fixedTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var finished = await new MigrationEngine(source, fixedTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + using (Assert.EnterMultipleScope()) { - Assert.That(MigrationCategoryState.Halted, Is.Not.EqualTo(MigrationCategoryState.Complete)); - Assert.That(MigrationCategoryState.Halted, Is.Not.EqualTo(MigrationCategoryState.CompleteWithErrors)); - Assert.That(MigrationCategoryState.CompleteWithErrors, Is.Not.EqualTo(MigrationCategoryState.Complete)); + // 120 skips in 600 rows is the first point past both the floor and 5%. + Assert.That((halted.State, halted.CopiedCount, halted.SkippedCount), Is.EqualTo((MigrationCategoryState.Halted, 480L, 120L))); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); + Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); } } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs index eb296bfd71..0ede2f09b3 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -27,8 +27,6 @@ public void Defaults_match_the_contract_when_nothing_is_configured() using (Assert.EnterMultipleScope()) { - // At one second a million-row copy in 50-row batches spends five and a half hours paused - // before it does any work at all. Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs index abf41f8ac9..141864afc8 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -19,7 +19,7 @@ static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigra new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); [Test] - public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes() + public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes_and_says_so_on_its_checkpoint_row() { var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; var source = new InMemoryMigrationSource(); @@ -31,28 +31,11 @@ public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_c // LicensingEndpoints has never run: no checkpoint row for it at all. var checkpoint = await engine.RunCategoryAsync(throughputCategory); + var persisted = await checkpointStore.Read(throughputCategory.Id); using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); - } - } - - [Test] - public async Task A_blocked_category_says_so_on_its_checkpoint_row_instead_of_returning_in_silence() - { - var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; - var source = new InMemoryMigrationSource(); - source.Seed(throughputCategory.Id, Row("t-1")); - var checkpointStore = new InMemoryMigrationCheckpointStore(); - var target = new InMemoryMigrationTarget(checkpointStore); - var engine = BuildEngine(source, checkpointStore, target); - - await engine.RunCategoryAsync(throughputCategory); - - var persisted = await checkpointStore.Read(throughputCategory.Id); - using (Assert.EnterMultipleScope()) - { // A row exists, so status can print it. Without one, an operator cannot tell a category // waiting on another from a category nobody asked for. Assert.That(persisted, Is.Not.Null); @@ -83,14 +66,15 @@ public async Task EndpointSettings_does_not_start_before_KnownEndpoints_complete } } - [Test] - public async Task GroupComments_does_not_start_before_the_archive_completes() + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + public async Task GroupComments_does_not_start_before_the_archive_completes(MigrationCategoryState archiveState) { var comments = MigrationCategoryRegistry.Find("GroupComments")!; var source = new InMemoryMigrationSource(); source.Seed(comments.Id, Row("GroupComment/g-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", true, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", true, archiveState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null, null)); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); @@ -99,34 +83,21 @@ public async Task GroupComments_does_not_start_before_the_archive_completes() using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); Assert.That(target.WrittenRows(comments.Id), Is.Empty); } } - [Test] - public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_Complete() - { - var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; - var source = new InMemoryMigrationSource(); - source.Seed(throughputCategory.Id, Row("t-1")); - var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, MigrationCategoryState.Complete, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); - var target = new InMemoryMigrationTarget(checkpointStore); - var engine = BuildEngine(source, checkpointStore, target); - - var checkpoint = await engine.RunCategoryAsync(throughputCategory); - - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); - } - - [Test] - public async Task LicensingThroughput_proceeds_when_LicensingEndpoints_is_CompleteWithErrors() + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finished_or_abandoned(MigrationCategoryState endpointsState) { var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; var source = new InMemoryMigrationSource(); source.Seed(throughputCategory.Id, Row("t-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, MigrationCategoryState.CompleteWithErrors, "e-1", 9, 1, 10, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs index e0b5fd13ed..fdc62cdd4e 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -57,4 +57,78 @@ public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_an Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates"); } } + + [Test] + public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_batch() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 6).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + // Both in the second batch, the last one to commit before the stop. + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", "Rejected"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + var afterStop = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That((afterStop.CopiedCount, afterStop.SkippedCount, afterStop.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present after the stop"); + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary { ["Rejected"] = 1 })); + } + } + + [Test] + public async Task A_hard_crash_after_a_committed_write_is_accepted_to_leave_that_batch_counted_as_copied_and_can_end_Complete() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 4).Select(i => Row($"row-{i}"))]); + var committed = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(committed) { DefaultBatchSize = 2 }; + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", "Rejected"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var crashingEngine = new MigrationEngine(source, target, new CrashAfterCommitCheckpointStore(committed, crashAfterCursor: "row-4"), new FakeTimeProvider(), options, NullLogger.Instance); + await crashingEngine.RunCategoryAsync(category); + + var restartedEngine = new MigrationEngine(source, target, committed, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await restartedEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // Accepted: closing this needs the target to save the real split with the rows. + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 0L, 0L)), "copied, skipped, already present at the end"); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "row-1", "row-2" })); + } + } + + // Once the target has committed crashAfterCursor, the engine's own saves are lost, as if the process died there. + sealed class CrashAfterCommitCheckpointStore(IMigrationCheckpointStore committed, string crashAfterCursor) : IMigrationCheckpointStore + { + public Task> ReadAll(CancellationToken cancellationToken = default) => committed.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => committed.Read(categoryId, cancellationToken); + + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + if ((await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor != crashAfterCursor) + { + await committed.Upsert(checkpoint, cancellationToken); + } + } + } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs index 8503b3c257..691f0a6f0b 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -24,19 +24,18 @@ public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them( var target = new InMemoryMigrationTarget(checkpointStore); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); - var ordered = engine.SelectCategories(MigrationCategoryKind.Required).Take(2).ToArray(); - foreach (var category in ordered) + // The reverse of registry order, so an engine that re-sorted by Order would run KnownEndpoints first. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("MessageRedirects")!, MigrationCategoryRegistry.Find("KnownEndpoints")!]; + foreach (var category in given) { source.Seed(category.Id, Row($"{category.Id}-1")); } - var results = await engine.RunCategories(ordered); + var results = await engine.RunCategories(given); using (Assert.EnterMultipleScope()) { - // KnownEndpoints then EndpointSettings, the order the registry declares, whichever order - // the caller passed them in. - Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "KnownEndpoints", "EndpointSettings" })); + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "MessageRedirects", "KnownEndpoints" })); Assert.That(results.Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.Complete)); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 1319ae20b3..2d2d0dda9f 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -46,10 +46,7 @@ public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnread var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("msg-1"), Row("msg-2")); - for (var attempt = 0; attempt < MigrationEngine.MaxBodyReadAttempts; attempt++) - { - source.QueueBodyAttempt("msg-1", () => throw new InvalidOperationException("down")); - } + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs index 44bda6a340..c4665da55f 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -3,6 +3,7 @@ namespace ServiceControl.UnitTests.Migration; using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; @@ -23,23 +24,28 @@ public async Task Optional_categories_pause_between_batches_for_the_configured_d source.Seed(category.Id, Row("a"), Row("b"), Row("c")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; - var timeProvider = new FakeTimeProvider(); + var clock = new TimerRecordingTimeProvider(); var pause = TimeSpan.FromSeconds(1); var options = new MigrationEngineOptions(pause, 5, 100, []); - var engine = new MigrationEngine(source, target, checkpointStore, timeProvider, options, NullLogger.Instance); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); var runTask = engine.RunCategoryAsync(category); - await Task.Delay(50); - Assert.That(runTask.IsCompleted, Is.False, "the first inter-batch pause should still be pending"); - timeProvider.Advance(pause); - await Task.Delay(50); - Assert.That(runTask.IsCompleted, Is.False, "the second inter-batch pause should still be pending"); + // Three batches, so a pause before the second and another before the third. + for (var pauseNumber = 1; pauseNumber <= 2; pauseNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"pause {pauseNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"pause {pauseNumber} should still be pending"); + clock.Advance(pause); + } - timeProvider.Advance(pause); var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { pause, pause })); + } } [Test] @@ -59,4 +65,32 @@ public async Task Required_categories_never_pause() Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); } + + // Signals each timer the engine creates, so the test only advances the clock once a pause is waiting on it. + sealed class TimerRecordingTimeProvider : TimeProvider + { + readonly FakeTimeProvider clock = new(); + + public SemaphoreSlim TimerCreated { get; } = new(0); + + public List DueTimes { get; } = []; + + public void Advance(TimeSpan delta) => clock.Advance(delta); + + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + + public override long GetTimestamp() => clock.GetTimestamp(); + + public override long TimestampFrequency => clock.TimestampFrequency; + + public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = clock.CreateTimer(callback, state, dueTime, period); + DueTimes.Add(dueTime); + TimerCreated.Release(); + return timer; + } + } } diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index 0bab996629..bd6400df32 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -66,7 +66,7 @@ public HostArguments(string[] args) { { "migration-source-report", - "Report what a migration would read from the source persistence, without changing it", + "Report what a migration would read from the source persistence", s => Command = typeof(MigrationSourceReportCommand) } }; From 746617cb5d7be27db7dc06ad77d83c419634eaca Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 16 Sep 2026 20:06:30 +0800 Subject: [PATCH 09/28] PR review --- .../IMigrationCheckpointStore.cs | 54 +++++++++++-- .../DataMigration/IMigrationSource.cs | 1 + .../DataMigration/IMigrationTarget.cs | 11 +-- .../MigrationCheckpointConflictException.cs | 6 ++ .../DataMigration/MigrationEngine.cs | 77 ++++++------------- .../DataMigration/MigrationSkipReason.cs | 4 +- .../Fakes/InMemoryMigrationCheckpointStore.cs | 13 +++- .../Fakes/InMemoryMigrationTarget.cs | 18 +++-- .../Fakes/InMemoryMigrationTargetTests.cs | 29 +++---- .../Migration/MigrationContractShapeTests.cs | 7 +- .../MigrationEngineCategorySelectionTests.cs | 4 +- .../Migration/MigrationEngineCopyTests.cs | 6 +- .../MigrationEngineFailurePathTests.cs | 10 +-- .../Migration/MigrationEngineHaltTests.cs | 6 +- .../Migration/MigrationEngineOrderingTests.cs | 13 ++-- .../Migration/MigrationEngineResumeTests.cs | 25 +++--- .../MigrationEngineSkipReasonTests.cs | 21 ++--- 17 files changed, 164 insertions(+), 141 deletions(-) create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs index 5d20b9af13..bc71283d01 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -2,6 +2,7 @@ namespace ServiceControl.Persistence.DataMigration; using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -12,29 +13,68 @@ public enum MigrationCategoryState Complete, CompleteWithErrors, Halted, - Abandoned + Abandoned, + Blocked } /// A category's saved progress: where a restart carries on from, and what the status and verify commands report. public sealed record MigrationCheckpoint( string CategoryId, - bool Selected, MigrationCategoryState State, string? Cursor, long CopiedCount, long SkippedCount, long? SourceTotal, - IReadOnlyDictionary? SkipReasons, + IReadOnlyDictionary? SkipReasons, DateTime? StartedAt, DateTime? LastProgressAt, - DateTime? CompletedAt, - DateTime? AbandonedAt, + // The moment the category stopped running, whatever state it stopped in. Read it beside State: a halt settles too. + DateTime? SettledAt, string? LastError, - long AlreadyPresentCount = 0); + long AlreadyPresentCount = 0, + // The optimistic concurrency token. A store sets it on save and refuses one carrying a value the stored row no longer holds. + long Version = 0) +{ + /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, so the saved counts are the real ones. + public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, IReadOnlyDictionary? skipReasons) + { + var explained = skipReasons?.Values.Sum() ?? 0; + if (explained != skipped) + { + throw new InvalidOperationException($"The target reported {skipped} skipped rows in category {CategoryId} but gave reasons for {explained}. Every skipped row needs a reason, or --migration-verify cannot account for it."); + } + + return this with + { + CopiedCount = CopiedCount + copied, + SkippedCount = SkippedCount + skipped, + AlreadyPresentCount = AlreadyPresentCount + alreadyPresent, + SkipReasons = AddSkipReasons(SkipReasons, skipReasons) + }; + } + + internal static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) + { + if (additions is not { Count: > 0 }) + { + return totals; + } + + Dictionary sum = totals is null ? [] : new(totals); + foreach (var (reason, count) in additions) + { + sum[reason] = sum.GetValueOrDefault(reason) + count; + } + + return sum; + } +} public interface IMigrationCheckpointStore { Task> ReadAll(CancellationToken cancellationToken = default); Task Read(string categoryId, CancellationToken cancellationToken = default); - Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); + + /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. Throws when the stored row has moved on. + Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs index bc5bd1c3bf..bd19e623aa 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -20,6 +20,7 @@ public interface IMigrationSource : IAsyncDisposable Task Count(MigrationCategory category, CancellationToken cancellationToken = default); /// Reads a category in batches, after the checkpoint cursor if provided, or from the start when it is null. Throws on a cursor it never issued. + /// A ceiling, not a target: returning fewer costs nothing, returning more fails the target's write. IAsyncEnumerable Read( MigrationCategory category, string? resumeAfter, diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index 6ab337d815..2d6aeeb84a 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -7,19 +7,20 @@ namespace ServiceControl.Persistence.DataMigration; /// Implemented by a persister that can be the new database a migration copies into. public interface IMigrationTarget { - /// How many rows to read per batch for this category. The target picks it because its own database sets the limits. + /// The most rows a source may return in one batch for this category. The target picks it because its own database sets the limit, and a batch over it fails the write. int BatchSizeFor(MigrationCategory category); - /// Saves the batch's rows and checkpointAfterBatch in one go, so progress never gets ahead of the data. Save the checkpoint exactly as given, without adding counts to it. + /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. Extend checkpointToExtend with this batch's own outcome through and save the result, so what lands is the real split rather than a guess the next save has to correct. + /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Not yet counting anything the target does. Task Write( MigrationCategory category, MigrationBatch batch, - MigrationCheckpoint checkpointAfterBatch, + MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default); /// How many rows the target holds for one category, for progress and verify. Counts only that category, even where two categories share a table. Task Count(MigrationCategory category, CancellationToken cancellationToken = default); } -/// What the target did with one batch. Every skipped row must have a reason in SkipReasons. -public sealed record MigrationWriteResult(int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); +/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs new file mode 100644 index 0000000000..cc2822822f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// Thrown when a checkpoint save carries a version the stored row no longer holds, because another writer moved it on. +public sealed class MigrationCheckpointConflictException(string message, Exception? innerException = null) : Exception(message, innerException); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index fb9c08c696..352097d470 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -58,27 +58,24 @@ public async Task RunCategoryAsync(MigrationCategory catego var predecessorState = predecessor?.State.ToString() ?? "not started"; var blocked = checkpoint with { - Selected = true, + State = MigrationCategoryState.Blocked, LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" }; - await checkpointStore.Upsert(blocked, cancellationToken); logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", category.Id, mustFollowId, predecessorState); - return blocked; + return await checkpointStore.Upsert(blocked, cancellationToken); } } - if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Halted) + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Halted or MigrationCategoryState.Blocked) { - checkpoint = checkpoint with + checkpoint = await checkpointStore.Upsert(checkpoint with { - Selected = true, State = MigrationCategoryState.InProgress, StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, - CompletedAt = null, + SettledAt = null, LastError = null - }; - await checkpointStore.Upsert(checkpoint, cancellationToken); + }, cancellationToken); } var isFirstBatch = true; @@ -114,31 +111,17 @@ public async Task RunCategoryAsync(MigrationCategory catego } } - // Absolute totals counting every handed-over row as copied, persisted verbatim with the rows. - // The real split comes back in the result and lands on the next checkpoint. - var checkpointAfterBatch = checkpoint with + // Prior totals, the new cursor, and the rows this engine already skipped. The target adds its own + // outcome inside the transaction that writes the rows, so nothing provisional is ever stored. + var checkpointToExtend = checkpoint with { Cursor = batch.Cursor, - CopiedCount = checkpoint.CopiedCount + batchToWrite.Rows.Count, SkippedCount = checkpoint.SkippedCount + bodySkips, - SkipReasons = AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [nameof(MigrationSkipReason.BodyUnreadable)] = bodySkips }) + SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [MigrationSkipReason.BodyUnreadable] = bodySkips }) }; - var result = await target.Write(category, batchToWrite, checkpointAfterBatch, cancellationToken); - - checkpoint = checkpointAfterBatch with - { - CopiedCount = checkpoint.CopiedCount + result.Copied, - SkippedCount = checkpointAfterBatch.SkippedCount + result.Skipped, - AlreadyPresentCount = checkpointAfterBatch.AlreadyPresentCount + result.AlreadyPresent, - SkipReasons = AddSkipReasons(checkpointAfterBatch.SkipReasons, result.SkipReasons) - }; - - var explainedSkips = result.SkipReasons?.Values.Sum() ?? 0; - if (explainedSkips != result.Skipped) - { - throw new InvalidOperationException($"The target reported {result.Skipped} skipped rows in category {category.Id} but gave reasons for {explainedSkips}. Every skipped row needs a reason, or --migration-verify cannot account for it."); - } + var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); + checkpoint = result.Saved; foreach (var id in result.SkippedIds) { @@ -156,11 +139,15 @@ public async Task RunCategoryAsync(MigrationCategory catego } } } + // A shutdown is not a halt, and there is nothing to reconcile: the last committed batch stored its + // real split with its own rows, so the row on disk is already correct and resumable. catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - // The target stored the last committed batch with every row counted as copied, so save its real - // split. The category stays InProgress and the next start resumes from the cursor. - await checkpointStore.Upsert(checkpoint, CancellationToken.None); + throw; + } + // Another writer holds this row, which no amount of halting resolves. Leave the state alone so their row stands. + catch (MigrationCheckpointConflictException) + { throw; } catch (Exception ex) @@ -174,12 +161,8 @@ public async Task RunCategoryAsync(MigrationCategory catego } // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. - async Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) - { - settled = settled with { CompletedAt = timeProvider.GetUtcNow().UtcDateTime }; - await checkpointStore.Upsert(settled, cancellationToken); - return settled; - } + Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => + checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); async Task<(MigrationBatch Batch, IReadOnlyList<(string SourceId, Exception LastAttemptError)> Failed)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) { @@ -239,27 +222,11 @@ async Task Settle(MigrationCheckpoint settled, Cancellation static bool IsDefect(Exception exception) => exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; - static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) - { - if (additions is not { Count: > 0 }) - { - return totals; - } - - Dictionary sum = totals is null ? [] : new(totals); - foreach (var (reason, count) in additions) - { - sum[reason] = sum.GetValueOrDefault(reason) + count; - } - - return sum; - } - // A configured pause of zero means "do not throttle", and a timer that is never going to be // waited on is worse than no timer: against a fake clock nobody advances, it never completes. Task Pause(TimeSpan duration, CancellationToken cancellationToken) => duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); static MigrationCheckpoint NotStarted(MigrationCategory category) => - new(category.Id, Selected: false, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null, null); + new(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null); } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs index c4f442305f..b79be632a8 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -2,5 +2,7 @@ namespace ServiceControl.Persistence.DataMigration; public enum MigrationSkipReason { - BodyUnreadable + BodyUnreadable, + // Never written by a copier. A database a newer build wrote still reads rather than throwing where the host decides whether to start. + Unknown } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs index ea6803524f..78edd726b6 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs @@ -18,9 +18,16 @@ public Task> ReadAll(CancellationToken cancel public Task Read(string categoryId, CancellationToken cancellationToken = default) => Task.FromResult(checkpoints.TryGetValue(categoryId, out var checkpoint) ? checkpoint : null); - public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) { - checkpoints[checkpoint.CategoryId] = checkpoint; - return Task.CompletedTask; + var storedVersion = checkpoints.TryGetValue(checkpoint.CategoryId, out var stored) ? stored.Version : 0; + if (storedVersion != checkpoint.Version) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row is at version {storedVersion}."); + } + + var saved = checkpoint with { Version = checkpoint.Version + 1 }; + checkpoints[checkpoint.CategoryId] = saved; + return Task.FromResult(saved); } } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index 436996e84e..38f005d1b1 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -11,7 +11,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint readonly Dictionary> writtenKeysByCategory = []; readonly Dictionary> writtenRowsByCategory = []; readonly HashSet preExistingKeys = []; - readonly Dictionary rejectedKeys = []; + readonly Dictionary rejectedKeys = []; public int DefaultBatchSize { get; set; } = 3; public string NoBatchSizeFor { get; set; } @@ -21,7 +21,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); - public void RejectKey(string sourceId, string reason) => rejectedKeys[sourceId] = reason; + public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; public IReadOnlyList WrittenRows(string categoryId) => writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; @@ -32,7 +32,7 @@ public int BatchSizeFor(MigrationCategory category) => public async Task Write( MigrationCategory category, MigrationBatch batch, - MigrationCheckpoint checkpointAfterBatch, + MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) { callCount++; @@ -54,7 +54,7 @@ public async Task Write( var copied = 0; var alreadyPresent = 0; var skippedIds = new List(); - var skipReasons = new Dictionary(); + var skipReasons = new Dictionary(); foreach (var row in batch.Rows) { @@ -75,11 +75,13 @@ public async Task Write( copied++; } - // Verbatim and in the same operation as the rows, as the real targets persist it. Adding the - // batch's own counts here would double every number the engine already included. - await checkpointStore.Upsert(checkpointAfterBatch, cancellationToken); + // Extended and saved in the same operation as the rows, as the real targets do, so what lands + // is this batch's real split rather than a provisional one the next save has to correct. + var saved = await checkpointStore.Upsert( + checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), + cancellationToken); - return new MigrationWriteResult(copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs index b10a340257..ae52e815dc 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs @@ -12,53 +12,54 @@ class InMemoryMigrationTargetTests static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); static MigrationCheckpoint EmptyCheckpoint(string categoryId) => - new(categoryId, Selected: true, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null, null); + new(categoryId, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null); [Test] - public async Task Writes_new_rows_and_persists_the_checkpoint_it_was_handed_exactly_as_given() + public async Task Writes_new_rows_and_commits_the_extended_checkpoint_with_them() { var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var category = MigrationCategoryRegistry.Find("EndpointSettings")!; var batch = new MigrationBatch([Row("a"), Row("b")], Cursor: "b"); - // Absolute post-batch totals, computed by the caller. The target does no arithmetic on them. - var checkpointAfterBatch = EmptyCheckpoint(category.Id) with { Cursor = "b", CopiedCount = 2 }; + // Prior totals and the new cursor. The target adds this batch's own outcome before it saves. + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "b" }; - var result = await target.Write(category, batch, checkpointAfterBatch); + var result = await target.Write(category, batch, checkpointToExtend); using (Assert.EnterMultipleScope()) { Assert.That(result.Copied, Is.EqualTo(2)); Assert.That(result.Skipped, Is.Zero); Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(2)); - Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointAfterBatch)); + Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 })); + Assert.That(result.Saved, Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 }), "the result carries the row as stored"); } } [Test] - public async Task The_persisted_checkpoint_is_not_adjusted_by_what_the_write_actually_did() + public async Task The_committed_checkpoint_carries_the_real_split_rather_than_the_rows_handed_over() { - // The engine's totals already count every row as copied, so a target that corrected them from - // its own result would double the counts. + // Three rows in, one copied: a checkpoint saying three would survive a crash as three. var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; target.SeedExistingKey("already-present"); - target.RejectKey("rejected", "Rejected"); + target.RejectKey("rejected", MigrationSkipReason.BodyUnreadable); var batch = new MigrationBatch([Row("already-present"), Row("rejected"), Row("new-row")], Cursor: "new-row"); - var checkpointAfterBatch = EmptyCheckpoint(category.Id) with { Cursor = "new-row", CopiedCount = 3 }; + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "new-row" }; - var result = await target.Write(category, batch, checkpointAfterBatch); + var result = await target.Write(category, batch, checkpointToExtend); using (Assert.EnterMultipleScope()) { Assert.That(result.Copied, Is.EqualTo(1)); Assert.That(result.Skipped, Is.EqualTo(1)); Assert.That(result.SkippedIds, Is.EqualTo(new[] { "rejected" })); - Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary { ["Rejected"] = 1 })); + Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); Assert.That(result.AlreadyPresent, Is.EqualTo(1)); Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(1)); - Assert.That((await checkpointStore.Read(category.Id))!.CopiedCount, Is.EqualTo(3)); + var stored = (await checkpointStore.Read(category.Id))!; + Assert.That((stored.CopiedCount, stored.SkippedCount, stored.AlreadyPresentCount), Is.EqualTo((1L, 1L, 1L)), "copied, skipped, already present as committed"); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs index b6a21b7f85..3df3d4e123 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -30,7 +30,8 @@ public void Category_states_keep_the_integers_stored_checkpoints_already_hold() ["Complete"] = 2, ["CompleteWithErrors"] = 3, ["Halted"] = 4, - ["Abandoned"] = 5 + ["Abandoned"] = 5, + ["Blocked"] = 6 }), "checkpoints store State as an integer, so a reordered or inserted member silently changes what every saved checkpoint means"); } @@ -47,7 +48,6 @@ public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() { var checkpoint = new MigrationCheckpoint( CategoryId: "EndpointSettings", - Selected: false, State: MigrationCategoryState.NotStarted, Cursor: null, CopiedCount: 0, @@ -56,8 +56,7 @@ public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() SkipReasons: null, StartedAt: null, LastProgressAt: null, - CompletedAt: null, - AbandonedAt: null, + SettledAt: null, LastError: null); Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs index d48c44dc35..b153258775 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -42,7 +42,7 @@ public void Only_configured_optional_categories_are_selected() public void A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() { var engine = BuildEngine([], out var checkpointStore); - var previousRun = new MigrationCheckpoint("EventLog", Selected: true, MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null); + var previousRun = new MigrationCheckpoint("EventLog", MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); checkpointStore.Upsert(previousRun).GetAwaiter().GetResult(); var selected = engine.SelectCategories(MigrationCategoryKind.Optional); @@ -50,7 +50,7 @@ public void A_category_removed_from_configuration_leaves_its_checkpoint_row_unto using (Assert.EnterMultipleScope()) { Assert.That(selected.Select(c => c.Id), Does.Not.Contain("EventLog")); - Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun)); + Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun with { Version = 1 })); } } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs index c48c726e82..c1a663d1f9 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -35,7 +35,7 @@ public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skippe Assert.That(checkpoint.SkippedCount, Is.Zero); Assert.That(checkpoint.Cursor, Is.EqualTo("c")); Assert.That(checkpoint.StartedAt, Is.Not.Null); - Assert.That(checkpoint.CompletedAt, Is.Not.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(3)); } } @@ -47,7 +47,7 @@ public async Task A_category_already_Complete_is_left_alone_on_a_second_run() var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("a")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - var alreadyDone = new MigrationCheckpoint(category.Id, true, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null); + var alreadyDone = new MigrationCheckpoint(category.Id, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); await checkpointStore.Upsert(alreadyDone); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); @@ -57,7 +57,7 @@ public async Task A_category_already_Complete_is_left_alone_on_a_second_run() using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint, Is.EqualTo(alreadyDone)); + Assert.That(checkpoint, Is.EqualTo(alreadyDone with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); Assert.That(target.WrittenRows(category.Id), Is.Empty); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs index ba354ffeca..18d109dd44 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -105,7 +105,7 @@ public async Task A_halted_category_is_re_attempted_on_the_next_run_and_resumes_ using (Assert.EnterMultipleScope()) { Assert.That(restarted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); - Assert.That(restarted.CompletedAt, Is.Null, "a copy running again does not keep the finish time its halt recorded"); + Assert.That(restarted.SettledAt, Is.Null, "a copy running again does not keep the time its halt settled at"); Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); Assert.That(finished.LastError, Is.Null, "a cleared halt does not leave a stale error on the row"); Assert.That(finished.CopiedCount, Is.EqualTo(4)); @@ -154,7 +154,7 @@ public async Task An_abandoned_category_is_left_exactly_as_it_is() var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("a")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - var abandoned = new MigrationCheckpoint(category.Id, true, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, null, DateTime.UtcNow, "Halted: the body store was unreachable"); + var abandoned = new MigrationCheckpoint(category.Id, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the body store was unreachable"); await checkpointStore.Upsert(abandoned); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); @@ -163,7 +163,7 @@ public async Task An_abandoned_category_is_left_exactly_as_it_is() using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint, Is.EqualTo(abandoned)); + Assert.That(checkpoint, Is.EqualTo(abandoned with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); Assert.That(target.WrittenRows(category.Id), Is.Empty); } } @@ -214,7 +214,7 @@ public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() source.Seed(category.Id, Row("a")); var checkpointStore = new HaltSaveFailsCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); - target.RejectKey("a", "Rejected"); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); var logger = new CapturingLogger(); // A floor of zero lets the one rejected row halt the category. var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); @@ -234,7 +234,7 @@ sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); - public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => checkpoint.State == MigrationCategoryState.Halted ? throw new TimeoutException("checkpoint store unreachable") : saved.Upsert(checkpoint, cancellationToken); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index 21e0b59be4..ba24b8dfaf 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -29,7 +29,7 @@ public async Task A_systemic_failure_halts_the_category_partway_through() var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) { - target.RejectKey($"row-{i}", "Rejected"); + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); } var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -40,7 +40,7 @@ public async Task A_systemic_failure_halts_the_category_partway_through() { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); Assert.That(checkpoint.LastError, Does.Contain("Halted")); - Assert.That(checkpoint.CompletedAt, Is.Not.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); // Stopped partway: the 1,000th row was never reached. Assert.That(target.WrittenRows(category.Id).Count, Is.LessThan(800)); } @@ -81,7 +81,7 @@ public async Task A_restart_after_a_threshold_halt_counts_only_its_own_skips_and var failingTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) { - failingTarget.RejectKey($"row-{i}", "Rejected"); + failingTarget.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); } var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); var halted = await new MigrationEngine(source, failingTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs index 141864afc8..67c1e3b53b 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -34,13 +34,12 @@ public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_c var persisted = await checkpointStore.Read(throughputCategory.Id); using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); // A row exists, so status can print it. Without one, an operator cannot tell a category // waiting on another from a category nobody asked for. Assert.That(persisted, Is.Not.Null); - Assert.That(persisted!.Selected, Is.True); - Assert.That(persisted.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Blocked)); Assert.That(persisted.LastError, Does.Contain("LicensingEndpoints")); } } @@ -61,7 +60,7 @@ public async Task EndpointSettings_does_not_start_before_KnownEndpoints_complete using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); Assert.That(target.WrittenRows(settingsCategory.Id), Is.Empty); } } @@ -74,7 +73,7 @@ public async Task GroupComments_does_not_start_before_the_archive_completes(Migr var source = new InMemoryMigrationSource(); source.Seed(comments.Id, Row("GroupComment/g-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", true, archiveState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); @@ -82,7 +81,7 @@ public async Task GroupComments_does_not_start_before_the_archive_completes(Migr using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); Assert.That(target.WrittenRows(comments.Id), Is.Empty); } @@ -97,7 +96,7 @@ public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finish var source = new InMemoryMigrationSource(); source.Seed(throughputCategory.Id, Row("t-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", true, endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs index fdc62cdd4e..ff7246d3bd 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -69,7 +69,7 @@ public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_bat var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; // Both in the second batch, the last one to commit before the stop. target.SeedExistingKey("row-3"); - target.RejectKey("row-4", "Rejected"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -86,12 +86,12 @@ public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_bat Assert.That((afterStop.CopiedCount, afterStop.SkippedCount, afterStop.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present after the stop"); Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 1L, 1L)), "copied, skipped, already present at the end"); - Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary { ["Rejected"] = 1 })); + Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); } } [Test] - public async Task A_hard_crash_after_a_committed_write_is_accepted_to_leave_that_batch_counted_as_copied_and_can_end_Complete() + public async Task A_hard_crash_after_a_committed_write_loses_nothing_because_the_target_saved_the_real_split() { var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); @@ -99,7 +99,7 @@ public async Task A_hard_crash_after_a_committed_write_is_accepted_to_leave_that var committed = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(committed) { DefaultBatchSize = 2 }; target.SeedExistingKey("row-3"); - target.RejectKey("row-4", "Rejected"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); var crashingEngine = new MigrationEngine(source, target, new CrashAfterCommitCheckpointStore(committed, crashAfterCursor: "row-4"), new FakeTimeProvider(), options, NullLogger.Instance); await crashingEngine.RunCategoryAsync(category); @@ -109,9 +109,9 @@ public async Task A_hard_crash_after_a_committed_write_is_accepted_to_leave_that using (Assert.EnterMultipleScope()) { - // Accepted: closing this needs the target to save the real split with the rows. - Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); - Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 0L, 0L)), "copied, skipped, already present at the end"); + // The engine's own settle was lost, but every count came from the target's own transaction. + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present at the end"); Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "row-1", "row-2" })); } } @@ -123,12 +123,9 @@ sealed class CrashAfterCommitCheckpointStore(IMigrationCheckpointStore committed public Task Read(string categoryId, CancellationToken cancellationToken = default) => committed.Read(categoryId, cancellationToken); - public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) - { - if ((await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor != crashAfterCursor) - { - await committed.Upsert(checkpoint, cancellationToken); - } - } + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + (await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor == crashAfterCursor + ? checkpoint + : await committed.Upsert(checkpoint, cancellationToken); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 2d2d0dda9f..618edd403a 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -24,9 +24,10 @@ public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkp source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; - target.RejectKey("a", "KeyTooLong"); - target.RejectKey("b", "Unparseable"); - target.RejectKey("d", "KeyTooLong"); + // One reason exists, so this pins the total rather than the split between reasons. + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + target.RejectKey("b", MigrationSkipReason.BodyUnreadable); + target.RejectKey("d", MigrationSkipReason.BodyUnreadable); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -35,8 +36,8 @@ public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkp using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); - Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { ["KeyTooLong"] = 2, ["Unparseable"] = 1 })); - Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { ["KeyTooLong"] = 2, ["Unparseable"] = 1 })); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); } } @@ -54,7 +55,7 @@ public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnread var checkpoint = await engine.RunCategoryAsync(category); - Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [nameof(MigrationSkipReason.BodyUnreadable)] = 1 })); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); } [Test] @@ -74,7 +75,7 @@ public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_t { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); Assert.That(checkpoint.LastError, Does.Contain("reported 1 skipped").And.Contain("reasons for 0")); - Assert.That(checkpoint.Cursor, Is.EqualTo("a"), "the target committed the batch, so a restart must carry on after it"); + Assert.That(checkpoint.Cursor, Is.Null, "the target refused ahead of its commit, so nothing was written and a restart reads the batch again"); } } @@ -82,10 +83,10 @@ sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : { public int BatchSizeFor(MigrationCategory category) => 10; - public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointAfterBatch, CancellationToken cancellationToken = default) + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) { - await checkpointStore.Upsert(checkpointAfterBatch, cancellationToken); - return new MigrationWriteResult(0, batch.Rows.Count, []); + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, null), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, []); } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); From 7c88e2df0b6ebca80d5a1773d2b52f7a74980bec Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 17 Sep 2026 10:16:01 +0800 Subject: [PATCH 10/28] Enhance migration engine to handle benign skips and update related tests - Introduced `BenignSkipped` property in `MigrationWriteResult` to track rows that the target would have deleted anyway. - Updated `MigrationEngine` logic to ensure benign skips do not count towards the halt threshold. - Added tests to verify that benign skips are handled correctly and do not cause category halts. - Updated skip reasons to include `PastRetention` for better clarity on skipped rows. - Created approval files for category selection tests to ensure correct order and configuration. --- .../ravendb-to-sql-migration-overview.md | 103 ++++++++++++++++-- .../DataMigration/IMigrationTarget.cs | 3 +- .../DataMigration/MigrationEngine.cs | 9 +- .../DataMigration/MigrationSkipReason.cs | 1 + ...ategory_runs_in_a_fixed_order.approved.txt | 18 +++ ...ional_categories_are_selected.approved.txt | 6 + .../Fakes/InMemoryMigrationTarget.cs | 15 ++- .../MigrationEngineCategorySelectionTests.cs | 34 +++++- .../Migration/MigrationEngineHaltTests.cs | 27 +++++ .../MigrationEngineSkipReasonTests.cs | 34 ++++++ 10 files changed, 226 insertions(+), 24 deletions(-) create mode 100644 src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt create mode 100644 src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 3f04217729..de2c322b14 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -77,14 +77,14 @@ The copier runs inside the ServiceControl host, so every row and every message b 6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. 7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. 8. ServiceControl opens, and whatever [optional data](#optional) they asked for is copied in the background while the instance runs normally. They can watch it from ServicePulse custom checks and events, but not steer it. -9. They run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. +9. They run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. It tolerates more rows in SQL than in RavenDB, because RavenDB keeps expiring rows the copier already took. 10. RavenDB data can be removed. - If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. - A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. - An explicit override exists for a customer who has changed their mind and accepts leaving data behind. It marks the outstanding categories as abandoned, which is a deliberate end state rather than a failure, so the progress check settles and the guard stays armed for any later migration. - **Steps 5 to 7 are the abort window**, which is not the override above: see [the one point you can go back](#the-one-point-you-can-go-back). -- A category that stops because too many rows failed is *halted*, and restarting will halt it again. Fix the cause and restart to resume it, or abandon it deliberately if you accept the loss. +- A category that stops because too many rows failed is *halted*, and it stays that way until someone acts: fix the cause and restart to carry on from where it stopped, or abandon it deliberately if you accept the loss. See [a halt stops one category, and clearing it is a restart](#a-halt-stops-one-category-and-clearing-it-is-a-restart). ## Architecture @@ -172,7 +172,7 @@ flowchart TB ### Not migrated -- The fifteen RavenDB index definitions, which map to a much smaller set of ordinary SQL indexes, and two of which are dead already +- The RavenDB index definitions - The transient in-flight collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations` - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works - `ConnectedApplications`, which only versions 6.0 and 6.1 wrote and nothing has read since @@ -201,11 +201,13 @@ flowchart TB - **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. - **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. +**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). Everything in the [required](#required) set is therefore safe, since unresolved and retry-issued messages have their expiry removed when the retry is issued, so only the archived and resolved messages category and the event log category can shrink underneath the copier, and both copy in the background where the window is longest. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere: the counts are of rows the source actually handed over, and there is no expected total to fall short of. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. + **A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. ## The one point you can go back -While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. Back up both RavenDB databases, or disable expiration on them, before you start. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. +While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. @@ -222,7 +224,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - `UniqueMessageId` keeps its value, but converts type: the source holds a string and the target column is a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. - `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages have no `@expires`, so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. - Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. -- Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. +- Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. Throughput is a required category, so it copies while ServiceControl is closed, before any collector has written to SQL. Setting is what makes the category safe to resume after a crash, where adding would double-count. Copying the rows is also what stops the audit and broker collectors re-gathering the same days when the host opens, because `LastCollectedDate` is derived from the newest throughput row rather than stored (`LicensingDataStore.cs:45`). The checkpoint is what stops a second pass overwriting days the collectors have written since. - Identifiers narrow on the way across, and the dry run counts every kind. What narrows, merges or cannot be stored at all is in [what does not come across](#what-does-not-come-across). ## Batching and throttling @@ -232,12 +234,50 @@ That window closes the moment ServiceControl opens. From then on new failed mess ## Checkpointing and resume -- The checkpoint table is a new table on the target, created by `--setup` along with the rest of the schema. -- It holds one row per category: the selection that row ran with, the state, the resume cursor, copied and skipped counts, timings and the last error. -- Only the copier writes to it. The status command reads it. -- Rows and the resume cursor commit in one transaction. -- If a message is in both databases the SQL row wins and the copier skips it, so every category is safe to run twice. -- Each category has its own cursor, so a half-copied category picks up where it stopped. +A copy that runs for hours will be interrupted at some point: a restart, a dropped connection, a machine reboot. The checkpoint is what makes an interruption cost only the batch that was in flight. It is one row per category, kept on the target and created by `--setup` along with the rest of the schema, and it is written in the same database transaction as the rows it describes. Only the copier writes to it; the status and verify commands read it. + +**What one row holds:** the category it tracks, its state, the resume cursor, how many rows were copied, skipped and already present, a count per skip reason, how many rows the source held when the category started, when it started, when it last made progress, when it settled, the last error, and a version number used to spot a second writer. + +**The states, and which ones a restart re-enters.** `Complete`, `CompleteWithErrors` and `Abandoned` are terminal, so a restart passes straight over the category. `Halted` and `Blocked` are not: a halt is resumed from its cursor once the cause is fixed, and a block clears itself once the category it waits on settles, which is how group comments end up behind archived messages. `NotStarted` and `InProgress` both mean there is work to do. + +### One batch, and why nothing provisional is ever saved + +```mermaid +sequenceDiagram + participant E as Migration engine + participant S as RavenDB source + participant T as SQL target + participant C as Checkpoint row + + E->>C: Read this category's row + C-->>E: State, cursor, totals so far + loop One batch at a time + E->>S: Read the next batch after the cursor + S-->>E: Rows, and the cursor they end at + opt The category carries message bodies + E->>S: Read each body, up to three attempts + S-->>E: The bodies, and which ones could not be read + end + E->>T: Write the rows, with the totals so far,
the unreadable bodies and the new cursor + Note over T,C: One transaction. The rows, the target's own skips,
the new totals and the cursor all commit, or none of them do + T-->>E: The checkpoint exactly as it committed + E->>E: Halt if too much of this run was skipped + end + E->>C: Settle as complete, complete with errors, or halted +``` + +The thing to read twice is that the counts never travel back through the engine to be saved on some later write. The engine hands the target the totals so far, the target adds its own outcome to them and saves the result beside the rows, and the engine then keeps whatever committed. So there is no window in which the stored row claims rows that are not there, and a crash at any instant leaves counts and cursor that both describe exactly the rows in SQL. + +**What that buys, and why each part is needed:** + +- **Progress never gets ahead of the data.** The rows and the cursor commit together, so a restart cannot skip past rows that were never written. +- **A crash costs the batch in flight and nothing else.** The next run reads from the committed cursor. +- **Re-reading a batch cannot double-count it.** Unreadable bodies stay off the checkpoint until the write commits, so a batch that is read twice is counted once, and rows the earlier attempt did write come back as *already present* rather than as fresh copies. +- **Every skipped row has a reason, or the save is refused.** The checkpoint rejects a batch reporting more skips than it explains, because verification has to account for each one rather than report a healthy migration as broken. +- **Each category resumes independently**, so a half-copied category picks up where it stopped while its neighbours are untouched. +- **The halt counters are per run and deliberately not stored.** If the skips that tripped a halt stayed on the row, a restart with the cause fixed would re-trip it on its first batch. +- **A second writer is caught rather than merged.** Each save carries the version it read, and a save against a row that has moved on is refused, so two hosts pointed at one target cannot quietly interleave their progress. +- **If a message is already in SQL the SQL row wins and the copier skips it**, which is what makes every category safe to run twice. ## Error handling @@ -246,9 +286,48 @@ That window closes the moment ServiceControl opens. From then on new failed mess - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. - The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. -- Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. +- Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. +- The percentage is measured against what the run has processed so far rather than against the category's total, so a run that starts badly looks worse than it is. The floor is what keeps that harmless, since fewer than 101 skipped rows never consults the percentage at all. More than that, bunched at the start, does halt a category whose overall rate would have been fine, and the cost is one restart: the skipped rows commit with the cursor, so the next run resumes past them with its counters back at zero. +- A source therefore must not read a category in an order that puts the rows most likely to be skipped at the front of it. - Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. +### A halt stops one category, and clearing it is a restart + +A halt is the copy refusing to keep going on one category because something is wrong beyond the odd bad row. It is not a crash and not data loss: everything already copied is committed, the cursor points at the row after the last one that committed, and the reason is written on the category. Nothing is retried in the background and nothing waits for a timer. The category sits halted until a person does something about it. + +```mermaid +stateDiagram-v2 + [*] --> NotStarted: nothing has run yet + NotStarted --> InProgress: the host starts with MigrationMode = true + NotStarted --> Blocked: the category it must follow has not settled + Blocked --> InProgress: that category settles, then the next restart + InProgress --> InProgress: the host was stopped mid-copy,
so the next start resumes from the cursor + InProgress --> Complete: every row reached, none skipped + InProgress --> CompleteWithErrors: every row reached, some skipped + InProgress --> Halted: too many rows skipped in this run,
or the copy hit an error it did not expect + Halted --> InProgress: fix the cause, restart,
carry on from the cursor + Halted --> Abandoned: accept the loss, deliberately + InProgress --> Abandoned: accept the loss, deliberately + Complete --> [*] + CompleteWithErrors --> [*] + Abandoned --> [*] +``` + +**Two things halt a category.** Either the skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental, or the copy hits an error it did not expect, in which case the error type and the cursor it stopped at are recorded. A host being shut down is neither: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. + +**A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the archived messages they belong to. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. + +**What it costs depends on which category halted.** A halted optional category means the instance keeps serving traffic and that one slice of history is missing until it is resumed. A halted required category means the host stays closed, so the outage carries on until the halt is cleared or the category is abandoned. That is deliberate: opening the host is the point of no return, and it should not happen with required data left behind by accident. + +**Clearing it:** + +1. Read the reason on the category, in the custom check or the status command. It names the count that tripped the threshold, or the error, and the cursor either way. +2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or the target down, or the disk full. +3. Restart the host with `MigrationMode=true`. The category picks up at its cursor, its run counters start again at zero, and the skips already recorded stay on the row so the totals still add up at the end. +4. Repeat only if it halts again. A restart that halts at the same point is telling you the cause is still there, and a restart that gets further has made real progress, because the rows it skipped are committed and will not be read again. + +**Or abandon it, on purpose.** Abandoning marks the category as deliberately given up rather than failed, which lets the host open and lets the migration end. It is the right answer when the data is not worth the outage, and the wrong one if it was picked by accident, because nothing goes back for an abandoned category afterwards. What it leaves behind is stated in the counts rather than guessed at. + ## Dry run Runnable before anything starts, and again later against whatever is still outstanding. It never writes to RavenDB. diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index 2d6aeeb84a..78026fc571 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -23,4 +23,5 @@ Task Write( } /// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. -public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); +/// How many of Skipped the target would have deleted anyway, such as a row already past retention. Counted and reported like any skip, but never counted toward the halt threshold. +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null, int BenignSkipped = 0); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index 352097d470..c64ac05f2d 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -128,8 +128,15 @@ public async Task RunCategoryAsync(MigrationCategory catego logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); } + // A negative fault count would silently disarm the halt threshold for the rest of the run. + if (result.BenignSkipped > result.Skipped) + { + throw new InvalidOperationException($"The target reported {result.BenignSkipped} benign skips in category {category.Id} out of {result.Skipped} skipped rows. Benign skips are a subset of the skipped rows."); + } + processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; - skippedThisRun += bodySkips + result.Skipped; + // Rows the target would have deleted anyway are not faults, so they never halt a category. + skippedThisRun += bodySkips + result.Skipped - result.BenignSkipped; if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) { diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs index b79be632a8..4129eb939d 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -3,6 +3,7 @@ namespace ServiceControl.Persistence.DataMigration; public enum MigrationSkipReason { BodyUnreadable, + PastRetention, // Never written by a copier. A database a newer build wrote still reads rather than throwing where the host decides whether to start. Unknown } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt new file mode 100644 index 0000000000..b77ea84b2b --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -0,0 +1,18 @@ +Required 1: KnownEndpoints +Required 2: EndpointSettings, after KnownEndpoints +Required 3: MessageRedirects +Required 4: Subscriptions +Required 5: NotificationSettings +Required 6: TrialEndDate +Required 7: RetryOperations +Required 8: LicensingEndpoints +Required 9: LicensingThroughput, after LicensingEndpoints +Required 10: LicensingReportMasks +Required 11: LicensedEndpointDetails +Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Optional 1: EventLog +Optional 2: CustomChecks +Optional 3: FailedErrorImports, with bodies +Optional 4: FailedMessageEdits +Optional 5: ArchivedAndResolvedFailedMessages, with bodies +Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt new file mode 100644 index 0000000000..6c96bee11f --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -0,0 +1,6 @@ +Configured: GroupComments, EventLog, ArchivedAndResolvedFailedMessages +Runs as: +Optional 1: EventLog +Optional 5: ArchivedAndResolvedFailedMessages, with bodies +Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages +Never copied: CustomChecks, FailedErrorImports, FailedMessageEdits \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index 38f005d1b1..4e50306469 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -11,7 +11,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint readonly Dictionary> writtenKeysByCategory = []; readonly Dictionary> writtenRowsByCategory = []; readonly HashSet preExistingKeys = []; - readonly Dictionary rejectedKeys = []; + readonly Dictionary rejectedKeys = []; public int DefaultBatchSize { get; set; } = 3; public string NoBatchSizeFor { get; set; } @@ -21,7 +21,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); - public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; + public void RejectKey(string sourceId, MigrationSkipReason reason, bool benign = false) => rejectedKeys[sourceId] = (reason, benign); public IReadOnlyList WrittenRows(string categoryId) => writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; @@ -53,15 +53,20 @@ public async Task Write( var copied = 0; var alreadyPresent = 0; + var benignSkipped = 0; var skippedIds = new List(); var skipReasons = new Dictionary(); foreach (var row in batch.Rows) { - if (rejectedKeys.TryGetValue(row.SourceId, out var reason)) + if (rejectedKeys.TryGetValue(row.SourceId, out var rejection)) { skippedIds.Add(row.SourceId); - skipReasons[reason] = skipReasons.GetValueOrDefault(reason) + 1; + skipReasons[rejection.Reason] = skipReasons.GetValueOrDefault(rejection.Reason) + 1; + if (rejection.Benign) + { + benignSkipped++; + } continue; } @@ -81,7 +86,7 @@ public async Task Write( checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), cancellationToken); - return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons, benignSkipped); } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs index b153258775..9c74a7b605 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -6,6 +6,7 @@ namespace ServiceControl.UnitTests.Migration; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using NUnit.Framework; +using Particular.Approvals; using ServiceControl.Persistence.DataMigration; using ServiceControl.UnitTests.Migration.Fakes; @@ -20,22 +21,45 @@ static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalI return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); } + static string Describe(MigrationCategory category) => + $"{category.Kind} {category.Order}: {category.Id}" + + (category.CarriesBodies ? ", with bodies" : string.Empty) + + (category.MustFollow is null ? string.Empty : $", after {category.MustFollow}"); + [Test] - public void All_twelve_required_categories_are_always_selected() + public void Every_category_runs_in_a_fixed_order() { - var engine = BuildEngine([], out _); + var everyOptionalId = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional) + .Select(category => category.Id) + .ToArray(); + var engine = BuildEngine(everyOptionalId, out _); + + var runOrder = engine.SelectCategories(MigrationCategoryKind.Required) + .Concat(engine.SelectCategories(MigrationCategoryKind.Optional)) + .Select(Describe); - Assert.That(engine.SelectCategories(MigrationCategoryKind.Required), Has.Count.EqualTo(12)); + Approver.Verify(string.Join(Environment.NewLine, runOrder)); } [Test] public void Only_configured_optional_categories_are_selected() { - var engine = BuildEngine(["EventLog"], out _); + string[] configured = [MigrationCategoryIds.GroupComments, MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages]; + var engine = BuildEngine(configured, out _); var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + var left = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional && !selected.Contains(category)) + .Select(category => category.Id); - Assert.That(selected.Select(c => c.Id), Is.EqualTo(new[] { "EventLog" })); + Approver.Verify(string.Join(Environment.NewLine, + [ + $"Configured: {string.Join(", ", configured)}", + "Runs as:", + .. selected.Select(Describe), + $"Never copied: {string.Join(", ", left)}" + ])); } [Test] diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index ba24b8dfaf..52edc7954f 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -46,6 +46,33 @@ public async Task A_systemic_failure_halts_the_category_partway_through() } } + [Test] + public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // The same 20% that halts the category above, except these rows are past the target's retention + // cutoff, so leaving them behind is the copy working rather than failing. + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((800L, 200L))); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(800), "the last row was reached, so nothing halted partway"); + } + } + [Test] public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 618edd403a..d8c5202d27 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -79,6 +79,40 @@ public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_t } } + [Test] + public async Task A_target_counting_more_benign_skips_than_skipped_rows_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new OverCountedBenignTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("2 benign skips").And.Contain("out of 1 skipped")); + } + } + + sealed class OverCountedBenignTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public int BatchSizeFor(MigrationCategory category) => 10; + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var reasons = new Dictionary { [MigrationSkipReason.PastRetention] = batch.Rows.Count }; + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, reasons), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, [], 0, reasons, BenignSkipped: batch.Rows.Count + 1); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + } + sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget { public int BatchSizeFor(MigrationCategory category) => 10; From b42522ffac71bb55d6b84b5486d4fb4089ecc905 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 17 Sep 2026 12:14:22 +0800 Subject: [PATCH 11/28] Enhance Migration Engine and Unit Tests - Improved error logging in MigrationEngine to provide clearer context on exceptions. - Added functionality to InMemoryMigrationSource to simulate shutdown behavior during body reads. - Enhanced InMemoryMigrationTarget to allow custom exceptions for failure scenarios. - Introduced TimerRecordingTimeProvider to facilitate testing of timer-related functionality. - Expanded HaltThresholdTests to cover edge cases involving mixed benign and fault skips. - Updated MigrationEngineBodyRetryTests to ensure proper handling of shutdowns during body reads. - Added tests to verify behavior when categories are empty or already completed. - Implemented checks for checkpoint conflicts and cancellation scenarios in MigrationEngineFailurePathTests. - Enhanced MigrationEngineHaltTests to evaluate mixed skip scenarios and their impact on halting. - Improved MigrationEngineOrderingTests to ensure blocked categories behave correctly. - Added tests to MigrationEngineResumeTests to verify that restart behavior maintains original start time. - Updated MigrationEngineRunCategoriesTests to handle cases where no categories are selected. - Enhanced MigrationEngineSkipReasonTests to ensure accurate recording of multiple skip reasons. - Added tests to MigrationEngineThrottleTests to verify behavior during pauses and shutdowns. --- .../DataMigration/MigrationEngine.cs | 3 +- .../Fakes/InMemoryMigrationSource.cs | 10 ++ .../Fakes/InMemoryMigrationTarget.cs | 13 +- .../Fakes/TimerRecordingTimeProvider.cs | 35 +++++ .../Migration/HaltThresholdTests.cs | 30 +++- .../MigrationEngineBodyRetryTests.cs | 86 ++++++++++- .../Migration/MigrationEngineCopyTests.cs | 66 +++++++++ .../MigrationEngineFailurePathTests.cs | 136 ++++++++++++++++++ .../Migration/MigrationEngineHaltTests.cs | 61 ++++++++ .../Migration/MigrationEngineOrderingTests.cs | 66 +++++++++ .../Migration/MigrationEngineResumeTests.cs | 30 ++++ .../MigrationEngineRunCategoriesTests.cs | 18 +++ .../MigrationEngineSkipReasonTests.cs | 32 +++++ .../Migration/MigrationEngineThrottleTests.cs | 57 ++++---- 14 files changed, 608 insertions(+), 35 deletions(-) create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index c64ac05f2d..3503321796 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -159,7 +159,8 @@ public async Task RunCategoryAsync(MigrationCategory catego } catch (Exception ex) { - var reason = $"{ex.GetType().Name} at cursor {checkpoint.Cursor ?? "the start"}: {ex.Message}"; + var position = checkpoint.Cursor is null ? "at the start" : $"at cursor {checkpoint.Cursor}"; + var reason = $"{ex.GetType().Name} {position}: {ex.Message}"; logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs index bf5d522856..66341cbf20 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -24,6 +24,9 @@ public sealed class InMemoryMigrationSource : IMigrationSource public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); + /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } + public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; @@ -70,6 +73,13 @@ public async IAsyncEnumerable Read( await Task.Yield(); var attempt = bodyReadAttempts[sourceId] = BodyReadAttempts(sourceId) + 1; + + if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) { throw failures.Failure; diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index 4e50306469..cdfaff9777 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -16,6 +16,12 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint public int DefaultBatchSize { get; set; } = 3; public string NoBatchSizeFor { get; set; } public int? FailOnCallNumber { get; set; } + + /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + public Exception FailWith { get; set; } + + /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + public (int CallNumber, CancellationTokenSource Source)? CancelOnCall { get; set; } public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } int callCount; @@ -45,7 +51,12 @@ public async Task Write( if (FailOnCallNumber == callCount) { - throw new InvalidOperationException($"Simulated failure on write {callCount}"); + throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callCount}"); + } + + if (CancelOnCall is { } cancel && cancel.CallNumber == callCount) + { + await cancel.Source.CancelAsync(); } var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs new file mode 100644 index 0000000000..6178cbde97 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Time.Testing; + +// Signals each timer the engine creates, so a test only advances the clock once a pause is waiting on it. +public sealed class TimerRecordingTimeProvider : TimeProvider +{ + readonly FakeTimeProvider clock = new(); + + public SemaphoreSlim TimerCreated { get; } = new(0); + + public List DueTimes { get; } = []; + + public void Advance(TimeSpan delta) => clock.Advance(delta); + + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + + public override long GetTimestamp() => clock.GetTimestamp(); + + public override long TimestampFrequency => clock.TimestampFrequency; + + public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = clock.CreateTimer(callback, state, dueTime, period); + DueTimes.Add(dueTime); + TimerCreated.Release(); + return timer; + } +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs index f0b02dbdfb..fbe2bebe97 100644 --- a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -36,12 +36,40 @@ public void Does_not_halt_a_large_category_with_only_a_small_proportion_skipped( [Test] public void Exactly_at_both_boundaries_does_not_halt_because_both_must_be_exceeded() { - // 100 of 2,000 is exactly 5% and exactly the floor. "Exceed" means strictly past, not "at". + // Exactly the floor, which settles it before the proportion is ever worked out. "Exceed" means strictly past, not "at". var exceeded = HaltThreshold.Exceeded(skippedCount: 100, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); Assert.That(exceeded, Is.False); } + [Test] + public void Exactly_on_the_proportion_does_not_halt_once_the_floor_is_behind_it() + { + // 101 of 2,020 is exactly 5% with the floor already passed, so this is the only shape that + // reaches the proportion comparison and depends on it being strictly greater. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_020, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_hair_past_the_proportion_halts_once_the_floor_is_behind_it() + { + // 101 of 2,000 is 5.05%: the same skip count as above, one row's worth over the line. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void A_skip_count_with_nothing_processed_never_halts_and_never_divides_by_zero() + { + // Past the floor with a zero total, which is the only input that reaches the division guard. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 0, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + [Test] public void No_rows_processed_yet_never_halts() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs index cfac8aa11a..dc938d6543 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -4,6 +4,7 @@ namespace ServiceControl.UnitTests.Migration; using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; @@ -43,13 +44,25 @@ public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_writ } } - [Test] - public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message() + // Every type the engine treats as a defect. Each fails the same way on every attempt, so retrying one + // would only turn a code fault into skipped messages. + static readonly Exception[] Defects = + [ + new NotSupportedException("this source cannot read bodies"), + new NotImplementedException("not written yet"), + new InvalidOperationException("the session is closed"), + new ArgumentException("the id is not a document id"), + new NullReferenceException("no attachment"), + new InvalidCastException("not an attachment") + ]; + + [TestCaseSource(nameof(Defects))] + public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message(Exception defect) { var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("msg-1")); - source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new NotSupportedException("this source cannot read bodies")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, defect); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; @@ -60,13 +73,78 @@ public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_f using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); - Assert.That(checkpoint.LastError, Does.Contain(nameof(NotSupportedException))); + Assert.That(checkpoint.LastError, Does.Contain(defect.GetType().Name)); Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1)); Assert.That(checkpoint.SkippedCount, Is.Zero); Assert.That(checkpoint.SkipReasons, Is.Null); } } + [Test] + public async Task A_shutdown_during_a_body_read_stops_the_run_instead_of_skipping_the_message() + { + // Retrying a shutdown to the attempt limit and then recording the message as permanently + // unreadable is the one path here that silently loses a customer's failed message. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + using var stopping = new CancellationTokenSource(); + source.StopOnBodyRead = ("msg-1", stopping); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1), "a shutdown is not a transient body failure, so it is not retried"); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(persisted.SkippedCount, Is.Zero, "the message is still there to copy on the next run"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task The_configured_backoff_is_waited_between_body_read_attempts() + { + // Without the wait, three attempts against a body store that is briefly down all fail inside a + // millisecond and the message is skipped for an outage it would have survived. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var clock = new TimerRecordingTimeProvider(); + var backoff = TimeSpan.FromMilliseconds(200); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = backoff }; + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // Two failures, so a wait after each before the attempt that succeeds. + for (var waitNumber = 1; waitNumber <= MigrationEngine.MaxBodyReadAttempts - 1; waitNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"backoff {waitNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"backoff {waitNumber} should still be pending"); + clock.Advance(backoff); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { backoff, backoff }), "no wait after the final attempt, which has nothing left to retry"); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + } + } + [Test] public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs index c1a663d1f9..aafb9d4935 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -40,6 +40,72 @@ public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skippe } } + [Test] + public async Task A_category_with_no_rows_finishes_Complete_without_a_cursor() + { + // The ordinary state of several required categories on a small instance: nothing to copy is a + // finished category, not a category that never ran. + var category = MigrationCategoryRegistry.Find("MessageRedirects")!; + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((0L, 0L))); + Assert.That(checkpoint.Cursor, Is.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task The_moment_a_category_settles_comes_from_the_injected_clock() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var settledAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var clock = new FakeTimeProvider(settledAt); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SettledAt, Is.EqualTo(settledAt.UtcDateTime), "a wall-clock read here would drift from every other time the migration reports"); + } + + [Test] + public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_second_run() + { + // Finished with a few skips is finished. Re-reading it would copy the whole category again and + // count its skips a second time. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var finishedWithSkips = new MigrationCheckpoint(category.Id, MigrationCategoryState.CompleteWithErrors, "b", 1, 1, 2, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 }, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(finishedWithSkips); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(finishedWithSkips with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + [Test] public async Task A_category_already_Complete_is_left_alone_on_a_second_run() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs index 18d109dd44..fb26628421 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -225,6 +225,142 @@ public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); } + [Test] + public async Task A_checkpoint_conflict_leaves_the_other_writer_alone_instead_of_halting_over_it() + { + // Two hosts pointed at one target is what the version token exists for. Settling this as halted + // would write over the progress of whichever host is still copying. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + // Save 1 moves the row to in progress; save 2 is the first batch, by which point the other host has moved it on. + var checkpointStore = new ConflictOnNthSaveCheckpointStore { ConflictOnSave = 2 }; + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + var persisted = await checkpointStore.Read(category.Id); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "no halted row was written over the conflict"); + } + + [Test] + public async Task A_cancellation_that_is_not_a_shutdown_halts_the_category_like_any_other_failure() + { + // An inner timeout surfaces as the same exception type as a host stopping, and only the token + // says which. Treating a timeout as a shutdown would end the run with no reason on the row. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) + { + FailOnCallNumber = 1, + FailWith = new OperationCanceledException("the query timed out") + }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("OperationCanceledException").And.Contain("the query timed out")); + } + } + + [Test] + public async Task The_halt_reason_names_the_cursor_the_copy_had_reached() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at cursor b"), "the cursor is the only pointer an operator has to where it stopped"); + } + + [Test] + public async Task The_halt_reason_says_at_the_start_when_the_first_batch_never_committed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at the start")); + } + + [Test] + public async Task Every_row_the_target_skips_is_named_in_the_log() + { + // The counts say how much was left behind. Only the log says which rows, and it is the way back + // to them while the RavenDB database still exists. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("b", MigrationSkipReason.PastRetention); + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Select(entry => entry.Message), Has.Some.EqualTo("Skipped b in category KnownEndpoints")); + } + + [Test] + public async Task A_stop_between_batches_leaves_the_row_in_progress_at_the_batch_that_committed() + { + // The stop lands in the source rather than in a write, so nothing is mid-transaction: the row + // still has to describe the batches that did commit, and stay resumable. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, CancelOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("b")); + Assert.That(persisted.CopiedCount, Is.EqualTo(2)); + Assert.That(persisted.SettledAt, Is.Null); + } + } + + // Another host moved the row on between this host reading it and saving it. + sealed class ConflictOnNthSaveCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + int saves; + + public int ConflictOnSave { get; init; } + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ++saves == ConflictOnSave + ? throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row has moved on.") + : saved.Upsert(checkpoint, cancellationToken); + } + // The store shares the target's database, which has become unreachable by the time the halt is saved. sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index 52edc7954f..47a77f85d7 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -73,6 +73,67 @@ public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_t } } + [TestCase(25, MigrationCategoryState.CompleteWithErrors, TestName = "A_mix_of_benign_and_fault_skips_runs_on_while_the_faults_stay_under_the_threshold")] + [TestCase(10, MigrationCategoryState.Halted, TestName = "A_mix_of_benign_and_fault_skips_halts_once_the_faults_alone_pass_the_threshold")] + public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_alone(int everyNthIsAFault, MigrationCategoryState expected) + { + // A real archive copy loses rows both ways at once: retention takes some, unreadable bodies take + // others. This is the only shape where the subtraction has to do arithmetic rather than pick a side. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 5_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + // A fifth of the category is past retention either way, which on its own is four times the threshold. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + } + // The offset keeps the faults clear of the benign rows: 4% of the category in one case, 10% in the other. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % everyNthIsAFault == 3)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(expected)); + } + + [Test] + public async Task Rows_already_present_keep_a_category_under_the_halt_threshold() + { + // Already-present rows are in the denominator because the run did handle them. Drop them from it + // and this category's 4% fault rate reads as 12%, halting a copy that is merely being re-run. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 3_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 2_000)) + { + target.SeedExistingKey($"row-{i}"); + } + // 120 faults spread through the whole category: past the 100-row floor, and 4% of 3,000. + foreach (var i in Enumerable.Range(1, 3_000).Where(i => i % 25 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(120)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(1_920), "the 80 already-present rows that are also faults are refused before the collision check"); + } + } + [Test] public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs index 67c1e3b53b..a7f2c99e25 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -87,6 +87,72 @@ public async Task GroupComments_does_not_start_before_the_archive_completes(Migr } } + [Test] + public async Task A_blocked_category_runs_once_the_category_it_follows_settles() + { + // Every real migration starts group comments blocked, so a block nothing can clear would strand + // the last category and leave the migration unable to end. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var archiveRunning = new MigrationCheckpoint(archive.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + var saved = await checkpointStore.Upsert(archiveRunning); + var target = new InMemoryMigrationTarget(checkpointStore); + + var blocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + await checkpointStore.Upsert(saved with { State = MigrationCategoryState.Complete, SettledAt = DateTime.UtcNow }); + var unblocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(unblocked.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(unblocked.LastError, Is.Null, "the block reason does not outlive the block"); + Assert.That(target.WrittenRows(comments.Id), Has.Count.EqualTo(1)); + } + } + + [Test] + public async Task A_blocked_category_has_not_settled_because_it_has_not_finished() + { + // Anything reading SettledAt beside State would otherwise take a blocked category for a finished one. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + var persisted = await checkpointStore.Read(comments.Id); + Assert.That(persisted!.SettledAt, Is.Null); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Blocked)] + public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState archiveState) + { + // A missing row reads as "not started"; a row that exists says what it actually holds, which is + // how an operator tells a category waiting its turn from one waiting on a chain. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, null, 0, 0, null, null, null, null, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); + } + } + [TestCase(MigrationCategoryState.Complete)] [TestCase(MigrationCategoryState.CompleteWithErrors)] [TestCase(MigrationCategoryState.Abandoned)] diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs index ff7246d3bd..d21760171f 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -17,6 +17,36 @@ class MigrationEngineResumeTests { static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + [Test] + public async Task A_restart_keeps_the_moment_the_category_first_started() + { + // StartedAt is what an operator reads to see how long a background copy has been going. Stamping + // it again on every restart would report minutes for a copy that has been running for days. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstStart = new DateTimeOffset(2026, 3, 1, 9, 0, 0, TimeSpan.Zero); + var halted = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(firstStart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + // A day later, the cause is fixed and the host is started again. + target.FailOnCallNumber = null; + var restart = firstStart.AddDays(1); + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(restart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.StartedAt, Is.EqualTo(firstStart.UtcDateTime)); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "the restart carries on a copy that started a day ago"); + Assert.That(finished.SettledAt, Is.EqualTo(restart.UtcDateTime)); + } + } + [Test] public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_and_no_gaps() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs index 691f0a6f0b..e4e8393de0 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -40,6 +40,24 @@ public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them( } } + [Test] + public async Task Running_no_categories_copies_nothing_and_reports_nothing() + { + // What an instance that selected no optional categories asks for on every background pass. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + var results = await engine.RunCategories([]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results, Is.Empty); + Assert.That(await checkpointStore.ReadAll(), Is.Empty, "a category nobody ran gets no row"); + } + } + [Test] public async Task One_category_halting_does_not_stop_the_ones_after_it() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index d8c5202d27..2e07501603 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -3,6 +3,7 @@ namespace ServiceControl.UnitTests.Migration; using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; @@ -58,6 +59,37 @@ public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnread Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); } + [Test] + public async Task A_batch_that_loses_rows_two_different_ways_records_both_reasons() + { + // The breakdown is what tells a customer what they lost and why. A merge that overwrote instead of + // summing would leave the total right and the reasons wrong, and verification would still balance. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2"), Row("msg-3")); + // One the engine skips itself because the body will not read, one the target refuses. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + target.RejectKey("msg-2", MigrationSkipReason.PastRetention, benign: true); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary + { + [MigrationSkipReason.BodyUnreadable] = 1, + [MigrationSkipReason.PastRetention] = 1 + })); + Assert.That(target.WrittenRows(category.Id).Select(row => row.SourceId), Is.EqualTo(new[] { "msg-3" })); + } + } + [Test] public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_the_category() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs index c4665da55f..e568edb6a1 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -48,6 +48,35 @@ public async Task Optional_categories_pause_between_batches_for_the_configured_d } } + [Test] + public async Task A_stop_during_a_pause_ends_the_run_as_a_shutdown() + { + // Most of a throttled background copy's life is spent in this pause, so it is where a host being + // stopped most often lands. + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + using var stopping = new CancellationTokenSource(); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category, stopping.Token); + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, "the first pause never started"); + await stopping.CancelAsync(); + + Assert.ThrowsAsync(() => runTask); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown mid-pause is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("a"), "the batch before the pause committed"); + } + } + [Test] public async Task Required_categories_never_pause() { @@ -65,32 +94,4 @@ public async Task Required_categories_never_pause() Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); } - - // Signals each timer the engine creates, so the test only advances the clock once a pause is waiting on it. - sealed class TimerRecordingTimeProvider : TimeProvider - { - readonly FakeTimeProvider clock = new(); - - public SemaphoreSlim TimerCreated { get; } = new(0); - - public List DueTimes { get; } = []; - - public void Advance(TimeSpan delta) => clock.Advance(delta); - - public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); - - public override long GetTimestamp() => clock.GetTimestamp(); - - public override long TimestampFrequency => clock.TimestampFrequency; - - public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; - - public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) - { - var timer = clock.CreateTimer(callback, state, dueTime, period); - DueTimes.Add(dueTime); - TimerCreated.Release(); - return timer; - } - } } From c72590e4fa801f0e29958043ddb14929eaf644ba Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 30 Sep 2026 11:20:28 +0800 Subject: [PATCH 12/28] Refactor migration settings and update documentation to remove RavenDB references --- .../ravendb-to-sql-migration-instructions.md | 2 -- .../TwoPersistersInOneProcessTests.cs | 17 ----------------- .../DataMigration/MigrationSettings.cs | 4 ---- ...pprovals.PlatformSampleSettings.approved.txt | 1 - .../Commands/MigrationSourceReportCommand.cs | 2 +- src/ServiceControl/Hosting/Help.txt | 5 ++--- .../Infrastructure/Settings/Settings.cs | 2 -- .../Persistence/PersistenceFactory.cs | 13 ++++++------- 8 files changed, 9 insertions(+), 37 deletions(-) diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index 7b3fc8c7ae..91b421c143 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -18,8 +18,6 @@ The source is a ServiceControl error instance on RavenDB. Keep its RavenDB setti | `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | A secured external server's client certificate | | `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. Don't change it during the move | -`ServiceControl/Migration/SourcePersistenceType` defaults to `RavenDB` and needs no setting. - ## Report on the source Run the instance's executable with `--migration-source-report`: diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs index ba677f8b42..b246735641 100644 --- a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -84,23 +84,6 @@ public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_ide "The source read its own RavenDB settings rather than the target's."); } - [Test] - public void Asking_a_SQL_persister_for_a_source_names_the_setting_to_change() - { - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", "SQLServer"); - - try - { - var refusal = Assert.ThrowsAsync(async () => await PersistenceFactory.OpenMigrationSource(new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)))); - - Assert.That(refusal.Message, Does.Contain("cannot be read as a migration source").And.Contain("ServiceControl/Migration/SourcePersistenceType")); - } - finally - { - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", null); - } - } - [Test] public void Asking_a_SQL_persister_for_maintenance_mode_is_still_refused() { diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs index 2b4faf5c91..0de8ddb892 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -8,12 +8,8 @@ public static class MigrationSettings public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; /// A comma-separated list of the optional categories to copy, such as "EventLog, CustomChecks". public const string OptionalCategoriesKey = "Migration/OptionalCategories"; - /// Which persister holds the old data being copied from. - public const string SourcePersistenceTypeKey = "Migration/SourcePersistenceType"; public const int DefaultThrottlePauseMilliseconds = 100; public const int DefaultHaltThresholdPercent = 5; public const int DefaultHaltThresholdMinimum = 100; - /// RavenDB is the only source supported today. - public const string DefaultSourcePersistenceType = "RavenDB"; } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index b851fb956d..b7303ff3a8 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,7 +63,6 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", - "MigrationSourcePersistenceType": "RavenDB", "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs index ce0ac0501b..21446e6c5e 100644 --- a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -18,7 +18,7 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel Console.Out.WriteLine("ServiceControl migration source report"); Console.Out.WriteLine(); - Console.Out.WriteLine($"{"Source persistence",-20}: {settings.MigrationSourcePersistenceType}"); + Console.Out.WriteLine($"{"Source persistence",-20}: {PersistenceFactory.MigrationSourcePersistenceType}"); Console.Out.WriteLine($"{"Version",-20}: {description.Version}"); foreach (var fact in description.Facts) diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index 3c3d0d5774..6fba5341f9 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -27,9 +27,8 @@ MIGRATION SOURCE REPORT ServiceControl.exe --migration-source-report -Reports what a migration would read from the source persistence named by -ServiceControl/Migration/SourcePersistenceType: the facts the source reports about itself, with the -setting each came from, and a row count for everything it holds. The source reads the instance's own +Reports what a migration would read from the RavenDB source: the facts the source reports about +itself, with the setting each came from, and a row count for everything it holds. The source reads the instance's own RavenDB settings, so keep them in place when switching PersistenceType. For a RavenDB source: an EXTERNAL server can be reported on while ServiceControl is running. An EMBEDDED source diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index fbdc0e50a4..c601c07b57 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,7 +16,6 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; - using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -186,7 +185,6 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } - public string MigrationSourcePersistenceType => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.SourcePersistenceTypeKey, MigrationSettings.DefaultSourcePersistenceType); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 5aac4cdda8..c431680696 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -28,15 +28,14 @@ public static IPersistence Create(Settings settings, bool maintenanceMode = fals return persistence; } + /// + /// The persistence a migration copies from. RavenDB is the only source supported today. + /// + public const string MigrationSourcePersistenceType = "RavenDB"; + public static IMigrationSource CreateMigrationSource(Settings settings) { - var persistenceType = settings.MigrationSourcePersistenceType; - var persistenceConfiguration = CreatePersistenceConfiguration(persistenceType, settings); - - if (persistenceConfiguration is not IMigrationSourceFactory sourceFactory) - { - throw new Exception($"The '{persistenceType}' persistence cannot be read as a migration source. Set {Settings.SettingsRootNamespace}/{MigrationSettings.SourcePersistenceTypeKey} to the persistence that holds the data being migrated away from."); - } + var sourceFactory = (IMigrationSourceFactory)CreatePersistenceConfiguration(MigrationSourcePersistenceType, settings); return sourceFactory.CreateSource(Settings.SettingsRootNamespace); } From 79084989c62f17840d04243c9181276ddddafaff Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 30 Sep 2026 11:48:35 +0800 Subject: [PATCH 13/28] Clarify event log handling and data retention during migration from RavenDB to SQL --- docs/migration/ravendb-to-sql-migration-overview.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index de2c322b14..663f79ae0f 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -160,11 +160,11 @@ flowchart TB - Licensing report masks - The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again - Subscriptions +- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from the newest event in RavenDB rather than from when the copy runs, so a restart copies the same window. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history ### Optional - Archived and resolved failed messages: the biggest category by far, and most of the copying time -- The event log, without which the ServicePulse activity feed starts empty - Custom checks, which cost almost nothing to skip because every check re-reports on its next interval - Failed error imports, the record of errors that could not be ingested - Group comments, **copied last of everything**, after archived and resolved messages. A comment survives only once the failed messages its group is built from have arrived, so on a large archive the comments are the last thing to appear. An empty comment field partway through a migration is the copy still running, not data loss @@ -181,7 +181,7 @@ flowchart TB ## What does not come across -**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. +**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. Neither is an event log item raised more than 7 days before the newest one, which falls outside the [required](#required) event log window rather than being skipped. **Rows skipped one at a time, and counted.** Each of these shows up in the skipped count for its category, broken out by reason, so you can see how much went and why: @@ -201,7 +201,7 @@ flowchart TB - **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. - **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. -**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). Everything in the [required](#required) set is therefore safe, since unresolved and retry-issued messages have their expiry removed when the retry is issued, so only the archived and resolved messages category and the event log category can shrink underneath the copier, and both copy in the background where the window is longest. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere: the counts are of rows the source actually handed over, and there is no expected total to fall short of. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. +**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). Unresolved and retry-issued messages have their expiry removed when the retry is issued, so only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere: the counts are of rows the source actually handed over, and there is no expected total to fall short of. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. **A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. From 54a657127a946d9ed0ae34504aa7a2239ecc2fa9 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 30 Sep 2026 14:18:59 +0800 Subject: [PATCH 14/28] Update migration documentation to clarify integration events and error imports handling --- .../ravendb-to-sql-migration-overview.md | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 663f79ae0f..6af075e895 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -19,7 +19,7 @@ This covers the error instance only. The audit instance has no SQL persister, so - **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. - **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and the migration has written nothing to RavenDB: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. - **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. -- **Every identifier anything depends on is carried across**. The event log and historic retry operations are renumbered, because nothing references their keys. +- **Every identifier anything depends on is carried across**. The event log, historic retry operations and pending integration events are renumbered, because nothing references their keys. - **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. - **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. - **Bounded impact on a live instance**. Throttled behind normal ingestion and streamed, so memory does not track the size of the database. @@ -160,24 +160,25 @@ flowchart TB - Licensing report masks - The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again - Subscriptions -- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from the newest event in RavenDB rather than from when the copy runs, so a restart copies the same window. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history +- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from the newest event in RavenDB rather than from when the copy runs, so a restart copies the same window. That newest time is capped at when RavenDB last stored an event, because event times come from the endpoints and an endpoint whose clock runs ahead would otherwise push the window forward. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history +- Integration events still waiting to be sent when you switch over. There are only any if the old instance was falling behind or could not reach the broker, and each one is an event a subscriber has not yet received. They are sent once ServiceControl opens, later than they would have been. RavenDB already sends them in no particular order, so no ordering is lost +- Custom checks, with the status each last reported. They are required because not every check reports again: an endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one behind could hide a known failure until that endpoint restarts +- Failed error imports, with their bodies. Each is a failed message ServiceControl took off the error queue but could not ingest, so it exists nowhere else, and it never expires. After the move, the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB before you start is better still, and the [dry run](#dry-run) tells you how many there are +- Group comments, copied straight after the unresolved failed messages, so a note such as "do not retry this group" is there the moment ServiceControl opens. Every comment except a blank one is copied. A comment on a group whose messages are all archived or resolved waits while those messages copy, and once the migration settles ServiceControl's own clean-up removes any comment whose group has no failed messages left, exactly as it always does on SQL ### Optional - Archived and resolved failed messages: the biggest category by far, and most of the copying time -- Custom checks, which cost almost nothing to skip because every check re-reports on its next interval -- Failed error imports, the record of errors that could not be ingested -- Group comments, **copied last of everything**, after archived and resolved messages. A comment survives only once the failed messages its group is built from have arrived, so on a large archive the comments are the last thing to appear. An empty comment field partway through a migration is the copy still running, not data loss -- Failed message edits ### Not migrated - The RavenDB index definitions - The transient in-flight collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations` - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works -- `ConnectedApplications`, which only versions 6.0 and 6.1 wrote and nothing has read since -- Integration events still waiting to be sent when you switch over are never sent +- The `ConnectedApplications` document, which only versions 6.0 and 6.1 wrote. Since 6.2 the MassTransit connector status that ServicePulse uses to turn features on and off comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document - Broker and audit service version details, which refill on the throughput collector's next run +- Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if the message fails again afterwards: on RavenDB it can then never be edited again, and after the move it can be edited once more +- Heartbeat state, which neither persister stores: ServiceControl rebuilds it in memory from live heartbeats after every restart. The list of known endpoints and which ones are monitored is copied, so after the move heartbeat monitoring behaves exactly as it does after any restart. An endpoint instance that is down sends no heartbeat, so it is counted as failing on the dashboard with no last heartbeat time, and no heartbeat alert is raised for it ## What does not come across @@ -190,7 +191,6 @@ flowchart TB - A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. - A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. - An archived or resolved failed message, or an event log item, already past its retention period. SQL's retention clean-up would delete it on its first pass, so it is counted rather than copied only to be deleted. -- A group comment whose failure group has no failed messages in SQL once the messages are copied. SQL's clean-up removes such a comment, where RavenDB never expired one. - Endpoint settings for an endpoint ServiceControl does not know. ServiceControl removes those settings shortly after it starts. - A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way, because ServiceControl never stores one. @@ -199,9 +199,9 @@ flowchart TB - **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, in the one category every customer copies. A message that failed five times arrives showing one attempt, and the other four are gone. - **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. - **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. -- **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. +- **Event log items, historic retry operations and pending integration events are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. -**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). Unresolved and retry-issued messages have their expiry removed when the retry is issued, so only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere: the counts are of rows the source actually handed over, and there is no expected total to fall short of. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. +**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, when it is unarchived, or when a retry is issued. The exception is a database last written by version 6.18 or earlier, where a message that failed again after being archived or resolved kept its old expiry, so a few unresolved messages there can still expire during the copy. Apart from those, only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere. The copier counts each category before it starts, and if the copy comes up short of that count it counts the source again: rows that no longer exist were removed by RavenDB and are an absence, while rows that still exist but were never read halt the category. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. **A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. @@ -286,7 +286,7 @@ The thing to read twice is that the counts never travel back through the engine - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. - The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. -- Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. +- Rows left behind because SQL would remove them anyway (past retention, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. - The percentage is measured against what the run has processed so far rather than against the category's total, so a run that starts badly looks worse than it is. The floor is what keeps that harmless, since fewer than 101 skipped rows never consults the percentage at all. More than that, bunched at the start, does halt a category whose overall rate would have been fine, and the cost is one restart: the skipped rows commit with the cursor, so the next run resumes past them with its counters back at zero. - A source therefore must not read a category in an order that puts the rows most likely to be skipped at the front of it. - Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. @@ -315,7 +315,7 @@ stateDiagram-v2 **Two things halt a category.** Either the skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental, or the copy hits an error it did not expect, in which case the error type and the cursor it stopped at are recorded. A host being shut down is neither: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. -**A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the archived messages they belong to. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. +**A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the unresolved failed messages their groups are built from. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. **What it costs depends on which category halted.** A halted optional category means the instance keeps serving traffic and that one slice of history is missing until it is resumed. A halted required category means the host stays closed, so the outage carries on until the halt is cleared or the category is abandoned. That is deliberate: opening the host is the point of no return, and it should not happen with required data left behind by accident. @@ -342,12 +342,14 @@ What it resolves and reports: It runs the same startup checks that gate startup, so a missing setting surfaces before a customer books an outage. -It counts, before anything moves, the rows that cannot cross as they stand: +If the source holds failed error imports, it says how many and advises running `--import-failed-errors` against RavenDB before the move. They are copied either way, but ones imported first arrive as ordinary failed messages rather than as imports still waiting. + +It counts, before anything moves, the rows the target says it would skip or merge, by reason. These include: - Documents whose `UniqueMessageId` will not parse as a GUID - Subscriptions that differ only in message-type version, and so merge onto one row - Subscriptions whose message type or transport address exceeds the 200-character key limit -- Integration event dispatches still queued, which are not copied and will never be sent +- Rows already past their retention period, and rows missing a value SQL requires It reports no duration for the optional categories, and nothing about load on the source. From 54befb318b196a4e824b1ce05a707a9bf40a7757 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 30 Sep 2026 14:24:12 +0800 Subject: [PATCH 15/28] Refine migration documentation for clarity on data retention and copying strategy --- .../ravendb-to-sql-migration-overview.md | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 6af075e895..9c167c3c0b 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -8,8 +8,8 @@ This covers the error instance only. The audit instance has no SQL persister, so ## Strategy -- Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention is shorter, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. -- The required set is small because unresolved failures are the only category a customer can act on, and the strategy assumes customers keep that number low by resolving and archiving. **A neglected instance breaks that assumption**: unresolved failures can legitimately be months old, and a large backlog of them makes the closed window long rather than short. The dry run is what tells a customer which case they are in. +- Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention defaults to 14 days, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. +- The required set is what the instance needs the moment it opens. Most of it is small, but two parts grow: unresolved failures, and the last 7 days of event log, which on a busy instance holds a row for every failure, retry and submission. The strategy assumes you keep unresolved failures low by resolving and archiving. **A neglected instance breaks that assumption**: unresolved failures can legitimately be months old, and a large backlog of them makes the closed window long rather than short. The dry run is what tells you which case you are in. - Anything not selected simply ages out of RavenDB, and the customer deletes the old database when they are ready. ## Goals @@ -18,11 +18,11 @@ This covers the error instance only. The audit instance has no SQL persister, so - **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. - **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. - **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and the migration has written nothing to RavenDB: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. -- **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. +- **No duplicates and no gaps**. Rows and the resume cursor, a marker of the last row copied, commit in one transaction, so a crash needs no reconciliation. - **Every identifier anything depends on is carried across**. The event log, historic retry operations and pending integration events are renumbered, because nothing references their keys. - **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. -- **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. -- **Bounded impact on a live instance**. Throttled behind normal ingestion and streamed, so memory does not track the size of the database. +- **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. The one exception is rows RavenDB's own expiration deletes while the copy runs: they are never read, so they are not skips, and the recount is what tells them apart from a real loss. See [what does not come across](#what-does-not-come-across). +- **Bounded impact on a live instance**. The background copy waits a fixed pause between batches, and reads are streamed, so memory does not track the size of the database. - **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. - **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. @@ -30,7 +30,7 @@ This covers the error instance only. The audit instance has no SQL persister, so - **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. - **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB, so once the host has served traffic there is no rollback of any kind. -- **Steerable while running.** No pause, resume, or abort. Changing anything means editing configuration and restarting. +- **Steerable while running.** No pause or resume, and no abort command. Going back during the closed window means stopping and reconfiguring, and changing anything else means editing configuration and restarting. - **A general-purpose migration tool.** The source is always RavenDB and the target is always a ServiceControl EF Core persister, both at versions this build can read. - **Custom migration UI via ServicePulse.** Custom checks and the event log will be used for progress reporting, but migration configuration and migration engine control will not be available via the UI. @@ -70,19 +70,20 @@ The copier runs inside the ServiceControl host, so every row and every message b ## Migration workflow 1. Upgrade ServiceControl as normal, still on RavenDB. -2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated-categories) they want copied. +2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and whether you want the one [optional](#optional) category, archived and resolved messages, copied. 3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. 4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the dry run reports](#dry-run) before booking an outage around its estimate. 5. Start ServiceControl (`MigrationMode=true`). 6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. -7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. +7. The copying of [required data](#required) starts, with ServiceControl still closed. How long it takes depends on how many unresolved failures you have and how busy the last 7 days were, and the [dry run](#dry-run) gives you an estimate. If a required category halts, the host stays closed until you fix the cause and restart, or abandon that category. 8. ServiceControl opens, and whatever [optional data](#optional) they asked for is copied in the background while the instance runs normally. They can watch it from ServicePulse custom checks and events, but not steer it. -9. They run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. It tolerates more rows in SQL than in RavenDB, because RavenDB keeps expiring rows the copier already took. +9. You run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. Counts can differ in both directions without anything being wrong. SQL can hold more rows, because RavenDB keeps expiring rows the copier already took. SQL can also hold fewer, because once ServiceControl opens it sends pending integration events, removes group comments whose group has no failed messages left, and removes failed error imports once they are imported again. 10. RavenDB data can be removed. - If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. - A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. - An explicit override exists for a customer who has changed their mind and accepts leaving data behind. It marks the outstanding categories as abandoned, which is a deliberate end state rather than a failure, so the progress check settles and the guard stays armed for any later migration. +- While a selected category is still unfinished, ServiceControl pauses its own clean-up: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. Your SQL database grows until the copy finishes, and these restart on their own once it does. - **Steps 5 to 7 are the abort window**, which is not the override above: see [the one point you can go back](#the-one-point-you-can-go-back). - A category that stops because too many rows failed is *halted*, and it stays that way until someone acts: fix the cause and restart to carry on from where it stopped, or abandon it deliberately if you accept the loss. See [a halt stops one category, and clearing it is a restart](#a-halt-stops-one-category-and-clearing-it-is-a-restart). @@ -126,9 +127,10 @@ flowchart TB B --> C["Open the old RavenDB, read only"] C --> D{"All checks pass?"} D -->|"No"| E["Host does not start.
Says which check failed.
Nothing has been copied."] - D -->|"Yes"| F["Copy what cannot be recreated.
Minutes. ServiceControl still closed."] - F --> G["ServiceControl opens.
New failed messages go straight to SQL."] - G --> H["Copy the selected history in the background,
throttled behind normal ingestion"] + D -->|"Yes"| F["Copy what cannot be recreated.
How long depends on unresolved failures
and the last 7 days of events.
ServiceControl still closed."] + F -->|"A required category halts"| M["Host stays closed.
Fix the cause and restart,
or abandon the category."] + F -->|"Required data copied"| G["ServiceControl opens.
New failed messages go straight to SQL."] + G --> H["Copy the selected history in the background,
with a fixed pause between batches"] H --> I["Verify row counts on both sides,
category by category"] I --> J{"MigrationMode = false,
everything complete?"} J -->|"No"| K["Host does not start.
Names what is outstanding.
An override exists."] @@ -160,11 +162,11 @@ flowchart TB - Licensing report masks - The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again - Subscriptions -- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from the newest event in RavenDB rather than from when the copy runs, so a restart copies the same window. That newest time is capped at when RavenDB last stored an event, because event times come from the endpoints and an endpoint whose clock runs ahead would otherwise push the window forward. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history +- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from whichever is earlier: the newest event in RavenDB, or the last time RavenDB stored one. Counting from the data rather than from when the copy runs means a restart copies the same window. The second limit is there because event times come from the endpoints, and an endpoint whose clock runs ahead would otherwise push the window forward. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history - Integration events still waiting to be sent when you switch over. There are only any if the old instance was falling behind or could not reach the broker, and each one is an event a subscriber has not yet received. They are sent once ServiceControl opens, later than they would have been. RavenDB already sends them in no particular order, so no ordering is lost - Custom checks, with the status each last reported. They are required because not every check reports again: an endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one behind could hide a known failure until that endpoint restarts - Failed error imports, with their bodies. Each is a failed message ServiceControl took off the error queue but could not ingest, so it exists nowhere else, and it never expires. After the move, the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB before you start is better still, and the [dry run](#dry-run) tells you how many there are -- Group comments, copied straight after the unresolved failed messages, so a note such as "do not retry this group" is there the moment ServiceControl opens. Every comment except a blank one is copied. A comment on a group whose messages are all archived or resolved waits while those messages copy, and once the migration settles ServiceControl's own clean-up removes any comment whose group has no failed messages left, exactly as it always does on SQL +- Group comments, copied after the unresolved failed messages, so a note such as "do not retry this group" is there the moment ServiceControl opens. They follow those messages because a comment belongs to a failure group, and the groups ServicePulse shows first are built from unresolved failed messages. Every comment except a blank one is copied. A comment on a group whose messages are all archived or resolved waits while those messages copy, and once the migration settles ServiceControl's own clean-up removes any comment whose group has no failed messages left, exactly as it always does on SQL ### Optional @@ -182,7 +184,7 @@ flowchart TB ## What does not come across -**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. Neither is an event log item raised more than 7 days before the newest one, which falls outside the [required](#required) event log window rather than being skipped. +**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. Neither is an event log item raised before the start of the 7-day window, which falls outside the [required](#required) event log window rather than being skipped. **Rows skipped one at a time, and counted.** Each of these shows up in the skipped count for its category, broken out by reason, so you can see how much went and why: @@ -192,22 +194,22 @@ flowchart TB - A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. - An archived or resolved failed message, or an event log item, already past its retention period. SQL's retention clean-up would delete it on its first pass, so it is counted rather than copied only to be deleted. - Endpoint settings for an endpoint ServiceControl does not know. ServiceControl removes those settings shortly after it starts. -- A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way, because ServiceControl never stores one. +- A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way: RavenDB can store one, but SQL never does. -**Things that change shape, and are not counted as skips at all.** The dry run counts these before anything moves, so they are a number you see in advance rather than a discovery afterwards. They are also the ones to read twice: +**Things that change shape, and are not counted as skips at all.** The dry run counts the two merges before anything moves. Attempt history and renumbering apply to every row of their kind, so there is nothing to count. They are also the ones to read twice: -- **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, in the one category every customer copies. A message that failed five times arrives showing one attempt, and the other four are gone. +- **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, whether it is unresolved, archived or resolved. A message that failed five times arrives showing one attempt, and the other four are gone. - **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. - **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. - **Event log items, historic retry operations and pending integration events are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. -**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, when it is unarchived, or when a retry is issued. The exception is a database last written by version 6.18 or earlier, where a message that failed again after being archived or resolved kept its old expiry, so a few unresolved messages there can still expire during the copy. Apart from those, only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere. The copier counts each category before it starts, and if the copy comes up short of that count it counts the source again: rows that no longer exist were removed by RavenDB and are an absence, while rows that still exist but were never read halt the category. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. +**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, when it is unarchived, or when a retry is issued. The exception is a message that failed again after being archived or resolved while the instance ran version 6.18 or earlier: it kept its old expiry, and upgrading does not remove it, so a few unresolved messages can still expire during the copy. Apart from those, only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere. The copier counts each category before it starts, and if the copy comes up short of that count it counts the source again: rows that no longer exist were removed by RavenDB and are an absence, while rows that still exist but were never read halt the category. The dry run's count is a snapshot rather than a promise, and the recount is what shows that RavenDB removed rows while the copy ran. **A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. ## The one point you can go back -While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. +While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data. To start again later, start from an empty SQL database: drop it, create it, and run `--setup` again. Do not reuse the old copy, because a second attempt skips every category the first one finished, so anything that reached RavenDB since is left behind, and integration events RavenDB has since sent would be sent again. That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. @@ -222,7 +224,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - A whole `FailedMessage` is written with its stored status intact. No existing caller does that, though the dialect upsert already accepts a status, so the gap is smaller than it looks. - `UniqueMessageId` keeps its value, but converts type: the source holds a string and the target column is a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. -- `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages have no `@expires`, so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. +- `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages normally have no `@expires` (see [the exception](#what-does-not-come-across) for messages from version 6.18 or earlier), so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. - Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. - Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. Throughput is a required category, so it copies while ServiceControl is closed, before any collector has written to SQL. Setting is what makes the category safe to resume after a crash, where adding would double-count. Copying the rows is also what stops the audit and broker collectors re-gathering the same days when the host opens, because `LastCollectedDate` is derived from the newest throughput row rather than stored (`LicensingDataStore.cs:45`). The checkpoint is what stops a second pass overwriting days the collectors have written since. - Identifiers narrow on the way across, and the dry run counts every kind. What narrows, merges or cannot be stored at all is in [what does not come across](#what-does-not-come-across). @@ -230,7 +232,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess ## Batching and throttling - Batch size comes from the provider: SQL Server divides its own parameter budget by the column count, PostgreSQL uses a flat 50 rows. -- The throttle is a configurable pause between batches, defaulting to 100 ms. Lowering it, or turning `MigrationMode` off, is the only remedy for a copy competing with production. +- The throttle is a configurable pause between batches, defaulting to 100 ms. Raising it slows the background copy and eases the load on production. Turning `MigrationMode` off is not a remedy: while a selected category is unfinished the host refuses to start, unless you abandon that category. ## Checkpointing and resume @@ -238,7 +240,7 @@ A copy that runs for hours will be interrupted at some point: a restart, a dropp **What one row holds:** the category it tracks, its state, the resume cursor, how many rows were copied, skipped and already present, a count per skip reason, how many rows the source held when the category started, when it started, when it last made progress, when it settled, the last error, and a version number used to spot a second writer. -**The states, and which ones a restart re-enters.** `Complete`, `CompleteWithErrors` and `Abandoned` are terminal, so a restart passes straight over the category. `Halted` and `Blocked` are not: a halt is resumed from its cursor once the cause is fixed, and a block clears itself once the category it waits on settles, which is how group comments end up behind archived messages. `NotStarted` and `InProgress` both mean there is work to do. +**The states, and which ones a restart re-enters.** `Complete`, `CompleteWithErrors` and `Abandoned` are terminal, so a restart passes straight over the category. `Halted` and `Blocked` are not: a halt is resumed from its cursor once the cause is fixed, and a block clears itself once the category it waits on settles, which is how group comments end up behind unresolved failed messages. `NotStarted` and `InProgress` both mean there is work to do. ### One batch, and why nothing provisional is ever saved @@ -285,7 +287,7 @@ The thing to read twice is that the counts never travel back through the engine - A body is read up to three times before the message is skipped whole, and the exhausted attempts count toward the halt threshold. - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. -- The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. +- The halt threshold is proportional, 5 percent by default, with an absolute floor, and skips halt a category only when both are exceeded. The one exception is a category smaller than the floor, which halts if it loses more than half its rows. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. - Rows left behind because SQL would remove them anyway (past retention, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. - The percentage is measured against what the run has processed so far rather than against the category's total, so a run that starts badly looks worse than it is. The floor is what keeps that harmless, since fewer than 101 skipped rows never consults the percentage at all. More than that, bunched at the start, does halt a category whose overall rate would have been fine, and the cost is one restart: the skipped rows commit with the cursor, so the next run resumes past them with its counters back at zero. - A source therefore must not read a category in an order that puts the rows most likely to be skipped at the front of it. @@ -304,7 +306,7 @@ stateDiagram-v2 InProgress --> InProgress: the host was stopped mid-copy,
so the next start resumes from the cursor InProgress --> Complete: every row reached, none skipped InProgress --> CompleteWithErrors: every row reached, some skipped - InProgress --> Halted: too many rows skipped in this run,
or the copy hit an error it did not expect + InProgress --> Halted: too many rows skipped, most of a small category lost,
an error it did not expect, or a shortfall the recount confirms Halted --> InProgress: fix the cause, restart,
carry on from the cursor Halted --> Abandoned: accept the loss, deliberately InProgress --> Abandoned: accept the loss, deliberately @@ -313,7 +315,14 @@ stateDiagram-v2 Abandoned --> [*] ``` -**Two things halt a category.** Either the skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental, or the copy hits an error it did not expect, in which case the error type and the cursor it stopped at are recorded. A host being shut down is neither: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. +**Four things halt a category.** + +- **Too many skips.** The skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental. +- **Most of a small category lost.** A category with fewer rows than the floor loses more than half of them. +- **An error the copy did not expect.** The error type and the cursor it stopped at are recorded. +- **A shortfall that survives the recount.** The copy came up short of the starting count, and the missing rows still exist in RavenDB but were never read. + +A host being shut down is none of these: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. **A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the unresolved failed messages their groups are built from. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. @@ -363,9 +372,9 @@ A containerised instance runs all three as a one-off `docker run` of the same im ## Configuration and control -- A customer sets `MigrationMode` and the list of categories next to it. A status command and a custom check report back. +- You set `MigrationMode`, and next to it whether to copy the one optional category, archived and resolved messages. A status command and a custom check report back. - Categories are read fresh at every startup. Adding one copies it on the next restart, removing one deletes nothing. -- There is no HTTP API, no pause, no resume, no abort, and no way to add a category to a running instance. All of those mean editing configuration and restarting. +- There is no HTTP API, no pause, no resume, no abort command, and no way to add a category to a running instance. All of those mean editing configuration and restarting. Going back during the closed window means stopping and reconfiguring, as [the one point you can go back](#the-one-point-you-can-go-back) describes. - The checkpoint table is a record of what happened, not a control channel. - Stopping a copy takes a restart, so it cannot be stopped in ten seconds. From 1cd87ebf5313faba044967e44adfd08df8734f3f Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 9 Oct 2026 17:53:57 +0800 Subject: [PATCH 16/28] [Migration Engine Part 2] Add a migration checkpoint table to SQL Server and PostgreSQL (#5897) * feat: Add Migration Checkpoints to EF Core - Introduced a new table `MigrationCheckpoints` to track migration states and progress. - Implemented `MigrationCheckpointEntity` to represent the checkpoint data structure. - Added `EFMigrationCheckpointStore` for managing checkpoint data with methods for reading and upserting checkpoints. - Created `MigrationCheckpointConfiguration` for configuring the entity in the DbContext. - Enhanced `ServiceControlDbContext` to include the new `MigrationCheckpoints` DbSet. - Updated `EfCoreExtensions` to provide an `UpsertCheckpoint` method for managing checkpoint persistence. - Added unit tests for `EFMigrationCheckpointStore` and `MigrationCheckpointTable` to ensure correct functionality. - Implemented transaction handling in `MigrationCheckpointTransactionTests` to verify checkpoint behavior during commits and rollbacks. - Updated RavenDB integration to support embedded server readiness checks. * feat: Enhance migration engine with improved error handling and configuration options * feat: Refactor migration checkpoint handling and add UpsertCheckpoint method * docs: Update migration documentation for clarity on RavenDB to SQL migration process * [Migration Engine Part 3] Implement the RavenDB to SQL migration machinery (#5911) * Update migration documentation * feat: run the required migration copy at host start Adds the RavenDB source and the EF Core target for the KnownEndpoints and EndpointSettings categories, the startup checks that refuse an unsupported or unready migration, and the stall watchdog that stops a copy committing nothing for 30 minutes. Documents the migration contracts. * test: cover the RavenDB to SQL migration end to end Adds unit tests for the startup checks, the stall watchdog and the refusals, target and reader tests for both persisters, a SQL Server collation test for the endpoint settings key, and acceptance tests for a copy that is killed, restarted and opened on. Approves the two new migration settings. * tests: ensure DB schema is setup before the host starts * refactor: update BatchSizeFor method to return Task for async compatibility across migration targets * docs: update migration instructions and system design to clarify settings and behavior * Refactor migration checks and improve error handling * Refactor migration commands and error handling * PR review * Fix flakey test --- README.md | 2 +- .../migration-system-design-diagram.png | Bin 319535 -> 209456 bytes .../ravendb-to-sql-migration-instructions.md | 234 ++++- .../ravendb-to-sql-migration-overview.md | 708 +++++++------ .../ravendb-to-sql-migration-system-design.md | 289 +++++ ...eControl.AcceptanceTests.PostgreSql.csproj | 7 + ...ceControl.AcceptanceTests.SqlServer.csproj | 7 + .../When_hosting_error_ingestion_only.cs | 33 +- .../.editorconfig | 6 + .../MigrationAcceptanceTest.cs | 472 +++++++++ .../MigrationSourceServer.cs | 82 ++ .../StopMigrationSourceServer.cs | 11 + .../TestMigrationTargets.cs | 121 +++ .../When_a_copy_is_killed_mid_category.cs | 182 ++++ .../When_a_startup_check_fails.cs | 266 +++++ ...When_categories_change_between_restarts.cs | 38 + .../When_endpoint_settings_are_migrated.cs | 43 + ..._api_is_called_during_the_required_copy.cs | 44 + .../When_the_host_opens_on_the_target.cs | 258 +++++ .../MigrationCategoryCoverageTests.cs | 100 ++ ...000244_AddMigrationCheckpoints.Designer.cs | 991 ++++++++++++++++++ .../20261007000244_AddMigrationCheckpoints.cs | 46 + ...SqlServiceControlDbContextModelSnapshot.cs | 66 ++ .../PostgreSqlMigrationSqlDialect.cs | 43 + .../PostgreSqlPersistence.cs | 1 + ...000240_AddMigrationCheckpoints.Designer.cs | 794 ++++++++++++++ .../20261007000240_AddMigrationCheckpoints.cs | 46 + ...verServiceControlDbContextModelSnapshot.cs | 51 + .../SqlServerDialect.cs | 5 + .../SqlServerMigrationSqlDialect.cs | 114 ++ .../SqlServerPersistence.cs | 1 + .../Abstractions/BasePersistence.cs | 6 + .../BodyStorageIsWritableCheck.cs | 35 + .../DataMigration/EFCoreMigrationTarget.cs | 113 ++ .../EFCoreMigrationTargetReadiness.cs | 39 + .../DataMigration/IMigrationCategoryWriter.cs | 38 + .../DataMigration/MigrationCategoryWriter.cs | 35 + .../DataMigration/PreparedBatch.cs | 18 + .../DataMigration/SchemaIsCurrentCheck.cs | 26 + .../TargetHoldsNoServiceControlDataCheck.cs | 74 ++ .../Writers/EndpointSettingsWriter.cs | 68 ++ .../Writers/KnownEndpointsWriter.cs | 52 + .../DbContexts/ServiceControlDbContext.cs | 2 + .../Entities/MigrationCheckpointEntity.cs | 27 + .../MigrationCheckpointConfiguration.cs | 38 + .../EFMigrationCheckpointStore.cs | 27 + .../Infrastructure/IMigrationSqlDialect.cs | 31 + .../Infrastructure/MigrationInsert.cs | 141 +++ .../Infrastructure/SettingKeys.cs | 3 + .../MigrationCheckpointExtensions.cs | 96 ++ .../DataMigration/IMigrationCategoryReader.cs | 25 + .../DataMigration/MigrationCategoryReader.cs | 40 + .../DataMigration/RavenDocumentStream.cs | 82 ++ .../DataMigration/RavenMigrationSource.cs | 40 +- .../RavenReadOnlySourceLifecycle.cs | 65 +- .../Readers/EndpointSettingsReader.cs | 19 + .../Readers/KnownEndpointsReader.cs | 18 + .../EndpointSettingsStore.cs | 2 +- .../PersistenceTestsContext.cs | 2 + .../PersistenceTestsContext.cs | 4 +- .../DataMigration/KnownEndpointSourceTests.cs | 66 ++ .../MigrationCategoryReadersTests.cs | 37 + .../RavenMigrationSourceTestBase.cs | 67 ++ .../RavenMigrationSourceTests.cs | 145 +++ .../ReadOnlySourceLifecycleTests.cs | 14 + .../PersistenceTestsContext.cs | 2 + .../EndpointSettingsKeyCollationTests.cs | 172 +++ .../PersistenceTestsContext.cs | 4 +- .../EFCore/EFMigrationCheckpointStoreTests.cs | 125 +++ .../EndpointSettingsMigrationTargetTests.cs | 143 +++ .../KnownEndpointsMigrationTargetTests.cs | 199 ++++ .../MigrationCategoryWritersTests.cs | 59 ++ .../Migration/MigrationEntityCoverage.cs | 34 + .../MigrationEntityCoverageClaimTests.cs | 33 + .../Migration/MigrationTargetCoverageTests.cs | 26 + .../MigrationTargetReadinessTests.cs | 226 ++++ .../EFCore/MigrationCheckpointTableTests.cs | 172 +++ .../MigrationCheckpointTransactionTests.cs | 178 ++++ .../IPersistenceTestsContext.cs | 7 + .../PersistenceTestBase.cs | 1 + .../DataMigration/CheckpointMigrationState.cs | 42 + .../DataMigration/HaltThreshold.cs | 13 +- .../IMigrationCheckpointStore.cs | 78 +- .../DataMigration/IMigrationSource.cs | 47 +- .../DataMigration/IMigrationStartupCheck.cs | 21 + .../DataMigration/IMigrationState.cs | 13 + .../DataMigration/IMigrationTarget.cs | 55 +- .../IMigrationTargetReadiness.cs | 26 + .../DataMigration/MigrationBatch.cs | 16 +- .../DataMigration/MigrationCategoryIds.cs | 5 +- .../MigrationCategoryRegistry.cs | 23 +- .../MigrationCategoryStateExtensions.cs | 39 + .../DataMigration/MigrationEngine.cs | 131 ++- .../DataMigration/MigrationEngineOptions.cs | 74 +- .../DataMigration/MigrationSettings.cs | 22 +- .../DataMigration/MigrationSkipReason.cs | 23 +- .../PersistenceSettings.cs | 29 +- .../EmbeddedDatabase.cs | 4 + .../EmbeddedDatabaseConfiguration.cs | 4 +- ...rovals.PlatformSampleSettings.approved.txt | 1 + ...ategory_runs_in_a_fixed_order.approved.txt | 9 +- ...ional_categories_are_selected.approved.txt | 6 +- .../CheckpointMigrationStateTests.cs | 111 ++ .../Migration/ClosedWindowProgressTests.cs | 232 ++++ .../Migration/CopyableCategoryIdsTests.cs | 19 + .../Fakes/InMemoryMigrationSource.cs | 25 +- .../Fakes/InMemoryMigrationTarget.cs | 75 +- .../Fakes/PollObservingCheckpointStore.cs | 64 ++ .../Migration/HaltThresholdTests.cs | 10 + .../MigrationCategoryRegistryTests.cs | 14 +- .../Migration/MigrationContractShapeTests.cs | 19 + .../MigrationEnabledSettingsTests.cs | 52 + .../MigrationEngineBodyRetryTests.cs | 26 +- .../MigrationEngineCategorySelectionTests.cs | 34 +- .../Migration/MigrationEngineCopyTests.cs | 125 ++- .../MigrationEngineFailurePathTests.cs | 122 ++- .../Migration/MigrationEngineHaltTests.cs | 165 ++- .../Migration/MigrationEngineOptionsTests.cs | 46 +- .../Migration/MigrationEngineOrderingTests.cs | 23 +- .../Migration/MigrationEngineResumeTests.cs | 21 +- .../MigrationEngineRunCategoriesTests.cs | 74 ++ .../MigrationEngineSkipReasonTests.cs | 82 +- .../MigrationIsReleasedCheckTests.cs | 40 + .../MigrationPairIsSupportedCheckTests.cs | 35 + .../MigrationStartupCheckRunnerTests.cs | 75 ++ ...tionalCategoryWindowsAreValidCheckTests.cs | 51 + .../Migration/RecordSourceOutageTests.cs | 80 ++ .../Migration/RequiredCopyGateTests.cs | 395 +++++++ .../RetryHistoryDepthIsSafeCheckTests.cs | 22 + .../Migration/StoppedCopyExplanationTests.cs | 194 ++++ .../ScatterGather/IncompleteResultsTests.cs | 4 +- .../HostApplicationBuilderExtensions.cs | 6 + .../Commands/ErrorIngestionOnlyCommand.cs | 24 +- .../Commands/ImportFailedErrorsCommand.cs | 12 + .../Commands/MigrationSourceReportCommand.cs | 17 + .../Hosting/Commands/RunCommand.cs | 47 +- src/ServiceControl/Hosting/Help.txt | 14 - .../Infrastructure/Settings/Settings.cs | 2 + .../Checks/MigrationIsReleasedCheck.cs | 32 + .../Checks/MigrationPairIsSupportedCheck.cs | 31 + .../OptionalCategoryWindowsAreValidCheck.cs | 27 + .../Checks/RetryHistoryDepthIsSafeCheck.cs | 28 + .../FinishedCopyBeforeAnIngestionNodeOpens.cs | 58 + .../Migration/MigrationStartup.cs | 607 +++++++++++ .../Migration/MigrationStartupCheckRunner.cs | 39 + .../Migration/RecordHostOpenedOnTarget.cs | 40 + .../RequiredCopyBeforeTheHostOpens.cs | 30 + .../Persistence/PersistenceFactory.cs | 21 +- src/ServiceControl/Program.cs | 10 +- 149 files changed, 11303 insertions(+), 595 deletions(-) create mode 100644 docs/migration/ravendb-to-sql-migration-system-design.md create mode 100644 src/ServiceControl.Migration.AcceptanceTests/.editorconfig create mode 100644 src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs create mode 100644 src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs create mode 100644 src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationState.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs create mode 100644 src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs create mode 100644 src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs create mode 100644 src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs create mode 100644 src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs create mode 100644 src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs create mode 100644 src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs create mode 100644 src/ServiceControl/Migration/MigrationStartup.cs create mode 100644 src/ServiceControl/Migration/MigrationStartupCheckRunner.cs create mode 100644 src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs create mode 100644 src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs diff --git a/README.md b/README.md index 86493536c0..356a0cbc18 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ the `launchSettings.json` file of each instance. When started in setup mode, the ## Migrating from RavenDB to SQL Server or PostgreSQL -See [Migrating from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md). +- [Migrate from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md): what an operator can run today, and what is not built yet. ## Secrets diff --git a/docs/migration/migration-system-design-diagram.png b/docs/migration/migration-system-design-diagram.png index 15845976ae747d176e438dfb06641e47bc39df37..85636355e726a8a43aaa1bbd30256b7e68e799f9 100644 GIT binary patch literal 209456 zcmd42cU05c(=QBq0BNG4ARr*pq$^cwq)S&o>Am;fA)wNc-fIM;N$;H~z4u-c={1BB zN+9KOp8NB7p7&kru6I5Eyf~V1ZD!g;+=841Y;dvaKdpHU*lA7LG2P?$-4>utMM+5@OE3)_6oD*B$Jn&`eZi|oH z*!)&vvlYTQGO1dSp22*=o??^kImOGxc$St@NXtWITkIjzm0!&E@#(F#kfFT_p_3Dd zvEhTmgTvt(Vq(q%xF5aN|zha7rS$_g#WyPlJNif{TT=6HGV?> zAL=Qwg83gRE`*8q57jUAfa(ur&JaQJhpLH_!T&?)ym@x}A1a8GDIWJ9>NUmxpW;gI zm#~pi^ZQnnIW*dw2fxnr-sBID~FCIzl^K_CvQI?V{i0rez)=e-?6c zkBDcm>+kMgUkt2>Ho{k0y+F&Xzxi-*jHy`HriBA&96~1aIodt;eUY%tf9(SMP_s)S0locfT{^K!`=q9;j>}JIU}~c6 z`t)$mXHBvzJl%$KMpe=otxD@@i+?fN%jddl2s$YoLfc*0Vl5LD&a8Kqo7hV56)9s= z98g$puz{|=_x_g0zqAy5ZwTCP80@2=;ahM7 zX>a|PnD4kMKOjxO@_08h%DI%yO`)5KW$nvxtLq8qoE;YyD^ceoHEYEG=AcsPolN3X z8J%N(4R?m}gVhAj--YUPYjcHO^nZKAYk2`hIE?^vASM zwH^(~Pu&U7d_?%z_p}Os=2tqpfqUd(Z6t zwIbUmDr1vOTq-jUmZ~lO-czD0s9s|Q`5#HeF8;nLYBP(GT#5bBG(zoxpd@IA!k_W4 zy%PvdmB~p>u4-Dnb6emKNtv7nd{Fs@W1Qd);s-cqK6?_Zy6LzBd=! zF!o30FC2T6u{e#j=*|q)7)hi(RB>TGoo6L`X>Y9)>)_*c(r5E+QH2^Z@);e-XrbMx zEp_)E4N)MAO>eOwtX*86npk7dskAZ+X@Q&%9{FR-A9ojY9yDL*LBSWwX;G`)a?!~S zx3KrAK z2JJpVzYSM9KY@^UF#NGpvay!J#?Ch(W!`>)a_5m?h+TN}nXeV{lgY=?Z!FOnFCDa> z`sI$Y{ENf6f zSr!P|f(BUavvrVh4=kWADPk)zGefN}2NuO@zs{{emc9GF9ii2dacY+ncrDn^A7UE6TZ1)yOXgUvPJO6vZLQs;%dWAv@$ zWeQO=T67AsU}*hXJ=p)HdEpCB`FXymt#*WVT38gq>zIlk4P)zS-2)#uoGri&{9pbI$``+uDD!D1{(@aOsk75!ts1!|`02K98sJy8+G&Th{QY{~!mrr1AHS9kK%^!Izo~TT>2TWq5hg0b zELo+X?c3_JS8n-g`35!_IyZ_3oji+Sk zfIR{=aADKa0!U)`yk#%qVts9O<0a%HJh@_Oz>KfJ|6?m0(L6Q;SIF_^+|Sl4M?Yma-U?-_!fXVvx>Zk`efP zjikhG6`hM*Uk@HTh~2Go`vqWC=FQ>a9F|6GUw+fNs!xV24pvz~Ft3Z1!ni;H5oHDQ zFGlUw+E4Y~AdyLUjC6-Dp1JsqIy7JetH-LCWxyi5x|c^t7@>P;A1h0l@&KFVse6D& zQk4n(p<|$*c~}*Oe5(S*yss$ypCcovg|+3g-{L4E)AR2YR>$lGU0<6)q^_bTngA$PQw?KLWOTSQLC+|;Q{$3BO5)&jXV=eu8>xNa=(a&%)TS2=V_!s&y#SOxJ^j7@}{t#;|hCKIW|EghiD^(9?TT34UQ=Cqr#_)3jgS z=W6TfC9etfCbMOASK!9Djad%xdiXJaNR8l@X!jXO;Q3JWD45L8ueUfQB}u-{Dh0~M zbYe*|Q7B*|;tL~CfZI!`%<*nVPtZ0GIOKz9mG4rK6gi7p>8)+(^oelmb{JR1m7fv% zgvJGu4;7PDW@Q`=Ftj-6CQwysw|Yj+I-@eZEIS&rNisSvVaK%1!)0!7Qjg6ysZ9~G zkZfVQ+LocJnCL>lMQFoQi@v$Qqq|&h=BpYXK)ge-O=|9_rwI+&%D(KnnLgY+%V5jR zJ|!WLBaJv_vvF-6XKD(&5MuP^IG3joea#e5?Xxx3b)|CnrZHixOFE#R%1aAY{T&blHn3JOwzgs*T51@#=q#h%@E2T=jR{sxcc%tq^54c=q9P9NaxbHptJ%^r_p@ zB)|_ee%InV57@hQe0Mpi*i%vD^FXJp+g{{%tRaF-KuGa3uz__8j1Bg-oEqs=Vd(T+ zm(XrF-JGHg?j%sX)lBu0<gwaaFphqYA)wJR^A|JL*CPAhxZ00*0{=p3QSUOE|l=vj(|x&Q4jFGnh2*n z8MYUCjUd10XX0e`jIttq5@0dT;q--`VhSQHI{n%#mV2GDl6zOggf!y(zH3-?rySPb zv~D#lnGQFU+>svZxz-o9U@GKBwFa|`ZC@0f4eM|vjHFYZwWoQ+f~x2V#BgRy7ru{< zZLkLq)23I&`uVt?ijKLSg3aFh3_pe%ivGYKfyDr=YO7MxX#OwL^qm_Gr6inti(y1gwZcxVE003`de5id?9E{7|J^(xTrLT#az-OL1b=Xz}rUN~!&h zp#E3n;K}FPd2`tb+%xF{U=X$}^KIJQ`;y$k1S6Jl%z{G|0(`-pQxR&>yK!>l`BIKo z77lk)m2`Z3no6R09&>4mu`UH^^imS)O#e{LwXqjbs=0vGT8=;jl&s23_6ukvlh~7& zIg$I8feI8IFI@e5QH=K)b@0=<+<4IGJM5+v)`?q|cqDt0fuG1unqx_9!BN2Aigvx- zlV(uXWPlC>@$z{@RW{EUIyn56bZ9sH8I9D(+ntSL-ae#_o2@Q1f-Usd+)B|2(|&-o z*3@fR-N*LYiTBokY#%yJyS`NEP8!bcoRF%@B*oT`WN}*pi471bn9jtb^x3Y^G@ouACW!&?Nmb^N6xJ& z4e#D)Y!z9hc$^&N@yr`FFCh_|A#1$MLe*;CPYLh%gDkat<|^un(sg!w?(W@x>v{AL ztjx|5tG>yw71?Y;dGF5=W}oMbw3wv;ER^= zU~C3Y1#fY*$XZAFZuJ}ORnyfJJhK5m+3lq)4A2yH-MDU_b);}nv(5OC8zQnvL$&8A zG4uT0{{F7&lwq$@LAnJxt5p>g_=)-F0NbHo`DP)AotqI>N3NpaNYP?qL+5kN0ct-2 zKHNXjOFv*m*Z3DEC$oP6&hT*#92RErf)jtHZhr7JCj%34gX~fU;A_k@Qp-F;-vT$d%J(iHN)n|FYBIGta>z!B=MIN46RfNH~I!0kH2DXg>!^ePCRkF4#=Rs3qbsF4ib<~Wmnl6E)wm+bd1mVAW!RxN#eNm&MV zusFO9vtBv!+`dv5D{X0{EVB$KrjePC@l8@mWpn*eveIikxNe4Qds?}<$sL#U=$xI0ThnJu9q=B& zoCLZ3W)n`z7IG?Df8AVSdJ91kKATj)hK30=OMAWP5*#{K6o^X5AK~(I53uMdwdZ2; z*Pg5aF5)^8jeX#)NTT&sBPun{vDIm zOOwY}$=Q%L_eG8NR&h+PCrpusBfuRV!0RCculNjEu4sR81NNx=C;RxSo5jxU7I8QS z!ZNlPB`qNnYX&lidn_D4kD+0j;rm(FQ~EHz%?>fSMOn{#?P?l89wz{Na`Gw1ux}91 z=OBY-KWxZ17yqs|V}^FgSdeu>x(PSj$M*W?@{hYt2$<%{wFuw9hux92na+?TgAMg` z^fl3KayR5!!>azf>)5&I)mAU73=!jT4m4@XprG;a*YoB5^YzyH_H(V4=3pMZ8ah`A5I~s?L$SU>C97yTPEylj3t=#YyXwTZc$)y|FRDezA zfK?9Dds1H-$lDE>iQX<9IKCpbB6CCpDX>tjo)+nxA7#>FGas)+(M-!y@$SrQem0yL ziK|^m*xZ+|v)+Jp0SsWM>JjkJ7-Qe~duv`Tc=BL^d3?hy2lf=G#RU!QxYM^;h z%x{vu{Pgj@rIXX~!c$n)?Jc_0zQ3%ur^Sa<`49inG7f-Gx3jw8i{1Ce6bVc!-Shpf z=tv<*-TN0t!E0_ZW)uHJJpKY$)qdM(+Rmml@ELh$a|wTLbJkAEQ^eCknxklBXX4>} z644cMQrSJ04du3a$@R2Zr-@)c*?t>e5z=woUq=X1Io%MV97?kb!I8Bo$Ne-{6S!x_WQHv^_EOuM3b zYJ^b(3r_@@ewz1ID0!71!nv;Wv8<3;(w8rR9vMgd0%xnt1|Wkj!?T==sTz}yy>^=P zzLiVCPMpulW;{`me?v8TS0!G;ne@O<7P{?yJVw{LP7%!UYy2xgZ3;_jI~`e`NOcRA zjh_0zHtL1>Vlj34?f1SgIhhbPPWuBGm*G=1l@U!+lTpVo$;t;Ps#J?6!r?tiRLHX0 zvteex&`W^O(8TNKz;hhI3NZ+L)n^>d zdQo&x+a$I?;9z>P@3$pN zC*}$@9tsMNd}plX=~7>>-8MzD$tYN@6j_j5x(hEJd73HP_#W;;+>p)P;i;GAy4`TK zwrljEtz_(}Tdm5D@aFo%t_9a~xZ0{A-0z~~ekgz{pz-MPTrH){Bv8LOPOq%PMX#~y zq^5Iy2|_#a2(D;&*Psw%#jZv9&B^EmK9%`@4$~?s=*yQMTjaZN=fzw1B1q1H{3*@; zGlp{yU;G82>horjeIfa-Wn~_5&8UjX5q%N zzu9jfe53zU5#vh)NkhG*6oUE>e0_%J`+sSj<9}Mkp}sFVtGv7TP3N?iOweKe2zO29 zkA3!HuQv`?z)sK>m(#16ecMOXTIXppp0?xT?a8rv#A9+~uciP)H^Y8KD2abndEUm3 zCvD#*V>ifX{8kA^85n-h)pTwCb>rUMqw@w27JBO6+f}~KkeJoyYW6JOXp!^4Ld5@A z`7OP$fc3cY>fYKKq^2HpBS?wTwW5TZ+|SfV!8}$F=&rQV282odG-`CT{!$N6b{{;CaNbRowf6RBp=62Y)n_cij(V!7hW!#co?? zBw=g-KIVQ3&sv)rr11Z7C7dTuKu-$>CQtN)VCMQ!_}zD330TiBavW`)+4VpD2h$Jw zoNuF^!CAnjk@1Sz^9OfA|6#d5wfHap%@dzLy8Sw`u1P9XwpmM>$8CM{thN4VTA%A@ zBI3`I?g&Zns-#bLvsK@&2gx`Vx)%ta+07~b(G$nndN7Ga)#fT`x?xf6nUKwF7UrTw zPWI0fL0v`4NQ4c{R>oFuoggy>+VS5FjPoRew;)0zLxQ=!{5-Lrlt#k&LiIP!_4hRQ zGhP+!YN#*S0wR>N`U6=1tRKhJGvQS^`sKe0d_Nrj=Y_!!doNti58`1FjE^ri;k#6< zK(uStMkWaB`AmHMVhqImk5X?z0?*E*t|y4M08qwpgWci4T-UR7hbXzxB?Wu?A6B5z zys&p7-Ll0_zDPd* zmLB=pzZ~iOD;-mOd$P)I;7lNf-8`;@Z{O;Rp4!25KCzq&)kYS*m00>wrlad>j0MlZ z)3;>q={1On$z*zmS1RA%%?CBIcrff5wYLg_rUi1%jA-b06^)AbEs=Kg7yDDX!8gcX zdMjVzZ+$dNedAgP^sa^Etm6Psb4th2ltIIDz#)&A;2y$ns`2_T#P=aznJfUno+M$JSv-qB#X~k z?`j+;;nZv+kr0&$lW)&_v0IHp#;o_s;Of{THu^fQ9Rc+HIZ5eT=TyE?w2f**jWh~Q z@?ZRzFlgINLbVgiQkh5af-f>YN zbw#J+RJ@HH!nPB*Qd=&}8-J0A$4ev?Pf-BQZ%qI<7|+J%uKyAoSvfh0d@vHwlle2ez$#~kJ#8{ zD2rQe9V{-PVlA8mv6vqDtM?)7;C_eTp%4ve+T`C@012W()*IBl_6x?))_m{*=yXF&5kXD1-b@LPz zk(!ghDUNvVaHpwGdNRXph~nKNpzH916)n4Ypt}0Q=7dbQF2-@al7knn5LT<56Jl3a zH)@~1u4+a{dvqu(?w{d3Vvy<>H`}p$_faJT?@q|K%0cdphfvSsyGKVczUSEjkWrUP z>>JL`gzP9~LQAig=2#Hw_CV)8@^*iWPc294bN*O0#t%dz5Q#%IzD0Vp;_raKVb{Dd zG?N2F!>pE~?V=L*qoRc|ln{RQMUU+os`wiHu&?}SjVv;T#Y)M1u_p7kaYpx_t7$r=OFF8I}_|mo`bS>QYVY$R_l;zIB{FA}rV55<@fM6AY>LvXC#VsI&c!{fHna4> zHudKrxVX9wwFs1pERKB7OV6hAlLmhs=Jz+!^ao!I^N3AmieJ14-0uJA*5tKB_N16e zw1f5{aXe8syZu;!o65kgJKOEl>yCkpO~+X-z$@XSzmsWzmMJ&c(3ssk_!XyE+!Kss zKeb=O+Ye3LD}AlXI*XVw+KuCn`hXKRQL8{rep}4x$H3hd!hQLH@p1z#)n|I_v+MPY zMkEcxUGBnU$kLd~q@4J_rZYOU1>x!+#6Mc)`DC6?gL~QZX_Hy^zJoD~H zM2YlD4VnpfU+L*pF_}}2K|6j5jvNd1WUPTD&srbL;w|Z;#)^yzz3s_Y&#=$xL+I0* zoWBAschz@ObtFrE`^R^cfH%*`1@D|>vCdV}CcoNW_;??p(w+zM!^#VB-=?9wKOasS z^CCe0%Kex?KM7`Yo*o#mJPrUVd1~} zv(>zPf~)zfU$~w`f1!0E^I2pjd5w zL*M7aNue6$8n+7gk8^-pW@DC8f+x-X_5PxNioFM*%`&4~yDqlJ;~i~UQ8v&U8FNmo zllL}NeFG6i563_krtY3rQg+O4YBBIVKDv|^wNg{ss*TR?lvs3r>SZ!^^fqCPV!pDy z@~TpNSqg}@hE$@w@2tI7O5UrUn@=OGttUKYL*M1$arR=i2+a60H!Pv*c~W4?wurnP zdC623nEC@zX<0d9MZVr*|0=+{YuSg!96#m-t!s{8$Q3F{DVg@i&!FAd zWF7-{LD?3Z!G8IHL7&>@X)bI#Q$w=PuyjsOx%FS)>ZR;JEmtwDQ0~&9DSzJRjtU zX$A)}WprBM=&(7=-kvnLxQC~`{$f#%?`3?qTk8m z+uA*i2k93}2{5Zrk8pLt_g6cthDv^RqZG|*%WOi5TaL){%WpXWZJ+ZEz4?QpE#CQy z$O69@Q)*?9BvEJ0B0Yab3KOf4TVC^lqQjjX0l%<0*XJRXn^T2~UKt~koAK>kZ8~(j z8CN9LB`jB?6T@zXoC*P5FOYg%#Nevc@*=^u8Q_kcS(x}lk=g$F*l@a^NuLrlPPfa$ zVzcGzB){n6c)|61wanzZ?2bIv=7aHHOXxUKttAwc>GUPS}dA~YmU<%b) zQLNA^XyPG4Y4e$0DRLu`EW|^nGj!d<8X4nTNWAL%i9E4m`}d(HP=H%_Y`C0qU za1qd+16#@2)PpUt*qPxX8?M0yYJbPsA-3*^6pore+fK4ds{LLuohkg^IU802m z$oQR>-N7+>sPgy1ybv;f$IjOu8u;`iGWZVb+YheU#cxH>UY_Oq)Ou~W1}JSl!JKB< z^)+#`Of5Vn1`loB3s`C*u8dUF?6|#7&qy(#e{J->^=KRjGpNt@FK2uIu0-V5@f-JaREhw>~ARC`QxM_Kfwm*p#|Xj%;h& z-93j(2okTU<=l3?eNp zZ|P2MHiE|}{^Nx-Hy0Rli~gWpBhoZOU~5)^BaNt=H2XuW7bVTiMUgC&1orCY_`cle z3lz(^Nz?ex?;Lq*6mbcH`x*6A^&ju-&m1!_H@jJ4izdl_DqXvq>86%U(Ba;uL{Fcd zEpyC;E74Ti`L&3rR~;l`5!zSgLABWML@289ffCAhbx=9)+wKV* zXd4LgTx}^bOi8mjD-y-`KTqn2%Tgm*Y#?+>;tQQve* zYpv~>#9fT&93SStP&8DUCT;sc`Qz96L5ceo{Vz{%wX6JfR%TD8Jo))_k@nLSS}w|y zS{HOBvYHy9c61sDH2LODTs3VgU!aFaTVN5$nW#SS@O31jp{A2(z)lw|XxPdPlRj#C zM!Mmr3>Q1Z6g!?9dC?yVMg#=Z?oS400rS_lZw2S{Ulqads-P3YYQMwK__pdzdvADj z-7%qy?ReFO>&N#XTbd`U8`EE(UZ|-nNLvO9R2cd3kTDC%9N#qlB@p*{Rsy|_RUKIB z(8XSqZ71GA3G|hX`q_!+>w_W7iHL-bF>l_W4aexm_6a43?0T1_{&Rj0Kg;!?`&U1s zwjGq{sb9MSzxeZh1UdXb_-KTBAl4`iE4US|Sq?XO?(D zdGBJ{YvbwK#3G4n?VyVJ%{^l5Qu{HM!hT~`79e3a3CO>-LQRvJPJMxN;>%CkySX6R z^aZ|1pL9EKxsYqHD%logCV<5HHRlej zG4sSwY9~6SL1W9+2|lplrFn6f_xkY7%@6C@u`Y4#>Z;~+{sCDM9s5OBX7ASd^EP$6 zsTkwP0{MvPs^8ZpoV)$3zjCiH;MrrNReKsYxXmH7)vx+#)chOqo5tqX0k6rAZ~GSz zu>9_Lc2@RGkf{hHo4p+*vU_x56Gr%ZbLO*skBr5CQ#d%q-2bL^3bx=jF8=okS+@Tvc7lv%lbRx)_%Hg!VOI3>;R*k|0>CN#aCKc zatJ|Z;>?3WN$KCVG_^D}Xf`z&un#v^yo(={8kWuuN+XarOH3int;>9;=sUe#vV}#r!KkAbv@YQ zw6`FyEmLUv0uE)w@r=OVF4q&{w|&3ftFof!3vDSuwxTGi=}hVNh`PmIQD=7h+fS#3 zk|I*o!`p>^V*$uVD*CZja8?}8|FWS2jWd6^k28Nmq|cJ8i=9H0XFVd}Cejy&LxA)* zD5oi-2j-_QDUD#>T#ci?mQhDq7_rV<*74T`tBRjfLb*weO_{sJriK5Wxc{8zn!>F>7((3la&z=OQ(r{?V-%?o*uq zx0%^#PjNgR{@XO;4KLLj6Z;XiL3UBIbDCPtT*P}%sO$LMZE*BF-Vwhf^Ps%>x=Mjr z4$id7e-o%bJZI)18nLXrD~B!Z2z^(PtGXtgN@hmPu$8MS{r8gGhi^~6`)@ol|C5vd;Rs?gjxG6NY1*)xA&rPMRveGN z9rGXe%Np08_`cMo^C#Y4*ZfSi17WmQ;5a=PAY6 zhZFdLu}}=o^pm>t+)+E3Cu*IA(+rp^fIw>PU0|*CXfLO_hu07x8xL6JfTa9;p$r=- z%O>U_pQ!|{{3^Mqha4&~_L42D830eAJk;Yi_awqCF8Z?PlgBN7d}`%JQSdOq_8l-4 z_Q6x@;xpsRNTlm~2!zLS1lbWD6m2ZjDAao<<}g{r6L?SnHwPFOmAm68y#X%k$UHE$^zG7$u3(;1_!Lg$ATQ`bc0{xbFsnY< z8lM9`dtY~6pKHgFX0g0hp6|<#vGc*&JQ}fa+w8H5nqCCBawIC2^mIzUmtp117U+hi zFLREF4~4uY_p;FNIxJ^L7owrM5(0uUXgh#@9L_b<^eJgF1j9uKw4JR9EntL*E84?b z=dJ?facqrz4&m2)?}^~5t7Lx2Gf=~hC(vJ=8dvX!qhnF9Z%$x9NB3}6XgOj&xYF>D z<~w9cTq0tgWKs6*?eFVPW+flPa^)e>@N;h5YYK5U*m8U>QD;RAM*g}Z4XCn)oCJuH1B&tUAZXez8n9l zI^_SX@A|w1TUOo+f4?hqX1%|iD0{RwT9X3H`TMjCQk%`mj?^y1i6!9Q82{&od7QwC?1Vl9FfP^v=}^;>i5o?2PDRo=SnGVz(yL zcz`~dl)1s18E)4AKSrCx|AMG(AXRdl!4=$J+Xvl-~JMdzv?5m4@xRG%)2a-<3W4qTJR;eFgYX)K)3T|930(MB6#>OqU zvBCR_hSPbA&FoCH#E)0LZc7Mi&&`i%O_fyk?DWf%M6yO>+^>U+-ori{X24GL1pcwIJ43rbq+u30I7P&;EVX3Yra+v* z=I3)=8@I5C0Pw?1ZKOc)j!AZ_$=wUmdaKy(hQ=;XtFNh>!j)B2+bV{rDe{Rw=E4w?@x_TBlavdh^mn? z`m%WuGfdx9e~Oor{KDfcK`e`VL;-7usEsAZn>lRgx@h(FPq6AeTO4G2Z&mOY!xIhaHO)t6Zr7y32_nHe zfgi+?Z&Pgm`+YG}^-(*+v-QKcWegHw$=^TfT#f`Zly3+ByfzPz3kRGZdTSRtp);!SohJ{?^{B7qP67}Mhe@RRQIvxT1UMnQtz07$98?vbk zD0QIk5QPnR9$2dbk<-0^M`j){>0&tzIZr3aLkgH;#xo7K5A!4#NBJp9#x8CBM>j2# zHii>HWGv@oYn>QGO5=K}%g1l%T`zXf6S~f|hK=b(uSBA2ZeOGe054E3j^E{_7EbeQ zQXJl5H7DP+JJM3``qf3S$eo+!j_!h&rTC6^m#vsSxfUbw>cF`eG^^$)?(HH@KP4=R z6yASC3|01LilkE{2tdqEt)K^z&Fn`RQzOC}C#1xSI)n-W)7EBnw~CcdEXSeWv{xoDp~C< zstOwf7$7C?}`URtMM13c0uaeudDzimfQV+jVhSzo) z8?0<|XFMo*4rt7_h*Jo;7MOmQqk+Hoa6UIqReL_*dL%>T-w~cMI{HRSh2f9!}BcMMXq&lI*Ir=lZ=#E?T zKGmMGW6t7d5?Aq6Dgkbr?k*yASA_iPcdm$V60MZ1=!*O3mSah}cv(RwSn~gXCr# zKDj-6E#JI{F5+xUFV+XLB{cOxQ5_)`R@^FG@)DamS4$X1JnG5r+G}FMF*iJ8Sl0_Y zsDv1>%gO%HVwiD=3j>6|KBGM_#vOm*lYV!XG=T`noD0<2oj)}N_ZP2LOKfXLFg0f7 z%-qdxevQa_e>FwsEP*-A)fZqmr0r!X^A(;I_JU%_Q->n#`ulAN-!1FAmb2w}9VF3h zFcE7WpN!O>yBrdFB;;Sn+zl|@%j^<;m3T6Y;4+u$#0lJ9gHn?DIEec3h}i5GwcpAM zle2LO$zpv2D|>ctGYStY(%uFS3#pH#u+3Q8s%NDv`dF$D!e)d%L3OPL7(KHKaeLFzh!?x> zRvma8d?wpD-@lxCCfMk|$eqfL8qqN7QPUY6*%CLKnT=<7_qop}uvJ_8AVfNfJcLz; z7KwUmc`*K#+d&e%l61&0`VG4aS3uw;19d);o7aYO9TfnG#N%KOM(@m7n7lQIvo}R-ifo+%|XT&b$x45-F`^ zwelJ6?Dc!y-5W0jEOaP|DY7gU?P^^X(S4uyOpIzV z-4M&6oX%%1NC|3BXyD#uUhwq?KI7u^vz$&#?U*97R`W`UWH;HSk*{B$_a4=6H8Hn- zGkr%YrQ(W~!5UpwNZqy^bDOyMU&1r&MSyvCczu zPV9(eN6sjt+(~CFqgfi2ZSzZIvvfQaUv@&V=BY*RI8(*w7sECW7kZx0v5)0DkaC}N z()U>_QQqJ>|Eb`+>V7uccnN0!uePU@=fu$* zzG;`%N>W1Lj(m{nB9Hs?#05JN3W?@V6j~EM$r3zQseM%L^@!c~HFC`E$i?z#vzCvzo)n8&yzFc)T~_U#!#!0t?tsdKgyq_-Em0=KK^c zOWDy+E-jR)XFg7EIlRuzjsKi;f#lu=&1!8u7)f)Ds5KbSFt)k!Rc|n1(S-08kx@)< z=aD{h*^%0cZA5oh5<3d(__V&!L~*eIa8#s&R`v8FLAS_dV%PnL?r8ly>3q3gf&vnvp{_1*kw$D zSP*q(JG%-J^hfwa%CKF9R#7v#@60kv_gG9BRL?xjet0;mW8`i@{q9vQT_2kMVB>`x^3PJD$2L(B?|$+Zoz5AG-N_3zcToHStuwjl zG6T^qOz|K}bcU!)3r@e&;&Z7UJlcaV)RffnQ9dLSoVs}W;!=`>5&os3bX`r}t;pld z`QlrN;b$gSApo`3X8JWVA9aSCd`Q+EuslCgG$GY7S+jVf9gag2WO$JksX|kqd6^(3 zS<9->62<)JaPDp2bAQxjg}|m&W1FBY6=l10{tg>~%|w9X9=+R{o1hmlLx9eK4MyZ! zbXeZzc4SUqQ$n^1qK|mN7g(Ii!=*40F;;injNQQV;Bkr=?XRY77A6ZsXG`vlBq(Y( z4#nkpSVhT&`uY+~nA+>K!KV{*Y3IusK$RJjAqJB}$_uJrN2esTd8T=O0L=UKh8XE6 z)Qck>*FT{6#D4Tm6;!`%tmW>r+}GRwddXI?98JBbM2VJe+LG*LhrN)op_0vzFvLgC zeR&4;_s@8r)Ws@sQ)Y|^z2s<;n5@R-AIX(ymiLZ&2n1F*OA(Ze_D`Mx0^6R871Ar% z*ey2Otmr;pKJ#lCHe~$pE=tKD>0@rOUF6PB-wa}QOph0FLTiOzy&EV#haX<+|I=$$ zLb>IpzUv6cEUvP+`DSNKaAP>t-=fz`!FF@M6WkJ%Gf|w3yq$*}R8qU?fxn!R9WqMu zHeYVqCg_tvdeQNk`N;UKSH$RTe-TL(bmqfO&s8UmZEJ+5N1epJfb^-!2go?R}B1lgnYlBP*;r7YTy!?b(w~}M+-+f55|=+4(T5?& z+rHcIxbEp}=XDIn&y@l;aj8sw{jI)D&c?M|$`e1wgSgP(<>mmD6Em-AEk}-r6^DjE zgeXx-B+-Rku||`j4VI?P7&DELl!jv4!b^j*38RVB>x5ip3@6GqQ&r<7A6*nwk}UW_ z0@*nfkg&*Bj+Rr{w5uICLnz&caXapf?;^42YTxVl`?M{!deTiwzIgI;YYBz{U;V0b zoLfWR&cqsC_0)|Lc}&FDp(%+}zcaGN1f+?&@ajQRruG(+KP2SfN^A&mDDJZ8O?-J}kg3NTZ*j@798M;Z>7S3z=y9wtrc7@C8%Swkg z)wK8uMm@Pi@C2ViUof^STc!vH6n{`$SM7t4PLMXsp$s)&iFAwB?Q4SG581o8#Xe$9 zOMnsF+=qnyy@&pjF?Gd63{z(_%BlEpxl|wSvK2^Sia- z&l8m=>qO9seQlTnC8V;!Y-Hzu@phI$akk5wCm~3X5Zr>hI|K+2+?~M(f(Ccj;O@cQ zVQ^>A;O;IHf-|^Vke&COJ$3%IANJGMexI6}n&*D*?*8@FSEG9V-4VW=V@s#8A)mw* z68O9^@py7o_|EFrK~c>H8R$qw*-4K3It!#~fQV!gnDt= zJ%S{y^O^3_Ukaf;^OKekydhTo>~FRZq*^3=971>7J@?2k;}UHj!MLZKu`?W4I+myq zf^Vz1nxj`8?&X^7c3%0-|ODMwb5T$=y^5SAX;5j?X&R^1M1VJ-CmZ4PSJ}pFfV# z{mGVnB~VU#ld!sKr10;%mTy{G^$(Hme<#(8i)3ZyOMzr|Vx0blfTm2eUoYI7gQGj3 z$GXndnbpsCOR7oV>1cf2WXk6kC-#ok4KRb`7*{=)3>G}y)WTGFt7L1|o29WwV!_Hr$VWB9o^Q`)bF zL$`I8g|$paSh>cdx?#)i_wS9UaO?eug6}cf!!~7O&AE!?N_he@S$%SweKkOL$bGG_ z`f5u_Dj?G7PERL&jMs#E%8f~-2 zW4vrTp_StJm0pccafM*U0yNuw(J6M;{gVQoD8Enx_0&P{EWr%V^7WMpE@G?~C(I@E zd=2NM?1kuO&IrDcJ)^0)jW=~47UB04bt2dNX3%0 zcT!xDSvf?E@=Mg>hx8^lf1i1k?y^)Z+r!DlitEGax{;Zo!BL@kNp-)}WYhlA_a_TE z7$R6(`kR+AsNFV8Wjl37Vw+GKpuhwF1`Fgbg!7?dg2DPVZTb1)gDQ7nx~CIKC~74| zNT=l?*LlQY`sTsMxuIe|5m#{q218j<69CKHqaWkF17Pp0C_x>uo zz1=;#$r_N`NVy+AX>__~)@Ayswkhmf$i2A-Pto#=V(Y zC`2W1sR+P6VZ4S89e_v#>}oIpI;For>-Pv*V9Q# zXv>O!S<-=!PTMYQd3wf(w@X10yDH1No^4+$cMU2^q}+1qUCd9z2fBS->8~8Sr>FCe zn(s-3#orznOv^OJ5v&;T3NwMlZC8>p$rdUk@Az6Oql&{4KLA(3V?V!Eziia=xtWul zTm-&ziAAv%9c^&yWFwlC^sAX+Ss%s2*C0DtzUz20lp2>I6r>X#O~xtl;dO2vl5RJ5 zi;Fencoa2f8Px`Ph+Q1p3 z_N=wV1!nfuD9>JUKb32AM#|X?C)*3ml?5(t=|x^vrbmvL5~9iyjbRZ(=YNy45hHI( z`A56Uii+F%Fvl<#GJ>^kcMo~B3~x=R$pTcy{b{?pXQV;5JGy~V^^~rrb%)SI)Pd{h zV*Kp%_3gLx(pE(5J}(r@j?3+?ifazhPfpzhLi2@QCk zk$L}iiI#QERrg*aS1AW0WycK5X$PWn)hz<>5y$*Evbgq4v77Add5Nmi7@nIc5nw)7OADhsF+(%r3+D<`jU*=b zQA@Sah8bV02w`oYKs;lE9(1s%ydu=%0PQE-AT2;LkE^Y=c~t|UP7(UlpRWy6%<}ZF z*vs_J)(QWUkAJVqZDx%5n_o)fVkmMlFm-2L7^pdOy2EKWVC#PZIg8b&cXuldm}fe^ z&7ZoB*mnNA1z@;VN6DS^^=e@fsof1PZs{Lcyb6)WPO)-Ne|c5e@oU+=CrLMZ;MzA&Qs(qlN(S+GUmVHA*RKrrM-by1&sHRkj#2U z>V$En-w0#hSr<+aHYfdkTN2u@>HB=!>U#LW#QmMjdRG(exkNs zpyqB>t5;W(GZ;jllzjl8#aG3>tYUB;Z~ixvQs=_QFs8Eg5z8XtW(M0pMl9C;Ak&p1 zB7ps|jyzk?ZKv~QilZ#z7f7B80{~%-al-B{RcrS!7q-%KpOTQPrIob(T*s^y9!BAa zAL14A93p%Vadc1I&@nP^sG!=uhQ7yCx;0-k8Vl_5nqFcjcX9xKGmE z(iw>##*^EfU)&T4m(8_z2$lPD7;cD>*i==w?4R)To?KrW^_Q-Fmt=DCk0dmTlinlR zV&AWYduwgdC2;@li_7av=hX_KrtfHH^3=1pI~$&cIhwC5!Q@*jjrpr=apJ@+^}h+C zh;U_!Gus7$?-?Gf**J$)_&XCx-(yKFzbl*ic&O4%(Ob2}<@hZ)Fn$*G73Pri0a@0x zjDo64He~ESEMSl2WNuZcGKox;4A|}>_y+EWE(fQPWG)`*i2xG!{3o3jwAAZg#>)PU zLchCrYR1=R2MtIBgpXyF3wivX0ykCiev6igKZhc>DD|$MK#2FvUUXdAU1j?F{75H@ zo{s&OwurvKeWs>P$PeYVgRnBHiE8K&CzVu55+lInHFH~G;;ra7Y9)akUv}&{s>!sg zi&~>4nvRYqgs409ff=6Uo_Avceurdi$?dyLueV{wzKi)>#i~M*mFok1!%SqH^xXf1BxW!(ab@8xXGK zyo*cAO5^tQKrVkHL_49RKm-Yzw@kq9&t#&ud#r5z_7))3Vmf;arRtEqsTfJCkvz6m zilj~+^h;>#YFQ+1TulcS8z$j%3CwgIGM#!mzS8!`Td6IOys6xOcyfl{tM^Y3lAVCA z*#A08KrwE4OSONQS)5yHVRd4|RLj%f@oAv86mi@a1O#AwsXj3n5`Oz(!27<^%7#ev zra^TMg2m&3g|6aFgc&ALk@%Tx)Ae^w@C{M~`gEg>i_T@DVhHZ!;u4E5qyVzAPI!}s z?8NCgfT2^Ap{>n$H2uh7u%@|)yiq(jL*+@z)d}gz{-W`~2<(k0*-oWCrsXL>;h3tXjQ<&O zb$r~iS0BJ_*!;FCQn#eDYrro-HUY`j%_+G{pr?7?^4NzqtEfi4_GaPp36#d#JLX*2 z;xv9AvKmvyY3reY)0{Yy&T@+L2Cj@HV`AfMT_Nv7lI0Zq4;^0nwy}zyB#R%b+a9tz zf0RTcpCBcKrd}B&8Q{$R%j{B@=jLlj`Lz(rz*4|Xwg8x+>mma4_|+9SA_3{SH*?xQ z42#scjCFBf8Pfo}QXn#M9&^&3UR40IT~t}vlI_w%v%@7N0q;pPpjcv2%59RNOcL1Q z>re$O5(){)1v{pT0fQN9fqnMG6zus_y==ceIcCwPPK^TAwFHhzq!#E!-g%g~X9RFN zP<^Hm^wptaujsgTv^YgPpRMv)x)~Orc0`&O5F@%wssN;}9D;sWZd-Y6m%-$kE8o*uc)3EZcSU+{Gomgf z7hHqqJ9(c!qYn?O7`8D`*T}bHi)5}Df-{LC)>Dn8XeP>N6m&p^1 zajsU12hF1Q25mIYx~87qaUOhQ)ykJ$1a3v@s*l4|+ueaLkFzf+~i)YVrsNHyhKVMqO0Tg4O=p|7{Hh`+%=_5!)OL9^vJ<~qE z20l(u`l?j7c-<0zbh;&ofcD#lkNxdZ5+N6?F|>I<>U;8VKvB?hi_lFCm4(NXq= z4ddIcDn6x0NXgTGE;-_tADh-|r?ZrEt+?`e0V@d?tP~+gO`@yi+K~3MXzDTgc_}dY z;i|uXRE+6fnkL`;mM`4?^0&Lv?$ZHk9uw~KW5SL_fB#<(E^QB7S!_77l; zF+d-cF$aHxKkK3fV52 zBJ7JqmaH=;rUfsiP!p6^4}kxaa7 zbp7KiW&dI@25v z!^TK`#n+~pzbr{+k7Yo=V@rK7N;SC8biYse^h`+A7QYj6RUa<=MFQV@CE*9^BpDvd zljv48S$SG2I(7H?*XoKA)^7(x>SEYBOe#NJ6QexK)g1jFZqOk@Y(_#GDQ1GLE~>lt zHzygB19Yz!*JQ|@{Z2gUoVPwyK(OE3qpxOxpUi$H4<|fo?Z$veDc)H@SyYE0Vw;td zl4B|MZ#6*M^kXiazRiKLxsK0}=PfExCxuRa`XaEmd_Ziop3D&5+moJqU~q_fg`|h) z%VLn;Th!3BG{{PMIbHi;*kF8+{sY*WV3jx5UZ3*WD%cwNP}hdG*qFoK%e~Ym+0N@4 z$`Rq4Vu+Bf-P1{4A{eMqzAP21T}1X8!oEX9OZcb!GH%SQ3~pwJPoyxd%X|gn9%?lQ*vi^?ijiCt#7tfyuSML&E}H- z1)F8L7s5+&ws_@NtyP`Qm!oT&7gwA1Zxn{>+#OsxuQ!u6tS$8I%UR<|+1{b=65)@5 zQyG|lIdWj|EWXzZ;SWwIEo<5`8 z6cTcn##@I_2EN3$%lhaWwm^;!;G2ZIXRSzR&6}TM{T1T=qk?c=Rh>Pdrhle31poB1-hU%_!VY`&@65YRICU zQJF~Ug$$8C>tm(6zN0mpJ+tIgdw?ht98yqE8g7EVt*D8<8M`IA`+!F)myd+Cbwrmn z*no?Jge|P4axZR<*2ye>7f;$5QH(=s8$9VkD@mjIQx0pQ3jfpWB9%UAvnz6U$#vB_ zX{PT3sG2XA{oR`?Y&y;=E8}W+%t-<{#uYR zHg2^^q6_@y{(HS2)z!@EC4w`OxNWD}%--te3ThU#LU7s_9pQV-y!RWre+wJpKmO)v zwNt)axyaO3;*Hm!<8!`92NY0>ioLl2aQmi(SDR_6$yYXYbCDSaKak<8^0HRJz(sUlJv+FvoP_ zo5ulwCb<|Kb2rx za{j!(M@y>^vayhZ;sFjhi)dRptXN=CD@OjUd8(xhoP01QAHuVmn3}fhW|qKkQ^-!0 zN^rKvJ8l*kMILC*$jy72tpeQ{DdZ2A8uXd5eylQroAquD@qvXtyy<3iyZCu4KG&D) z!msSdk!=+pLOq8MVAdkVfA2G6$Pvu_i<3mxuy0sF@gijRq8?naMI5 zlO!G4Z}uVfXSin@y=0FeK`i(~-9d-Kd=6haJw(c^hS{25&)1l~r?E#I6lAIjZttHp zeRL+sRxz~~(2vJi4JsltWhv-rOqOw7Ni-GFzliaD+@rh`RQe;{sE!}CME)BydhIF{ z;o}){N_nf0GB$p7bqND4Q2LwKUPU-@5)O5jsxq%Vl{g5h#8;x4sZ%Y)w&c`&upU+= zmZ`WEIr>;>HkNB9vy1oZ1HV2N_*{+fy?evwIvPjm=(}^HHj3mJ0UJ4Xs1q4s;U57; z^Qx9_%v#OrH@FZ-6IN++#d4dV;`2uQFj?0g=Mm6*p(h9NJUrd_FJ@IO=g7}MYI8f@ z?a3tret z0A5W(c598NhZH=-?ZWt*GlL-7a?I+7&tIsY)yfP-_q^urFwFOEPDWYZBp!lPD>7o5 z9P~zJfy#0jRb_eX1*9~wOzib7akp+Bi%W^xQ`dH0f+y+A4Y__A0i!p#Rfn}${P-xw z+DKDOqE#r9haXxBRNv+yepVT;09c9Xcj~_vT0%Zq38b$!sJ%TfCb}Y+>dJ|?nHuB0 zbX;p?NXJHDMU_$(XPvWtNUwOmoOo@LrN$y#G+ejUpJHACmWEXFwMNut$i$V78&>5r zq^?9n9{y&c$(+_~%t~l+BtQ=@6=f$dE?=n?TXf0dI zZ#GQs!O2~oS<}%C} zn8sVvay?B`32Jr1Utbp0GO`n08Javy&CB$708tvBJIq;IySUn&_K<`m3;u@%)JwK` zz1QK@i;UhpZQb0e4HU%wAoZIuqLcTa&*2*l_h-m${E$(_jQbW&rF_1wykU( zVC=@P%N?BZfyv6ioKf(pbhkYeBdTF1Sia3K@4cv`X|ySVR^N7oO7Z8s`S~QG zE$mSsZffPMg2mi% zz&eX=eVI*wUp|MA>J2b%SBc)jE8yLr!ojNknj-OWIxb!Fv7e~XH2>Kq@cG(a<1C_z zA!P)Zn!hEk#cSUOX`ZzE6LrJY&h53*$J5cGI@h_nc2!ySkj^Y?yKC;aeA_|d+?q4I z?rF|xbk~jQdoyvZD`%K}EG%Te$4n(c$9I`<`$mH9>e&V98r)QM3NYPoHVJ`vK&F^%EoQT=|?{c16RVdlUzP;}ashqz#(!JPbb4EKojwifN zi$PS0M%_n@_HzU$TTeC<;8kYCjg zNe#(yuhasl2OOU8E|Qh!(o)%ysgnvKZ`l+FPm}E01dH_hLJAJ;aq{rTBmA^D-DNdF ze)RL2f~zQ_2SZzT;YBO?2c*X3y=1(le}}I6Vq^i(N73IaF*695qP?X>hpf*ew(Hv@ z<3RgVP5 zebkV|UZG626pz_M&hTjW8lZg$@MQW$9l)A=uOxn3c)%M4-K_Cbmq4bXu8j@$*^<3x zuKYcPelT%YqVQTlUV!c2?j<^?qX<`*Xnu7!q@RnO^ICGEQL&QcbbFJp{{|(tYRZBy zZCOm|(W@nGrgEn@awj)Ss(w!Wu8AP86Wve`RG7Lyc%Hd?ITB6$vhH25f9ALmcm`qy zSKYrwR_5aJqE9cbw{%KcWF~wRZT;}%l((xc6_;ZJ6uPn2X^*(zO4?O9h? zf7qc2ldD*Q>*1#ejM9Oxe6)g83ob2N{hByV*9QsJ@8>&Bh@1xFK)f3{EkAp(!O+N7 zRVkV)-gR}gR$on5P8!I9O(`^pDM{~0S?Kv^FVn-{2y(7m@U$EBwa~(n%Bii>j zDw&6;RbSzf{~4rGr)s%{U&j20>NeW`n)&BX`Sux=1oo-_^4 zix4H0WHPcpGbGMiQaH{@W-@v7@4!&)dkx9)gMQeGm3MH56<(?j@G=du9&;fIq@-dt zLwkZGEx)Lf@ThEYHjzKz=2w{|S-?i`xJ!>XruK7**;AjRj%BTsru^Uy;xytPN1Gr= zfVG^S1P*mg;hycq1h?})z{*Wrc1`c3CaYp{(LnN4TR$wstrUWp?$liWjbK3* z*doGe$@LDdjGIKZj=~S^`&9~b3;)nvk0tS^LmR0^op4$z&yf-_HB?GE2ffR@+p}ZE zwaJOd+>a*PY^aL`OJW6MppUpVWEV2>+Wzqx?I1kdLxi=Zt7NtxWl~j_u_}t2iuh|w zEpEIx%ZQ!Y1V^RMRQAfn#^aIZa6sVFmTQ#T_auRU=8P?Uu0$>CzO{_Ud|h3AwOK8? zqANgf&rT&o&zUwZ9Y-g-YE)pPmb1wAjFqv%ioI_&EhKo&sDiIvBiZ0cnk?3FypSIl zhwFReiYf2)0f2?{728v_7)e-R>7#@G4f&oWX-Bq|{GmBupkeVj?aoJ7%F$^|NxqAD z9{od_L@c2Y?bcyVFfI~!DFOydR;cTebEU<;B8dI3T8kvGqb*Oas-u0zQ{5UrjtX4=; zNlcgZ5d%*-2>o0EVyvnQ=Cn3o0DiTx%j5rQ)Qk6%J3j8$E;!oD%d}hemNE3Z;0%jz z0ENrgwM`o`v;v&PA{ARwHT+Q=-7%i6yuzgUpdyj)h+WEb)X`C_ItkN3?Cj8wm&*JP>7nzMp$Tl`6c@d<`-D+8-oI;p+_ z0Lnu4pG#G%>?A-^$!nNJ?f!Dd$ef!jIuG)TR}zm0;#?&8)%@j1aR~=TB!RN|=fw)3 zx3@YOg!o-Ne+6FnU1B86^+x8|Ri6KaYrWYOh2^x=ZKPGqxH0&UGlx|caUn%`SndvZ ze)CG{+yHlH&fBMVd8h>$M?wC5Yy)@u0VQ4nD z_9INE8d@p;6n+|EdfHSmFkWi?Gv79Q)6P4xfndDcoIo?qL2pXQ|7CDLN!`v$a9dz|A4tWj?P#!#^9GC>H9NEKN13dzLMnH0<>!461zy{ zDKO^NY-A+pUo^M=#_(`(WS;UhjT}J)a;fI1D>XPCVt0dU|2Lk{w+UhP`2A)RtgaUDhU-gJ=N;jKnV zJ`6ma>XkR^@5U98N;BN3h8|%pUeN*j#Am^XFxx)eVF>D=u4|w8eQ1~U=OeSPYvHT> z;3=|kYHZ_!_kg`a)YBOt_^PE6B9GI1>Z=-{n|eznL3sR2Lj0 zYRhBF&Iw&SxJacX^>fhwsw(l=RiA55uWfZeT-v0zn~8)`sm*Z|20v%`C`R^}YL{8CD(Vnp`Yp z0jjLQ=Qw5VgO=uzP}ob72LhGmd>666Uy9!G#0yT*h+%A@l3BP~C!-Y%bn%b}yhK|H zvAJJEF;}0~yTlSDClh)|#19gpGA-0_{{LQ~e?O;q6S|49kqs9Y%m(-TH4)fBkBtYN!LAi>eFC8n|_-Zz~Ym zo=8DwSeVgFN!_Ohy4xD2Ra!VY%1YH|*39bciZGpS6+k4M zlz)mmjmS+fho2Oz2?dmff2=NZ)OdPdW*Wmd(`@t<{KUqVJKUT_Qy^b27Z+WgQy6DD z9;F9eclflrKs3}rnla~^qpZ-Lr03>Wyu~S<5D8!J-Wv4?KRd4UbKhLJv=9xY+so!< zNz%r6->rvVUsv7Ac*>7CggaNhdd5|VOq-gz8<~$Tw~cbq*pqK58vcr@i^GtkvSH-; z?4P!xg8hls`TO#R)!@|wzrzRV<j!&Cj$k9h~Yb(atF&d$Y&v7Uqh`N#6UhMOUuERkZT zA+W6y3|p-G9uG`xWSG1m{pJ+MaO<%)EBuB710vl`;MsZ+Jqvu$<56GDD`)HH{(o_* zNrN7!O#>2=CnL&B?+3M^&wm{7>S1*7s;cjL99CPgJqG4bw=-^i53}B<&%*}uwq|gg z{D%eTh@xy5bd=4uws~1@C@BIah!l8h$f;ws6*d<`n>j~VF+Lx!9+Hrrfx&-RqVU^i_e!bc^`)*-mT3lhrJRxsYD+2%AbcG{vjYTk&< zZsyD?=9lGi8_~#%&N?Bn=kOXbVAQS2fPNUmqPw!Xo2u6gek8{=AbDEmPrcGgY7L6e zy!PlzZSx&=)Psdvj=GF^fjumgOyaNW6=J|sV#?T}374E(^xf2=-p~kl`JAzz zs+;{@1(q`&D3M4{F^UfW(~4V?znqL8Youg;TxXC;=w3Ux2G~jeIMfL75{^0%j$FR) z|K)-j8kk(BYPO$TzOU%^j!Aa@3Nbk17%#rU(kAAbT{P4}x-lMvGONzjcFr{&PfNJw zQp^d_Oys~=kjETxM{myCXmOJsW>ifecQgIYryuN4K%h>E8K6S;`J3e6VtW(ub8BsbDVbnJ#J4cU#gc?o z?A;Ikmj{!QzryL~b-Dm)jDW(?Mq9MAS%)Nh{xQi^T#(asLbPY^>T+1o2Pt!7(g|pr zQc}X?cfAi;o$G=!vUI}}45te+^M~Km`Um=VeSOTUG^$h&FF_^PLgGH*aZ^9Q9IH1e zv+x0&{riI>EQt(^cG0US2wo3mQ`IdN7FrX@6^h;%B!6;MjZPJyat**$&t-FHCE8vW zPUdo)UEMf*KNEWD$lM(-@eL_^T+pLwNnQfhVPhf`n|Q?;}aBe6|PfL z+8dcnz^wK+tH8m0vtuepgT0|*{fRZAQezC_K(X@OlQLl25gcC8{ zU;_fa+KP0Y6+$h;8A$)?j+iUj8;a5A0nNWc_sf_d>8ANBv<;@D#dz5U+?0^qR;lE4 zFyfJCc_GHwGp3fW`@6P$v{E4VIsA#4zliUv*L|J zDlQF2jHmru@_!>vKevA)PB#yla#5ceg$&KizxwLNGhYdL`dk|5P4`(oH+MBcV5Wb# zM&waM-%2^yuH#DH+hv3;q=D(TR_z?#VR~(7ou&u)v=}Z9d49E@GXgOGDuxQl#%9pE zO%)%EdV&*mES+IYEcnJyxZs*ja%~zNJZbs$6WWpHpE;h`cCis5_rRi3BA$2P1Wzt? zFsVS~Fa?#(x*WD$xRUnpOS-*a{hEBp+SWm-&$WH+-)((8fChW@6GyLW;cy!AQlP@v zuya(8ijZk{AtYH$t zn_xNTTwFV1EiRTIV5$qJI_A2IeOxOTJ8%_8q z1DNV-GQe4AYLG!rxEyn+VdVhp?t*&-bLYs@l5^7*6ZYKwyNAV$D4z|e%L-dPqyTS- z)#lh-)D%2cjIJOT)FwG5kQ`)9c}1ae3gP^=wwZdX$mja%&A#Z^C0l)53D@+BQ)A=a z0lLvfO&k((WdC<@1UA=E@_ZiSWWJ2FdJoRaA-`M+8GVT(swXFa*PXDy+4{g@5~7F5sL5wy5M_Xqp6jK9(`GYL%_8C4sKOY& zhmxvvLTFY?mDc}-O2MeyxzIhb>(Oh!>upd-1J128N8k9}J92%b%{>DlVCPfK;b3a~ zIx}y4_KQ|P+|re*D5wxIRlD^_3O=XB!ao?MZ|T#4<$iVFXKhYMQm=cvSYwWeSl87{ zjt?*%*Hg@NZi`e~Zc`6JwuinVo zT*QkUrEQy7boH6L0~S>M*1mNRn~Pbenpaj2CC%iO#5Q6m#BRD%on@ZI=q>YUD}Da> z+-DTLsj_`vhkxzsunfgkZ3XK&FtOtEJ{>j+z6Fw#5hD;nGrl@YDl%oVw8>{+PFB&3>qDHgrLYOi#$r1<7f7=pWK`+-vBT0HJ-usM?og%B4!7AIRBiUQ;5u`+y zn9-_959#=teY36%$A6N;5T{%o;YS-<8w(TP-BBzf-xcI4{9Vvn-Xhwg1Fe}VMm3Ha zU1)JZWvwtwIB57ee zUv4HnY5$KZnRfhIjqDza-o<&0^_04@0f2{tnbVQZg;@ExTYHM| zQ$(!rdMHn2l(CpE_kaZ=K%q@4*>){0-twP_C$Ux{5Gy>(;qS0}s^UipbVVV=2CX3v za?}Lvxn#rA#+;ggHttLJA!3oX$Rl1EY;MeKDUl(!AA!*g8F0ol6MJWIVBsapQ|Ihf zkq3jAj0uMChGe6-d4HB4quqicnj*Va3w(UbZWlee{5jeqgvAaRHK6*@nDB7avTwY@k-n?UY0<)Q`mX%@hprD3`7ms1>kACcyXUMNg>>I;PO zAC1VCfwP(ze}9Q!A^pk+TpUoX_Zp^?s33V4nABw=Vi2Q$6H83r*ITWNZ>0$MZhNO7 z=J0WlBW{ly?guWyP z%1}X%Yd2}!f^C(cp1}bYew_)5{>3dfkKdy%eIqWY{SUSHp59oCWYo8G<}Swn=8N*( zwpxGyA&`@cBW7av8nqMs*=EBQ5Q9ZO8)l&Wt1Gx)F)ogp$LV9EB3CHxGL>Q%1EsI& zepgALqGGB&<1$yfMd*Tpc%$Y*|40UIW?h?`GVpUk;7>WeDF53gUm6hjNYL0-Oi=x{eE6;12G zJC=$&Zz{O_G1nQwO>O2{m=u)gi{g%T+F|m~JrpU%fy-t^p1KagLG4v{(Nw*Cjl2+NaP#l~Puc6UppRRVl*YqXamhi6c;}JFww4kl`bz7yc?M1D ze%lMwK%UGxT{q>kMLUV)2t+ivujmfL)b*z&?5l9Q?JUzrx%ue@UvPwQ$)uWD86#v- z05Blea`4;XEH*%ki?29ox^RqSwSSh6&K1<{Zb{w5LTdD~LR`I+n#ALs2~& zn8bA^Yqj988$WZj<;tbuAx1V5XUe8cr&SSTOH%KiIR_mQv_vHn%=EMFn-vi7;_ZEG zkdd;dqllN-ta=^k=U~Gn)qS$>e>Po@^M=VyKjHHxa@1tfdlFM+TXf7pMXX2Sm`?MV zR|UDvOz$gKq>4cn?{Q&;*NJf(oq2MZOi=~CuTdj{%Z65ock4ZqMI`t&X$_mD0wp>{ zMc!S0o>{&yN-^b;#7qi-Z8)$$jdpxQRiBVxx?u*TUZGW|8^Z(Clh6xH%rxzX#Nihm)7H8C zb(mY3p>xKFNV>Ny8vC~Vap9T%A1Uz`jrMZ$%&p7JonE(<$8G*ALY1}iBKL}B@Vv6Q zQ$4M8$*zslN4mDAJH-OJrq$Xzl-`)~|5QIKNx^3~9E?*=`Fx_sS{m6~;RqS4vT z6wSK5l3g1I=m)!;-XAqQ{KL8M>!KPX!}FX+{-a;xY>RfZ?wdGSYOxmJ<2n*%|3VFH z6Mz)}D?A2ka4~3REsUjlO*$VEd;if+2IK}Q9g{qWr%YSWGN6L}@+#qufo%ia1Nbqi zj5$dQdvntIJJ!yU5=D`jle)N69T;|7UgBxL8R=nPQLQgyVPdSd)01raHdWNzP$bF# zZV2*y7KUD|Mp@-Lw}~eGu5Sv?OV(gy{!~?J+Z^rE8`WaZJ}Li^+z|f(r69&yMBsNk{q{g?y;nG)|HQr6G+vg|?Ip z%`$k@pjrHO-I9*CJKPDM$xMN2Z%I~@V zY;)JPNOW2#L+p zNon*EY?LTjTyj7$bXFj^c|nIZwM5dQVcI5FF`_`O89rp*Adq*T^B+=)b?f+&MbkVc z>xR--m2nvpRU)06tS?w<4GZ-8+j1ZnF`zG@A{}nAeF|nSubU5C5Y0i0Q(i=jZr)$xgOFAsz|^LYK(5_f6_`9)TBZ+N2T zpOkhAYVn(96;X_epKRBIBdhxwx(ZOV@`K=$q?aaU{T{-gNOaT2$MSU9F48=<5QQd7yeB z{HSsJWxYCBSryY)?pGVxC1<#85wz5N^1Rl8)GaxR@8#mT`a;3Z5Ms;<6XeoH0`3SP4^3U1rg|iE$5Z zKP0Ki=5!8+WKk1!`kH*F3ng-oJ;(H_Iq){UM!uoy@{4Fp7~OWK>9eb%&OJePQS$0F zX)C9>mAD=41^sC5j)-G3WVCx@_S;Di=u3wQbm}hTNT)3yk7Jt1*qG@hOXWc4*J)JR z>k_-&t)Pxy!#o&J+JJGx&Bhd3@kT0L8QYZW`DoTBG$|?%yrT1?Vrf3Dd?s#p_;;Mc za4Oj-{=?Bm5vr>OC$<-1IHZqMf=lDHK8NwSluMDd(LO=aCAQeu*h-Vt*i~YW!3A;% z^c@;|au}37RgtL3z&`_vc8m)MTx0AkAXCA3UN%76HJ`P<@lg9tYN{d&F9Vs3I|0NFMlcvPC5E=%{fR?gq7X^{8yh zmf8qvtM?9&vEfou_Yz>(C$^uX9Zr6pS<0wOHelnn3AzW54L6c}T@z~4 zvs5^{MW+P}@{mDkcIET^5Ymq#nvA80(LPbU{`sI;A?49n9;yp?p|wa3HD}W0w_zQ3 zy50#09{YY6qjME)|9Dgc9*9fVFy`x{^1vV~A%0#dqe?7h?mjahfQvmP;trr+wOh}b zT>PDSGm5FqeeZD|-qU8(4_+(TSu(xIp|4>TR;&vhevVCPxaKe5y(gK#`Sp2+xnHR< zyrOyzZw49m5bf|XXvr&6)&EWP4h>+FH86ss#kCS+(Wdm%yjI@PSF#X0jo77ZvsCAw%^Q{M- z`|ctO_h8k%$D-sT<)?E@hp;05c4x1CugMiJ)>{7O+6-q_TD|n=sOM@+8EISX(Tc>(3Mo@rq;Iqf~}en#*tW7Z)N!?*cP_A4~uk(^;p%^HfMk6 z>bo8SNS2!mv?iOxf5@Ii7~^Un-~&q9=?D^DNF=q9$huYRL<#`$|BJV;42x^&)+9lL z6C8pE2p-(sogg8&ySsaUK%*hJ6WrY;xVtni!QCCYr^)%wH|O4G=FXiTGy7@!X?pkG ztM;m@RaNg=wO+>N_kP;a@lQmNR8zko5gs4Ttq8up+uRw0pNc_XE%@pUW;yPpBfEc( z!4QySo$%uG_RWqn>>5T*?Lhi^EpvHncrAxFh^MsxL2AlZgRJ-QR43r1SLY-O--d=*}lQu?s|Oj7=P&q{1M&^J84O6;+*5E z-W7BWmDPN%A^5Yv+F|rl%18=Bb5)yi8NxJ2@m7Gu3B4$yn;NeMU(oWK-n>=iNbl#% z{6=a}vc>px%{x6o*Eka#ZcciEG_MnY+>%6!F|QJD<2zujzLnWACAlO9jkk7^%83-i z3OePY(T5J66EpVddS(L5WI%ZJigtm^2|bmCqZJQptPSn_?weq5x~{KRgDLNmlM?yA zIbB{(-^aw+IS`Gsrr?oC`qA4+dC4AUJXBCUI5dS#?N%nl+K$UY98yb&mML2nvG@?G z@jRf3#xOqAxBs|FTU&p&U=dD|i<|w!z;oP}@fI8NAh<|Ioe>IB0MTO57}-zD0?mf= z7B@(`0e)U+#<;>#)peRrof;17bVtbcKr>eLJ;NfKfg$NulJq|b>wx{Ah< z-79+agpzuYg*J;TQrWVo0u!A;m`~q0X@!-1J|NY*hUTzP@*Sj&HN* zZIy?~qN*ySfYNVzffEwsOHJU_8O3%{p$Y*?KC{&`*O#S!90XkSY2AT!N`ufdS#W<- zc|(Kb|pN#*%;#!P^kVNi5_Am2Mb;bLY4NB=&7sa$8hNTjd6n#t^*m36N zvoql-s!f^<(*Xb(7SrWv`~QkPLsBpJn+w6;GUz=El)u8r9&>;hkP^?tg@U?AvRE01 zQF^;(@Ento!Vc+QNK_QGw6XsKlcR{Txhdj4;bA{$Xl>LbBM-Bs@Voj`jJR0ELRxXe z&W5(SwvV7ThiDpF$4B|*`-~dMzUq_1;f|4|xJnS|#=}wwPNp*e*-XbnjXF3!g{G{M zu^p#nSOhf_DFU+$zz~ybKHXmXu}bViQrP5h27qX$D6}@@AI=5=X>j)dhYr>hvCD`%Y9e#J>SdN00$|t ze2P*ZQoEJ&zLrZ3>_#PHyTvMk9wZ)|Wc4wuswrc-jXVH9es}kQa1st8 zMyZ1-=C8(1`!h;x zg~QaLW;1NdP{#H z;Hia|790?WPpqGkU(wvjaH<0Ibf z@BDA}(Ymr%ub?Ei4-R@k75(pz-TQY;j&;Ascq>n*xhN)lb}t%XZ*_YQmW*DF4S$k5 zoEk0dx7~`tSj{g{phG&qdkcZ+GH7%4n2OnaeW=AX8`py#r=;~qzLt;HJ$o>$Z+D`} zk2I^&ksBE{3X1#=5ksfLG4ywF(oM{Ew8?u8#0_UhoYT+>H7R88#a)nn2|J|KUe9C6 z(IfbmPK4@&6pf+adn2(%_~Yc`svdK-a9L}@;lJop@C~31*#G6G36pkMF5N%G(MYwH z|Fc~{K!7DPO;GlG zXRCsRORQ+IjeD0;k}6;E_U6~B6CGU4pHneRQ^k43;N+ur$4^q(T9_~(MCFmK82y5l z6{Hk8m{KD@2hYt7oHjZ=|5aj0mgkfvcXUFFP0ZgjSq6jKuS*1GIk>t8i-d(wGJ8B% zZA1y#%$@fQiC-0_{&Q5398o9mf=l;PQ0wC|RjgGZ`=q7zf|~!Q@k9v1Bq6*6_XkLi z{NIln2|X{0vxi}3-Ux5ySM!w7eK`0rA-0k2H_P{t?O$c}ClZF_p*Y~Y@otrZC){dJTWPh3(g;sjq&KD&}LK!sZ0J_W<{t2mVX9y ze<8fQ{<5M#4k0;laP_S;5S@`!qMn?{dnMOBe-TakH>Zd82&O*|gcc+U?r&ZbNPT&h z+ET|>lnn3r#WR+}WTF0dc|g2xPl(j^DRF@bjfJo-s3@8Q&3}kJKr|4-6P-VeAJcK`12tCvz+Ej zctw}e#-8l!$_5J4rbC}?p6qpy4qylh*|aStrY6dIw(0-Dv(j<8%yD;V%l*INSwTrB zH@1`y4-3=9ygc6{QM@Z;hl1Z^t6FBKd=uN#Q$2~ z&zyZ`dA7~OY7GvB5Nq2^2UsYT2hnW#OB=6}9^te)SR5KvzcpKpXN66y!MN1V{122S zB}okoxD5$kmpe2)2O;z|Haj9GYd~!}N{-(%BfUZMQiPf41#l>`jJRK}i|RWu*sir9 zUdE!75ER1yMAws!;NJ$J>-h;k&&t1VrM920LjU!r;=`=x0wHX&6+6IvVAWI6DH>YGgK|N7aj_Y$*4wVN zOM|uC1>*eML|%->zyBaCG%pl8fA?Z2C?%i&$QSSWF|F4pkj(yn{vv{Fh5Gh1PE-An zKlXI>xy672!sPPrgPhGlX1XA^e%ANx{Y@F+SR{$FrGEhXzt2O#u$jnU7O?rqSZW;? z*xKoK$bgsRHvYvrOaBqHsiOtD+;>-|bSAUfy`h>t4pwYb$A_6w zYmXJ|FSFVM)_*gKkD?NnN=j*1d>WqfMN7b0Y0f75^MLwKS7Svsev zBbCmZ)T{9#$AOhj7Nbg#Fb<$&TR zw|A-72YmZYQPymq9#zL5)i3zY`}vGK4jIA&X?rv0S8Yu9d>12h7nqFhf*qqODrI~X@)0*O84tioZ zIvJWm1fmeqlQ9-|tGT@RQuw#_6nB@0u*Kf$IoR6lLB%09p2BG&Kx&( z`x_ip%Sl~Ux_N$}m|^9-CoON(L{nK3ChUBAo@tPYE#4ZgfH(ia#b?{!bJhtWvs7A3 znHoJzu2v2+wb6B=2=J)UTx28fCU}c*4btuVFEX*cHdGQCwvp=Nkud~Ty%bJ47Rj%H zCNWJF`P@$SdcjNK`9T)nb?&Oox~LrV5=>$S(L_EuebacviP&j~zz{dKtG>vD)6(+P zdN$+QXo&9-X&)K*RD1gA*@y=kvR1OmvQSQ;&!bWKO9`IjDnQl5bQ0^S>N*l1pQaCq zf7BqPMnisI-ieKDHli)~gbyrm(Gxx|y~$u>$6xLeca->)Xh2Jy-q$eC3nPRE&)99)@Nb;n@|c}#R2KtrpG(A z33}EwUM)*4SQq51;8;C11!g#CNJA~mG>2QL6F;O&yf7d7F%R!0O|Al&-a(PDMK3ox z#L*>=8+jYQJehy-hU6!GBFzGyr!L>azVT7UHi<^-YT>9x;Bq3xU~Jy9dDn6yifJNQ zY%7R`!t5@Ftfd|qBfKGGK4fct^l4(SHjN&GJ|dM&(oricG<&^Gs5orDzPMfu(!Tsx zJS&TGOiJqIQ<^4_Su*MCFkP7;@dhhw!5(JB_P&)iLXC-v-@5XGRiGbfAzj)3_{CCW z>R9d04KNHcc{gthwg{ANhJ~Gz065i8xeq&bPaff*xb;BpUqoN?Mgc)9BSC~zcfpD- z(2p_;4f=YnHH=Jz$9e(8h|1m>B;B{t2FQlQ@;r2!Fkw_j{C*3Rmft^Z&r#FN1?^xPhMvLVVq~n&A&uN_BJ~ME0Jd ztz|dZSajsnRM#v?0Y37+=x?FXfwr0mdOI(V=)?i575)QmN^O4LV)_#Bq&56u-AEWn`Y<7H)~+M z>}90V&XwYL=B)i>&vDUxJ?*8q>iwg{`^Q*rKt0Y>hk^zueE2YPukTBvZ^|6)L-e%g zy74Oq7vO2cK5Qm}P6X0TyNKK1nt+_IxkIZh7lN(fNyl`#57)6^Oa7bSExI^at5uN8 z9Q7hMlCRIUmMuwv+GkO5fPbiWj~+m+pbMt{UAd~CH?La-FjN#&V};P6lpRRKcNo|2 zEk-wlf#b4&z~1}Q!N|uk`g3JjLOF`BU1@pkazM}=cR?_d%kx!HV5N|)*0%SfWG8jB zgA>ltiuwfg*ERZw*Y?rxwmM30gHW!oD-*8jA3$5Y ze{Ms%j}XzOb0(Yh*3oiOdeI7miEk*c_4_caD+_9M(mw&N+6_$TR2rRBHY&eube8{|_!5sgUza`M)BIgcu>ncSX?-6t6~lc$2= zzG*OTi^JJ)rc;U5-6{U0cczYSGPAtSDfUU@9tJ^7I>~D8S+oey_kHFGs2O|z^C+&{ zNnZoWq~KJz*53)nQUq5?`25gP17=0a0{{XLROi}rk$3$4$wtC`^c35;WDa&tm``d; zj92hIBn4Mt;i3&TtKE`wG|^K&dRx9zyTnZzBH@q?{&PtDm%uwO3#4DqjMH5wd2N@m z(sdr7$V|^;0T2IEx|+OiXAi9M3CeQEyxn zD2wdkynvTh*-t*WjEGEkRen<*FLN~|%#Xi6LP2qbtmBN9$7jTQ)0rEKNo00DKO(Pw z$DVgHKu)|pQh!2Co$46>K24;X0!&Ojy)0VB;$R-FAxRyOU-;guoCz6J;gYF@%xdq% zlRPN!A9^plQmT~!T0O0JOF3NIB}n3Bo2yna zK|gwP!hAwRFZbi(x8Q4OsrB+UMMOUm;QXzk`}L2u2u%ILm|QF1xW3zdSgN&wz48=m~k9x9WB1Cr99%?nGP_4rNRT zGZi6p*IK~JM5${^=BrIj$lYHAFo^yU(2Yw7-g~@>%WS)fH?UP7$_S*h6TJW3mBbGx zJwl2G+w{F34am!;DeNq2WSpo=Vwsd-sv^PYb$(f-7bCOJ+fcDfa(Y{3?5;;AL%F|0 zG&N-8bkV=7`ZR%~z&VC?e`Zwi3qIZ^^ZBlu%o+9Eihmp(NcH}vJKC<^W((-M)UdHT zgy3-LH!yxLu^JOC=z6WrVev3tA>b?3o3KB1MwNa^#vuxjh%^)gRR+0B+eQ-JM`9&p zZO)del`Jf^6B&Mt9bK-ERb1MN?#|0d5(3P~i6RMph2ruDh~wfidH*$2n?ZQNLw*S# zB@2EZS-Im_YfHov92onh&eXA4bAG(va-)DddOy2U>&tq#+L zecwrUhdSV8Dv0^iTc7RKfU<46&NC}hnxChKyifR(Fd`6=(*l@r?V%Lw1A+sZ==6`ZpTRX9*q1=$H|8!rkmXP_C$La%ql6)Xq@SHmR-LX=1hj~ zhwZ?+23i^)osBFUk^=xs2UmF25&S94r}I%Dodh!H6NY3euy)9jbC9l*CUC z{P<8(0RiFpIYt}+{EP2y0c<4lzDLpU9~>SSpE$kDv(M+&LoVi-8-LsGKwZCDM0>fP ze#YNf0|v-Bru-q_`1`!E*vJ6~@xMLO+}C)W#$O|-c^vDgKnj^u zLk?B6KfCeu?F2HLw@Dr*KT6F-Kc9c&Zky@p6wW**dLWm^mY~@G)7{rdA~F{LTm>}? zBmFWheZ1@i1kv3q351@_4*Q7t9VZJ;1xlj-wcHglf%qwuh8Jrsx}&WTpyKVT_J02k zKK0VzC_=uG2b|mUO+!j4uh-z@2u2Qh~6=lZ_3ode6?JmvN zm{6G1JQlM2(q%ut%>koiaEAFuLZAe<8K51mP7_ESR0mFP^1Br%qESF0s6{>8*}+{M z=>}?S$WTKXklXDrd+}lKy@>}kQ#$Aw)J9-kivF8;qDq6VBj460#q*BuEUU{oq*kqh zP(Cc%I%$;HwB_-(?BWmzYR?{8ex7aP-Oep9GLqiH^{kf@Q1w3Ku$h|#uXKzn zuv2QfyPd8wrvxs1Bs4AQU1-%N?5$O~s9@54^m^X-p2;#~o1lKWpKMhFM?s>7x;IlA z#@e!-Bj%&-vS~ZYaR!yevR9Y+YYt5eR*Io zLm_awJsR11zLDnn@}}PJ`Kt?wW1@a^_r3T7%}Kowf;fe5!Nyvx0YIY!f+!tJ7UT@q9YQ+PA5 zmeTU0!CYQWr-O(4W_Ag>9kN-g_j}XhdwzP4z+sq!bevHF2{`tB%+;gVEGoA6R+eqr zQL4BDVW)oGpPS|nU(#cB9v&TBB<9v;)3{$3lF#_gZUyWhnutjOY>DDAI@#jA4W4aN zU z`UTz-OcA>wzHS`9ROH*+R!|x2nd^dU-&%VvvJ%%Qa4XN1HinGf=xt{z_E%;P(NwYB zaB_%#7cMDyJZ+m!$|!q9(}N_5+?-aro+mJ`{CT3rNmXu(THFY^*xj>=F2>3Ti0*3~ z4}{*{?@MY-wYfp^ zsSD0gC6vfgWcus|P%-vIEqJ}X7wfz>>lmYL@(qO&;%6_jI~-&#lZub@R$b_6>MZ7W z|9~RYChlh6oI0JJuBrhsBLsZ&jrQH+Fi^kfH9t)p1b!yS9SZd1U#WDMp5|453sqRK zhpOP_)Lausi*&-zVU13lg}9D>vfK>Xjlz~sX}CCLPMc?Lfp|1_`;w@jlqy`#EO{D# zYC#v!^2gK~KyxI(3cAw1^7jm1>)#(+t+YJGEeKJ7s9V*{cvIRFN{Ot!9P-3vJW|+@ zy*ulV146z>st_Gxbwp36w}p$|aC5Vp6F!R}f$d*IMQ!{Zgl?G0699jsa2;ufxL#f= z67)EJ&eD*N(gLrx&p265CZEE^Iqy}C$~TS$%4}l|02Et-?}Bau7yL=H*8NEV6yaY< zWn`2CrA}m1=HDlC@ad$Vi!8X7no<;S_ka zaoW@OL^iX1^^r<2ec@GJi@|?!>P(3uFxnR5MIqHf_z2j80V|N zTV4PA)bQiy3}_=bBMnk1)^%G4Sc^lBojMlDC>yYf>gW5jQyC%jTxro1Zi}PbK~aW} zH4V?VpY}T_-wL6_OujN?{5DCN4{h0r#1XZf7rgy;x%HKB3RiA&T%SS;-%Q{xgH1_> znt-qBADJK;x}oPwdq6@T-7G-`wWE8X&z+SBnqWw5CLvImaiDfmk=m;193 z4Rcikn2lP=9T}4NQ=(3W82#>d8n1znj`*{`8{`3nX^uBxNQ#idYtD=gQ)I_6O4J(0 zuZiPiREvc59+_B~Wc4egH7Wh1#1qZp5_x9r!rM_u^g3kU_C0g{y`B zmayker2k*e3TJO*|Qx#rQ^M2vh(2EmiHs!M{N?1I|7Q1NT zzSyF&-!zK7882;VZa;wsP8FT%lvyxH)6VWY1sN|jKEViEhW6Q+wO2Sk__WbwSe@ej z=JZqOC^g~to6u)ui^TeYff}Y%D6QGbhD9^|#H78$YT`KEpkR6U-S6I8L~_FZlYmOm zWI0QrTe5o#Kf;vtM^pgWAqV}#&5TKWeMO!z$}=S-+qmzWdahnVvga5bR^d%0ag}KC z)md(5;cSQz_hpmDxK}2Q^r3hMDHVhavDkOs&tgPf<+aX8HW%-qsjF^La+B=(O#I{O zaiP=s4d&WDQ?3JgCI!=Tf~^KmJQgsA{*;QLvNqU)_u^AO%^C=gyJx!fN}`KN@@;uy zN;8~-MxCe;b?=Oi>zx(OZB0$h#ZXq=gx>>gxu}W_cRnaKt|djV|L%w4LKUmyw-Ryj z&r=m0?-WN~Bc;|C zm7!}g{=NBRu?8YFtxr{TWHUwO+P;VWCbReCPM+LP!ZwSweo^hEl=y8EGZO%KpU=P_ z(0B0p(WbnpzQS%%ju+XwWsJ@ZU+@=ptr4&NHVm-fcj-~r#qP+rFn7-HL>H4#q$DV( zmgjexU9oy^ZGR-cJ-ouNKnN!NwrUP?HC1|`#bwlNaJIC|zGuxx`ecd0m4@M&YtDk( zN&$=gI??9dS%%asQ7R);UnZ1H`?A4CM_BQFUp70lN$i?$_BtbU{)%ba^!V>rON7j) ze8#R@=1?}cYAo@ZN3?c{(y|Tk!V!_Q-H+r1VZHj5M0X26Y=0r+)0!`#t^K-jyuu`R zzeX{6O0>=5G}NR&4*kOY8myGJNDN2@hhp-j*d(duwoCCh9vsBh>6fxeKcB9T^y;9M zrL7xGUtbRdiJIv>x)^Oews1vfi-S3PX0e98e~wCP4Vq+w1e>k#<^H_gLklfUzXmYo zrMlG)@>JhF2g6rj23?6i9Wt7peRSBKr&+QazC?bLd9a34jXFl~E>>#IErJh=xAW(1 zjCf@bCr5PhdS^fbJd*%ywLSnuVZM4W?0{Y`z4){Po}*gLzYOguP1IZEaKA96a#BaD zLu#65)o?}jb>erbrzV2heX*<~y|Hh)`_*qz4T9E~c-UroOW7Pu6d7?qhigw&Y!j6B z7wBBW{BZCsM|$Kq?SAOEm*kP@ea6hD41S?o^y30|u6w^>f}M2^EN-;t@`X2ns|g8q z(>A~Cxhg@&ori;^AaXxIb%__86YoeRKbQFB>$5{366f;;5*;kB7;!}A@@zJ1^1k8~ z_&3FB>Z0%0r8$bzBmL#F*A^n?HF)%H@r32%{--+Q$p2Cowc|Hbx~8?gvt-vdOFC5uegKH39EwB{Wud zTF|ylpAe0hyV-;}qi#l?fU;8l>2BN*?gVH)Wec1;YS!$+u!&Zat8U0Kl(|1*ca90b*M6I(E{5SMkqlPUnwVWnynqeSam!lEA3d7_)j`8eG_XTg&k*PE<@?g%2QA;CSC`ht; zqBE0Mczt3@J!A%8=V6lnLcSsuWwIw~+w3#%z51MDS~gViJoI4z=KSVsNsZTIDkPj6 z3dE$QHMTkIMOCp~>Px&rw^F)Wn`F00wA}3DHC1MrLs6K4hEgLcMjQC+}F;J>8oa8(VIzv&Ih0PZf#D2S@-qKV{q? z(Ix&WlGVtAwq~Fo;e{r;PiuoY^#&bN>k#oE*ua{amL8Q%V5r;~nh@6E>)4cW!5dl5 zp^9i*q^g*oT1={6Qn{b;P#m7wjM?GmdvV2tZ&TWSKM;O{k$54|V(ub=qX#}`^KvF- zU}Bl*i_oCnSnHr`OB&v@v!li5QgPlAMiupFaQwFQ{xJ#hZQouLDrjfo@ z_WQ!ttx}P`z$q4%OcRPU#7dn|$KjsD++vw;k(BZu230;a^N$o|(~Qg9U3*p;Hs4i; zu+R<3ZrRZ$19!9H8Mm>F2UnJX9UWaBNchu`vH1%Ymx1fzD&?sJIEVhw`awT$Da0+* znM%b2@VbAz9kfY(DF6yKi*HJYeU_nVeHzxtLzA5f!|HqBOx)H{fPrDgQ$s1G;6w#% zSjsvj2yhjC*Olkjz&esMl!jG}&hgl9VKconKR8NDOpi}X(%50Kmr{4K;S45~C)5RW z*1gUcJ}(wT%eT{1!ilGcfDHq>AR zS6ra;F5!g=;E%$lS$V4>Ur`?zq_SB=2qAY;-is|QLUolou zGcda-DxNPA9D(~ctGA&?I!j(drEXNH$23=@8s$$cDs66Rzqr^%2Tzf3i@CKtp#t$z z6pecNs*Jk_7$huuRS2{+iUfXG9%wwy{}iQyuijvJqGF8_Jv|~sT(m>1(fHumEU zG!GK=u7o5p5mDJ(SQjx;;GRg_@)x_wW*tDtK%3VCcdbdQLB?CBTCgP}_uGf@6!I0^ z#>$@+ugm-CaoRK59dW0keq6y zd2tl=Yv#GGw&)I(aERCz?-V^H>pLAf#Pt|wSyshk7VUqozqtgJA`3x#-4(4nNogjJE=tQ{X3d>)l2gKX>s^iD(t(xeV!*%B(nxq{vPsEO8u zX~@6>&ZRr^cZTH)EhQjVl~d$PAIy_Y?dZOm6ySi0u6c`s8Sw?6Gdp7<5+flRc1Lyp zs@vc_j%fIV7jZ*>e)m-}DEC}owAoW??Yqk)=L3p?-1K#j)h^0HlGF)^TM|f`QC-;| zNZLH{oVI~iC!AW&t+bWi^<90byfbUkkSu(N#Vfz~D462wN$&^p5Uq4xi?@gxsEjv75 z&dv2Iv(86ld5`#~4iTJnJG$TfRGr_E*VUeNT!9I|Fd zvSoE;z87LYvz-2Gn5fW%doiV#@2)Wg(w+=F>7U68<0J5Ef>B|+-Fr#Y3 zeLkO-A+e4PKuM7qdu43bh*v8%<|Ujl_@jGjOU8_xBK0p8^RoJ_)j%%O9-@19Yzdi0pQ7<-2bVK(Ykq9+BS2>YkwI82 zJDP4slANF!Dc3A!S3-SJrh5^?=zQDe>}1rJK(cX)@@?Sx(sV$i1uWEQpJxLu@NDVn zD7^9TIo}fj-rGk&;8EDaN(<JQVoGs&w1fmQ^8>+-9-bPJfgZ;oz(khsm(MJZT zzI}|4(IlDzy7sU$l+LKJ-;?U&-qz^nw0%va)V4=i_fGSuxQ`K2u>rmTD4GJ_XUNPo zWTW?Z5Pi~P7?Tk(k#>UjgtSRisty#dX(yvLzQGmkGSg}HZkw2y?FbF_CYH4+4CKe@ z`EW*1_p2vYd~t(a{|B42;+AZf^nwYt{>l3914AR!ZIgvJOM*$A$J%a|=ObvY?|0$@ z@&Hz5xbOfL7Mp9(Jk4@~QiHm0BQx14bG*K`jx`e6k(77owQ_@7D>wf`Vyek@K1VK_ zI<7%Ub=Twj=h7=$9FfysuRx5Zc@`Z<3qPxps%5=ym+P}A=ab@bwO2(ghZ$R^3ep~% z+=et`twlpxB}^|6D@ek8FZr>b_k=SVM`p%&?eAh%0`hhBH$Z znj0JI*`GuAfGVp`TL>NzruFI1@GzcB0;39C?jKRnbN43;=|f5da;YU{EAMte=?8}z zC}CC|iL2t0NGk2URvwdjwjDwe`a9)q&|XQp+K;1o;g)uY`qOhkK$59QdTm~$QRF59 zG+_etG8|Dt_@5mh_L3iLwdDamL`Z1f7ghmtN4)u8#lhdNE7REIz`)mSOTwSK0-&k) zsXc~QL2Ij4+C*s#C;g>QYEA=TDZZ%y^)anh>avJhPk2LRO~JeE8a*olAi2YNTJKU^ zk4bJ}MgX*DORcB-aU%b$%+xL8s1_i4eZf}kThNoSZo|d5OfIInWO}7QmeiJG+ftM=Sr^laL!|0L`MJlaccU&;g${lL~e5P;Zx4)&c%Qj z!jyKAiQJZp%HHQyr}|acT<-I2;NjpQFPIZIW~V1)pJ?E)mFPA*zmXwXSa%xN4oXb& z!56#3$iI7YIJLFE0ycL;_)s1EEn$qTBKeb6R{mA8sLdG3p{alD*;sqWj%Vuj^Mfh4 z!J)$%hMAse{Ac~|)L;A=j?nUeu7&KxS3qveDac%XrT3sD6OX(7~ zDt`(60BcgUW$OlgLO%{(;-uIg)Yc*jp=^5Z3{MAq@TV_Jt7AxsT~B5#xxb_)P%QPn4Qv zWZ@V%{h@~4)%~03I{{43;jYf!diE6o14%ZCCoy!R9;cqC;$pKR!LpE$S1-^cx|;q7 z+hF811l%@upBF^ae+`lT%^?QDMiRhS?mCiM0O_n@j~!9E8a{hw5Fk7O-owLWrMbDW z`qBRL4}K}x`Mb%kHXfTye^MzQBeEl8HIxrj*}XzBa9urxF2-1~gh|J7UN-Q5f$9{7 z)647fke{mjPV6=E>qN+ffb|k98WFugygz9~V)_un2mT=AFEBaeB_SH`P+=j7o@)!i zGK&AK{yzN&pq-gOPEJV@q93WSyL@KA0K=~@_oOqRZs8XvgogjCSd6~}|9OwVg18V` z=IbDsdJ2+SAqB-hMB!0GXp4noLUyO|s+j+;yZ|ssjfEe`DG&^;@$BS`{~-=i$uT?r z5Z18_ik+aUz;eHDug?6;E|m(7ex8;|K_+omAr`_{q`am(^>MSNx9^&bGX&Gt-qjs* z%y3sX3!iL&eLUVlFlN?>`{5rm4C@=sYAsV)t400&$`y0i!UVYU>JV4MB+LQXw^tUM zq&d72_z+hE5H;H3v}1NS+)E|pZnWzsFxytY*>GpdZHidAe=u|W$N-WfG{MH>-^i0dj5`k{uvc?F}Qnj!VPkt%RS|w zz2b-_t};NB(FfhbogY3svYrK=u6)j=v_8e3m%B^hkBdMgq=9&^UJBQ9_F)*yWEt4t zZ5gVpV;3+V&^gtvhM9>d!VhC{g;(*dm9IW4wg>9BH967e$kZEkov{X4Gf+_;=!$54 zq-17HnJIfPBMfDOvd5VY1Ez>%DoSFM5Aj;KAC=?mBetXoLQaJ9Sdzl~@l;o!zGm*E zZ?bFJ@*3ss499|3Z}JgSS0^dvyvaiJ;>`(yMFYQQgO^m)cIM2cQIX(bIH^3*Q3=FH z7U&ib7=?K}vc>k^m$l~8VR2z7=_vdrcB~y1Us3QkdN-`%F?Slp^x3ZD+nW`UJ%j0h zcOM@2E3va^Y&ngmkHUvVwx^vDG2K>(Ga3#$@3<0+csFy2oUuDT}f+z|7@<(Ykkyq=B<{gubB=<=whewz^ zC&3GS;wgPxTyYB~g^pF>(-+MQ2x<~@)mWzc_w;>MPj$-G3e^eXDK~x_S16PE9&bp; zIet$wNerm6x@_iaG6z2@`}xa+xwLpf3m;@0O7EMri4pM)-m;v5Ixqc_FX~>WcgStq z*q`kv!qff8+8_SWz}pGbzmI?h>;i0K?)am%CXRL1D4RzQabUuz*{{6aONtxA0cG)$ z@N^&eE+=-x(fl(wZl@8;9+)!^&c5U(Lc8$hR|)oM(Fq(DKMjB9MTrO|jP|_hVnkGq z*4Kl5q?M}a{-35RWdh1mDw5O(8EJ}mRx??o%-BXHXmDyd;M1#X^xRneo+|pRu#g&3 zn-=i1ovCfh7%|wbFtqQ~7wookFUa@QQMfr356fH}Q0ZS)-i%LHl)I>=n)=ApuvMhjuu~9j0)9H9b zXrA|k!j4U!x5YGqFWKK}LOyo$z4W3>QqTft6Za=yu=CLsV5}ILf=*>!;}>e9s2f&L z3Fv)3hqEu#AGF!X5%;Q&XFpr3=3^!=a9?i|640%)%pdM~u{N+5hO#T${biX^LI53v zHR#`6UnjwpG}#n8crSVN-{Tsk+#>qBfW4Sdu)^aNRiliX6w2>N=;mtG+Fz*w zva>%6Z?G<9Jgj;3FKjBVc`(Et@@d@N292JMwxUeE#iQuc-usc0(Pgwep95FI;Q@l* zV`hK?rjg6TMX!=yPF*1_wvT$W%Xw%8jOpde5voS_W!(>}w!Kt4J0biR?CPKt3`Wc-YChdc2}i)>_4|0TxU`bY2OMKrSV**w`dcpb82YveDr1) zhgck#B)qz)Wh`-Hp2C{S-}!DI7K}-AreUhZd3BsU_G0rD;1A4(A6Q z=7Eg7Jc-*f;DmLZH9iF{thbEHW7!%o3GNasf#B}$?k>UIHMlz@NN`>xxH|-Q zx8Uv?+}&NLIp^NF-aUdG`rH3_0Wzc!KkIwzi*q8`!nA6eaRR zaxSz14(*l&X%%l-{;gNp_4j%D6Dcv`&!V*mEb?GJcZS8Pe{sBd0FQ2GJN&IGswl^y zHcGJ=Cqgp@+-XU{M^nvjDIqw~34FDg$1QrZCYDv7ytD8D(@}n?ZK-;}kuU8GDbg;l zM{#nAl!l0z!-sTu+zTLvAtDkug4F=Km@MB(&YGY3`MU$_sy>WD;xX%@d#QMMUQ$q| z`k*GC(1G6euBy4SubR?4k)wntYem6QX+DY1ghEGU5jnT7X1Z-g03lleG8Ut!EG|v> zgIMHGq!}JI%|z|>VY~q38o36;)9mwozvXaw(n-SxZD2ZP$^Ru*0wHV@p z2qV#sONs(~F!PEPRT1=iJZD9|eMrYVQf1lE61Tyn0ygZJ7*BiE;dgAJ{(AvTI^QXq zD|5viz0G+heezt4*Kis0{hHUcghy1!0f#r*DN3`0&);O7O2;EPnF4r*2Fid_vD~CR z!$fR8y`wob?&97N#RpK_tb~MK>a7|>w7l$KR*gN%*`bLKZvhmisXkxvi>9Ujb%`1N zsHquMNkBRI)9y1#pG?Tc*bDOxx|Pxg$YL1Qdbz%Xh(-8iDWdohnbf~3Yvdt{;6W%! zt3UlhSV=8bveimjP-JwWkF4irCJmM)&!&I(8|)a>;c^^`)IKN`tHM}6Nls@$uk8ZU zt&TzYu)okbhhM+I8%ZtKfiU54(lQbPw;38Mi9WI*1LUcyx$>}q=PBt$lErasGzdIa!k&KQavmBN4CgMB=u)YD;qgqm5);Ff+=VV>Ki~XmSkL4 zUP09>jwnhoQNLeX-s*O!#rnt2+060~&tJ)T#WYGH;XgcKje35C*u@esszs=;z7o!~ z|2|wa@9)Xt=x^|u-gs#+k%_AfBvRn7lYgx6^K$bk4JcyTfz1o6RA-auc>_y-{VrLY03^m(7? z9NN@2M?kAhwd=aY%!aGJ57c zMBPbOwRj3H_-ip6&`1(eRcV!i%Oqjv>xJ-bsF~_y;3%!k?So}=AR;0g8;i26hw-!f z?F@cCyJ5~q_ZW*Zcryq?`%h#dpCq~FyO=)Hf9x7oHr ze1K~4VzK*s^umToFLMY>U!{f*!Jp8jH`1R1&_8CCw3KkXo;B*)-AQo)WJ-s-DI`7Zqv?C)2PFUCQ9kdLi9oFQiXbbim4~MXAufgRej!>M z{3yKeK>4~?1T_y1-HW#XntVNK=e-R$MVLBgL!{82MnkOKw)g8Q+GeG{HtH(;s47Kp@2uCxMWtVRxGs$#>WZK~FQ5$4 zQP(5O5%?2I#`Op3IzsLkzwDtwr?X zCKscceIgyCrMyaZRK)&p=vKdoJ}%^bcrc;lmGRS@9RM!<(`SchK)0enTJn42kk~GM z97W3^C}qI+0t4PZ3Ky~Q-dVLxp6+s{JIcUNGLb9R0YKwM)7NuIF_S##e|rG^Klf3g zt%<&4aK;d-xIp1~RsQYnlHHeS9#$p%3@Khsd~1`h0tw)8SdIEj{+|$3$1`5EX+0cy zI%wIhCYw4k68ND19mlOcU~B-L#xi*R`&a*Wln!=ELr9-D@52G3t6)nuL#}-UVIQ1j zYJm;ifC8>vjQy0=9q85Ju3!JdKke!X`Jd+duR6akM3Y@MeauwG<3iXUYn7a@Zp%N0 zDL_YG=4sZ$#}8>~mQ>5=F*cP`7dMgPWiF)NaN7+r+P3V(ar!3l4ECSVeF28R!^8SR zADHm_e|m}t;j52elx6>F@uIDHoe$gjtqnl{-vA18@zq&(?8i%|gC&IYFV?}W(&WC9 zTccX-5HA1P0-|vYT8S@#Zxk9W=5MXOA=TWKK| zSG(?YX(i$;5#ZM>AYZpJ{!?F($?(f3@w_NTdJwM<*2t~d5+6N6h6&){v+(0h?)3d3(Y9@&lS)$kR|l$y%qig!1hcmv`#S3FdK51pAUXI7 zgM&Gq++fRbZR&Wwp17tcBpxjeV_Bbf;hM%3#k&2}Kh96*g%}C{cZ{!1A7rt3_@E8< z1h0fXM1}&)T@hn)3#&cM{rf5VCtC)6{>tO7QG^N3ce~$>Y+M!ol4Z*9=Meu^+g2I1 z$(pxQswJO|fAH8S!pLw|rl|=@Cz8l0Rz8bBR49)J(Uvsgq=utRh{3lU{|G`CNK2i( z(3qd%DY%x-c?Uwde}0dLlhbKQmYW)V0r<4M=suYgM$3UbE|FaSeWA9(@>t%+^wtO9 zD+pFZ5sv2eXQiL&krW{rjb!DXy3Wcj3c_UuW?!)%qA8yeN_n%V3Ed|eIYm+1>Bpv` z>?o3bz3v+s))boME(D+Z*?sXN46&D zHNgQ3uGRwi2>V{2{{aHlYJ~)bsc!S-7BfZwY92lzQY0(K)V}022@)y}WE@W4BOuig zB1xW+70KTb*5N0u7>Ct$GE)4$zo+ymqQ;d?Jlfn}4kY6xz>L2K19 zTUEW9Kml$!P_(#23cls{33vDQTs{wi0Au9f0r14vuo)|1V>l^MIi zN{q0~{xckTlQ9JnI0ht+;;$gG$c3xgPv*sFd1&42hV@(=@Ur@q_P&-q=>niE2_pX) zmjB!xz(sO|^8Wcl%`v}Uo!NUF_hBVSemd<<;=v@kadSPr)92-yHxpxEvG`EJi$K-d6m z!{B`H(@06Rq(4OE`c@@EzW2Yf~}L6IA`8)J*^HR1~J z=?W3bfmj8n5Z_nkNkQCTdOv~20QsQ*J*0RxLrV`BtRR^_;!JqXP|3-O)?+^JWk6yA zbCOXKf^-xW2X1bc4q^EHV}cI&Utu%|seaN3Kx+V#HSmV)nD4;q6cL=Fi2}$Vyb>cY z#`)0MGLb0>bnA;Ozz&~Hh*W91m7JD?1;GZC%_#|CEebpV$Y0qp9KY?|0|=2MW^2ST zz+eC)_$M^BYJ8h7q6R%UP{=F)*#Q3CpMbxS|C%Y+r;~X9k$nKV6Lw7l5B+IxC(iHn z;-*z^Lb3I_t507FkG}u4_R+;6jrE82zqbPR%2=V^hPM4Tk;kSiH?`WDc^1Bxa5Enq3+#@@V+NU#7KWn0CUaZ@ZRtaqjj-} zZejC#g+L2H5#(cjY&#I)nS=-x-V9=Rwd=%I>|f?OeAzWuRG%59d#^h1lT?P`i@`;> z&EC-V6G*iu;@Y^YkO225j704~amyf)bDuUTk|FoO6G7~aA>;o}7{c^bL#yZ6?rk#! z)#KYf4Zhm-TlGh{_x@#lH$od{4>iu$;!?C>pHQR>7PBdT5W=DPe}%GZQ8RnkM@j`C zqvO9^fUle3LP&-@xbm)zY0!NGW;e-ZmwQs>e!%<+6edbV&)Rb#IN&W2B)+3SbW0$u z_{^J1AK`1fRm|nPyS=`#s36|%q(ZfK4kYQnwUj}&#Y6gX1($fQ;DOh$A+6*qCdM`M zadKe*1Q3jaS_Vt8?PZiLkH&9YNRKw;UNepu1hoF2xJ;WGMoLQuGbu<9pqq64BExR~#BWCtH)q-tYijr-eJP3Q;9-v;(GxNx$t=c+)8e_G z+T12CBJD!DaF1+{13Afoe*4WV%A>5MrK~f;YKDwUs!D(K?-}E=3C0qPbwq>$E4DJpa62@x$WIwLmzf0@yAJvytz;#l!fgb^=RC)|d97 zK(>zYyN%G_$W0Q@e{z6f6d)(R$t7Xg^{3E{t=@>&6c9Ad%K;}t&%00}ZeOgx+(MEnT*Q~Jz&kgCo05GLpq!ivZq?WYT_ zGFtl)2|j46iquB~z_4;*x1Ansmnrc_u#F z%%Znfowiu=*=R`%*jSp`?s1)_r_+xIkKTW9G0BUW3VfvQeQ2n5{0#Ap-r2@ViwbQI zgQ@zfZ8`|QP>kU7o?Sva+w~hxVf#!cf7!~j86g8L zXjr3C6*fmaO=2o^SW$-2AG)|EmRArh0w}S}p`AKlE+H;3d-c$EtH#huP+Twl4(I|L z>}8~H$BkO9tdsb}NMd7ER$^_QXb-bXB=(d&_(4(?o-DgU-dh%yfB9$AR)F=EP+o|> zrTwmEd@kQLx#9SYyy%f(7?sma$y5KURkWj zEe`$1PyD>jvEiYze2Hv<##r(X+i#punz5eiS0RxOyV(f*X&2nZKZ|WLD;GuF#Hl^G zQsUQ^a539X6Xs)Q&w^l!39Zi98Y%L06scz%_IKh$^6h5ISEr7DAp&d6kvB0+>Y3GB&HOGmjxxB%-7Sk(Q-{fG;2j^z?q|`4=`~~@>~Hci zb_w^{OaQ7RXAB@&xKq^hcNT^K(jqtfmICGU`hNG-;f9TamQ*;k7!v!MST5)Up}U?1 zCXT(g^@s+QB4~D=yMz04FXfLiOd&+1dW!GFW>l(HPT3HG8B2vjG4!N&EMvzZI^N%Z zQx6UIfblPR7#5*7><93*XHeGPVQWgO2YW>5;IOrIu(VJrVKH_4jKmH!bph>xe{mcc z(!?F3o)v8OZ<(G5L=Z`)F>9-GJZ;oV{|ld#L&{Y+-$BrpS<))}!(<@I?J-V9UM84( zl4Fpbo@lRpfv2FJ+ov&%B-p(&P`ecK4qIwSx2YhpZ+$uVY>;^;R zw;$u&(lS<8ShxD;3oAw-AdC4j`$8f;27e-Bytw4&*k#)KU@zINtEfMTwVx^`U(Cxr zwYN{{gsJ(jGd&FNY(2VWb+b=}fiNE;ZZIo%{g1lyNrlz@Hm*{%eFPS@Ud-Sq=x?GHyMSp(sndfHwiI=cB{_IEFgF`+1#5!W(@C? zg|gxKIH`7G#-@$pYGWxcJI+*!0Okf~W=Q3gmg$Y7cWh<%s9_fJ>tuFdK!ZHCZ>r(Z zr8=)TwgnrdBBSHBcGrBZW$sq7CSAfSvkHRD68)v|53JAgTHa2<$q0#1%`x+76)`}$N zm(rdEnD!GG*b3Ps$8*!Ls*V6^JyR5ZZ+X=ID$7qeKEo}{kI)isLp6Jup?Yr5^i7^B zmq$dx;yLve)1TMD_oKG|pnR3hiol~NN4Yr`6LVs5B8MHoT_uhm3gKwqSvt8pz&C5g zENk+t(Z4WCZ$x<2sXwN4y2(w1x$|BOfIbMIPi{-3CiJ ztia!D#(t82ej~!VhNoT9>jeG8lo`7^Ry?|qXGf@g&v2gZMC2h)qIU-R#HWr@QC#W` zj}+3!&(7kcoZ!Gd-d+(5mXy(9u@n}M{t~oX8?*X&GXH0lBwqw9S(Dp!OvMx2wBJ6LRF)dLfszHpSjG-hhRud{To*}1Z)h6^z?y|@TM83hyU}ojdG6g%EbwKiW*bQWrgz*TFI>teO*Yh! z8_VzZQDoZnH2X?j^me+~;72g%plN@@S5_kLF6Tvi4ao&lWw}K=i=1*7kmt)1EWNydG^k+!Q?v)?eLNpCtTofuf z(u|}(%D-A{nRvAi>GL|qJG07(0f~t+<-DY>c~x|z1bYDs#Xp*X#R$e}o8%)w4*&Ga z!d+Y89pwaj?;V~tI^ z$8CH)E^z^ARjQn|oH~NG$@#_FYjP&$(1YFrcKoiZdD=soal^GY8^1>J6Kye>9~Q^Y z`E8bmKSe@=;t3J;9-dw?!+$o@{L*Na>H4hcU>(4cC@OH*xo+%0;ad&F`nJtGx+4ANOn% z^26*f9eqW?ak2BCcXp<{i(cFsEuHq@v2&7ii zTwGLLsLN>cOtuTzNxty$-wugfdWa$8wd3&s#%m>xfBYa8f~>bypGQ_+G8A-pnWG=0 z5OpthXmbeFG4vE77NkRuYCdIq2P=>T{3L`W*bkK*(8pz$kR|Vj<_?7( zUF2k>eVIrRquI3NEVF#`8C}FU&CuiIjlDrOtIx29PdOLAw#3^_J0mZT2s7_?luPH& zilai37jT$ExfIlGBk7MGlYYz&jhkZ*9YIW0U;lD1)INkrf2!wz#@-uUuBs`2ZRBbh z>b_Ik_MpRcESi>UPWd@aKJ_)FgbG7nhsX5QR!nSbKj$T&d6Ab+Ix1{+RL6~}=`^n7 zZ}+aXK{XtiNFLdDE}s>-jg7@jH^2bJ99y3u zhk{RYnaqk23;J^o$1hx;jELW#={IK}z#Q`nyTjgu)mYE7g15Tb46>q3jSs~X7l{W*T)rf!#(hju!2GR>-~p7B(RdSZelmaX-AT4?P6QWt z-B{rc>itRl_|#W^|DgziZqEKyuD^ciODilESjW4(ewubrSbgRj&L_tkwMhDER(pVG z1WjVAeps3aabhVN9#fIT)Lzb^Z^HSjn+Z5R4@cq6V1n|v4iki8koH)}UCzic+Jqo?q*mn%dQlKY|GFOX;~5-% z;g|vqDF_g0(^FN15=+dL)XiDqhO{cX+wPpZ8JTd*7T-w%z#N!Th*$H-(E zolAM`(^GZa{>^!vnW-_q6wY*aXhcO%;?4zG6zR^$*|eI$ zNuEr0>@({@kMDVSR9@cm6_2kX4kdJx@?UBk4^odj;opM6axcwDZ@ibYWAef@%F}zv zOj%Sn>j)>`6llB+=WS2;9Y$Qp=0CDa)`SXlcvzKsQCK~xe{d~P%eXF|%V=fR*giSs zKrong7T{2(Z1y;$1a=ZZ;IM)6C{!2Y`qB^0a&mglr@4eUBbhui>6T^1+WEU=RBZZF z9RaZcAWrj=KRmB$6_~^yl!u*6+pP02@K&tDp2-UyZpw~t*AqAN&;>s;G5E};m-e-7@+AQZ}Q(N{?Q0h5#m|Q-t6-{ z-sP&R4sKrXs0|MJz(wFB_QgVBDJC*Cq`gA6!&|X%V2c7_com{gRAVvtLnRSOV7j$) z*xO&jB7zb^0jafz)l~guQYslE*C1FS{o7R^Mo?Lh;yrn1LzYscgj-zJ>U~+!uabz! z6-Xrx^2ycSC9rH=w=je?<1JO}XMSTNNO3{&6nMF^ia+65 ze{BJ7!;f%_uIk3U#>z`&94(<~KEl}pE}o{#c?z*ZU5Xkt zd`a+_`Xsv-GUd;e&)+wN?zT%7FTvnIW^#BA8RT!#+|;b+Q~(fsBGNj$$03Kf_?8DvRKpyXCV5s zR|f>rb+5HxxRKfJGOA67M#*@Fo{539y|B=P8$)G^d-B_s3vC#6+oL;P?Jry8&Ieb! zjaA(`wjlT?aKQgG1^GlhB&$*5(GJR@72rC!UTLjmUOR|uzEYsIY~yjw7I`f$Dl5cL zqB+%tg0!UHU(L}h%gsrfx6Az*2D32{$zvCimJa?1#jUNLN3fc5%u7~ z_&b4<7l+B1nj{LedLJzOt+$&7tO77daT~TW{p+YyW?&+x5Iwv1&TGB;rAtZ_P_$}}jHKNZwW z+VXbzj7~Q*(~fiG@Ms9#O}XBC+^VNgLo1!H_lTHXHJ0CQ_xZ(c>#<|Nyd zXB=foBu`D}o|{eLc2|5!PuECOakFE|BRTx9)#=M_g07!$2V>r}pEK{B8$Fhj%+vhq z-Sm#G%%9~~otZiOF^N-UmcG2 zFBM(`an?DnnX%oH9=NLMWgD)d>{h+nAu*`0PWrpX9{cy#Np@-&d`)k2!B`ACM_h0% zd)q-7TsbXmT;k1wI2>|?6@P~Fd~3rRd&wgbqf}7x#ErP zlPw{c)ST@m3zPe{XH1DWTv6q3@1mRD<`pjpxBJbe2-(xkIN@lV-j*P%sJK2>Vk6gJ z`){jB^X{W?aPMC#B@4GiMRpWfDIfw?$y?F(ujhH~t|XYwo;xdih26F*bR;;TWh^Vj z%8qnDJQK>eoT=Tv`US6#8>h#{IcCR!G9|Q&20~ua`X(OeXx?=7-3&Vh`1JK?Yl#Ug zwfUQFgL@`G&j<9Xw%FtPaY^C4Jj+kM_dEh=NLQav53f$g(tYBX!>A#J?G&c-y3hrt z^$ew;Vwy(z`n~ZM^GDiw8X9=IQ$);R=nwy9uO$|`l2e%e(roeq0#5Pq+i~%wGbWVD zebD6sV{7cnBJmoWVqx5LW6Kg7;w*Q%l?6&?tp&H-^pjTI3^)Xc8wv|gy6Eyy11>iR zvqPoAcd8=Wt;V26*n+}_{cPt`lw0|je6r#sSsllT3a>b_liR;!qA_r{5bI*JAD5L$r$tKw%Uj5iJ2f_&im^8<8@wKPu%Gokdivk6 z_pRk?w}vjOVbqbyItMsB4^byK_gfj#8b}1 z(~1NX3>xnbHl=-V*CD_@u!+4yu+t6ULV%qrvgG!`3IdLk!fo|nz4nhO%G@vo;qv4G(QgiD6QiThF?sLf}gqL=fU_q#P8vgXBCv76rQ4%IDJOG zh8)g275O^0)k?z(^KIOfz_x%Oh0AeYr{>KQg9A>+-Sh+Npk#2Vpb9eP8gK0e{F^uD z+LlN+fz*~lkl^cSW$*dUG8$jgk@Tr~&uNHGXj=3IfAWDRejEY42A6O=dBvGc9m^5i6&ou>j05XA7k z`B$Huc7C;FMj|Su)2}uebdG&f2)rgph1D!_HR*`t91UtE8voX1O~W zJixsa)qZt9SGXcHtAgye>}MLgzZ1$u7EGyckhzYIz2X(#xw)PR4Cp$(On}s(WiK&j zwoh-zG9SFJryO3>^XjO#J<-5Y+j*Ava;xxdT3{M5`}@HFc;-Eg&g^%i*aWPyFn76a zoMB8S7;ZS9r{I!b&kItU^vqQaug37nqfW9WJ9w`RVk7FcS>^216r zgn(~Wk(n>`2SW;dP@K{&FJhm~Yrupe5YThyV&wEHi)lu_uISDs=uv= zKKE+u?Pah$G47%fLica1@WF_Og`ww)bTffb=7f$zYxDVNy{>||9@8LO3YyTIu6G-_ zMf;~Zo2i`l>bSIB!s)2VJavG;Cw{(fje;F@gqKf&8|AUvYysw5CXE*Xw-bWTc1547 zKE6alnz7B?@0j(y~kU_li*`|EQZ)Hmr&u9@BE zD%+oW<_|A-HG8kduBLDBjNmD*`cPg*una5xfdVfLybmoXH{3@%9y4lbe;Hwbr zLzX$|`f^?d19;+fb^iMC`=XGHCgW&<8X8d1n%L(=(2iV@fy8DAm&~1l?~yM(3+Jha zB;VkI)D?1#{0V;+>N_i_GrJ`C9kCMjy$b?Du-f%!6S2u8t(GG~*>@8tbH3fw17KoO zRcpddCV!gk?#eRwYq8VSD$m&5yT&RspgymE9a7+Q6g!O00{QaZT!Dtcci@I^soG~p zXeRr8BaqbuY#~`s006lM)j0Rv&1Y3`#!PTnQ)=2l1}VfE=V?DXOJdcf(PfV%c@v>o zUEt?sgDlmwzgpcUY7e5@_^fKEc zy&5k2=bie;w8j&!wkeDigpus+)3!WKyN`gt#tsQ&#}#MKmC}2U)J{svB_GV=P@=xb zP^sL>7>d%6^i7pt@eW5B#S?#PykM4+VnK!@9FWCuWSgk~l~|!-)ynqtUxanyl9ku& zgy-o2K8tg!fB&JSNT} zLO|IB+bw^pllW=`)3GXwRosYLyAu~DMDTLok%0xKDS(_RGrLxNq!4MJZ%!M+R|iJ; zr1<8>0Wq@td?o3YLOlv?z|>bWTgMjetKPU;4-qObru8sHo-dl4ZF#3!N%4~qI;XJ+ z)?2jwme_z@xhd0Ow*@oY4~UgY!SACXcB`_23UR3PnlsS(7&gPCI;tQM1G6`<7w;0*xY&rIZ;#vZ$M z?~keG&?tJ$MP=<(?{%hc#1vgv?+89U)@i9RGZ9bx!+13rujhS>)@mtC><{7OHx|Gd zzVk2q2>=y6NrJP`@8M}c?lhc`0SIB?*pKKkGBW?6;J{!@S473^i(%6W;CPnN(GArh z*+lSrg`+fSUh@-1f`E!_1E#+m0KE*UngsmULO2u!W@f>Dp>S{Keth2E|CW>IYQh)& zy#bxH^zB7R+fLhOj5pb9Jg2PW*k-r8L)S}Ibx>JH^R;!6=38O&lJHmZe_i(fC(#`V z>{R3DPdLq%?tag=Vd$kUw{%)sS5>%Q;8AYu7sXfS zN}k$C#Ww5%DP^uKFK#VeZNB#U<>^X7NH(E0me=A&x>DNGsm*4}S3=@ntPX66GlZ7A z5s0hXpaps!zV_RZ_59IjZvScopD*q0{e=t__P^+(D1k5N%QcS|Mu!D0#c_Ei$M@Gl z)LEnHmTcaZ^~sYPj8$?`BpJm~rDlPO6POyc_KzL=ZkAKEa(05Anqfu%wFP*0to^66 z7np#PX@TT=nSI-UAuxnLu$|rFME$7VufgfK@dC?c=^X$CIEDabLeic;{+hJ$30Ds; z<6M(z9GAed1-9!$U_T1rd%00>G!7tQIY36b54BaF^c=|G|Uj z_Pz?eAEYnZ|8Ki-U|q3;Ww?41ZTTK;t7aB8CocdHQ}pvV+7;mB&VQ}N0eV9XNMRT7 z-Y-my_*()`=MfeSeAu{hLQSaAS9gz%o> zuJ6AD2r#`4APbP1&~hq;GQaJ z^-fz|h@RW*X1AgY(QRdZ;AWzbT)P<&%U;l^F6C!W=WPT|wI+%VvgF4py9M`Zy=NMD z?A`0xBG|M#d3+Qtnu4<)zU;SL?q2un^kYA3aqcT`V9VV34uc!L})Ce_?d!biJ zGx>9bv16_ltys&Z+Mem!@cm6Np^K1{lSp|#lW*>4of(FEjflXRfD)~GR=r{@F%#kS zz#;>RgVbhK=+DCznga7vkOzA=zA6e@EF|@g{T>%=`#SIvpi8?hgC$ zutr%y9BdapQsZ``CgT7$la;>(Ib<=c3o|iiwTw%=R?cUCCN~TdLaEt*a8w~P29EB% zn5Hj&qzKCZ#x-En?j4(sjBH-&70NDAv^MxkZvRKLA~r6@G5$lxD{3A+V5*$qW*A6dmwTbW8f2Md#~8$b{7qmyB^X$1bs+^BnY5#VF3LAaK%6b za3)>}fsJVFA9loV17`Ob#nVlRJn-Tyym_69NX#U*E+$_-p#j@Y-f?K@DAJN9d|3F| z?*g;awMG*GRzmZ7+7iE;j!bGNyzQ@Fovs8iz^KeS)kQ%&Gh3fJumZxN z3E<|4To`wSQwaaFPuwVU*nxss4ajL(A#P*22{_>_|VeoZppVl|3oMy!_MZA-)W z7|$7(fs0{n+^-|k#Uj5;D@fibdT@b?QL5w5+Zs@fKg#(`?e#=08&ee)dQ2T<_*GB( z0n=8EJ6W7L93D8uGxLyz2$tM25hfzP|xqcygCUOFd|~IGJ{|gH~EtIK@xHz(9w*{7~$D z|A5}G@GIn9R3h39m%@NG|Hz6+@BT=Q1H)`v^Uii|hexEsBJ`FXeKyoeJ0{sy(ZKSu~XI4R9zAsvRjjq$a}I^Db^=m z6&kbPk7OJy;z9>-WWtrlWSgFIux#J&urJ|GzPU)W2GHqx+JCN-C3!NjlK#NLCuGb( z8GOzSzQsAo$f^EABe*T9g>B=Qm;Qu(8oE4O{3%y>nbT#8N)n@i)RBSm7wS2+f;&tf zQ$q5NvLtCbuS`5ubzqy!$NM>FNck*>(FG3*BN|g*kEr%BS-2V5);+DLvm=1t0h?E90) z5~C4+$Z%6{a3=rckM={IVWG(y&WdL7$tWY@TJI+lD5WX-@e*lnE46@z|8UW9T#?KT zg+c+l4?9&Y=>Z))FcFYP*z~F$h}qfc*uivl&<^%@9bRRntx?TJ{zL80E?K-`bnNG<&Ez~|8>-=gS zoH`%gM12E9=darca0^b%^3Z{Sq-5maMzGI)6|i+Jcr9i=%rjqcVKM)U~cJi-$yEf(t{68g>1kbCe@dtSdW_Fd5hi9X}9WZutfD!u0 z2zkBfovaKB3U)BHKMMYoXr5$j-Ne&TEW_GdV8ZsnslM6#<{45DLzwB>Bhb8or+0yD z!a66dO1|XI$iy+!k%}9_!WCS26|ss0PQznV&m|UwxRw=C7SN3U=T3ulpD3l4g#QBG zGAt`(g!tUlT>aN>?ypTVENqrJI-8B{jzYa80AgqF3j>-{4k9O<`2`ZY(!2KC<{H3KY@= zj>4XLj8D3-*BtlgaT#A#5X+}TMbm-Q72gPn)qf>|J~21e{z8k=w}!d#uWZ9i?%jEl zjZNtci`VHyzeBw#5PE}le)XE3U|*ayytc~l?$M$f^$ToPjgL{j+EndP)7uiq#kO4k z_CjH_FYKQ~A_{NYloF|3Ry<~;QEyx_q)ZdmJX%5%IhTaESQd@;9|scgo?&Um226)xM+b$1z%eI_h?86}s4Jp$H*L1L0MCp#z?X0MLWzBPr$1>c#ooiepeawnyoDKMT#= zI)${4SZ8f1;BY9tyF(kA=-k%$8$ZW`w8#pI6}bY#5SlrwO6`#OtGIB=8)M&MOvJ%= zzVMpC)45(Z>9o*57yS+JL(h;gioKQtwm81Jj&&4&6nrtDkQ?BDuKb&CH+UI{_FveJPF{7`txo6$V(bk3^u^151E zE4O45`v@hdoBd>l4L{{txm+--O^97!wL_zC-K+bnL(Q9=z*4Q%WKY;v+E-_;^rfB>qVjRTgq3>Qsy~FF9xBvARhc2aYGP=fYCB;Y4 zN&bX;3ysU(dug}NzDCmLt6K{vUYZwUwPhsT<&aeApPIY26%j9=0~Jj@irra_l8 zX7gc~Nzeo_@pm{Y!&uzG(Tx+ z<&)ae1z$Lq^v=mf44+p{Cf;bm2}2p-xD|NDC;ZfJV|;g}d^W>^J_Q{nx=!`X=J3*%mhVMU^uD8 z_x!k}>=VjhmuRai!l7B7RB8NzZ>SA{jk32DVzKex@%!ZxSvvBskr85GTqnTorYD~7 z)Z*%;=E5xB%+H%F7tMaWpgB-kDnRSmFHDI{wYDs4Y!=nsK)UJH zrt$clGfEN3r_p>KChl#PgK4^8$*ie0CwtyxNzxZulKQmlvAuWFycQj|(ba8aPnPaA z@HK3Mkf%ThMVeka>{Fp}aeq~&cK_z}$+y?xFyt{EK{)b8NS`xa$DmAMAVG&>xQE$#%Q`Ur-Fp^^5f5PZ2@AtSgk=O9VA6B=(Pz>l=33j8n^_w}E-1=9L3 zyEDEg7GBVHsFKzgfjb&A8QDRJ3XjgLbc{cWjA5xGSAxXGCyD}0F(M!Y!MUFjNC0m_ z)&t`w?SO3T64;o>gC?|`U`u8BQC+iu<$&y~A(6==nX>C#80WBsWi)&~-@Y@6;aP@j z4aJaJkOhKBIciXPGB|f`?0~3EVo+LQkXhlY+Y~j^ZBPkaYTO89Ms3)-*~9OPFS3jE z)1E>}&M#2vs_Zf08Vu>$Bp_3Yz~DhOZpL{f?sPrLH7eR5PB7V^WUCVa{V#Ma&DO5J z`tXxKjWiSVYC7((5Uh=z?JTsbWNyaRHT=}mpz@nHR z#Z3~^9}4wHI^Be5^hfJU4jXH6iP zZmvJ9i7H;3ut6>1qHwD0E7aWPB{Jrk2^51t><_tP4%KRc*FEcTEKwmnc#xVZ^-V-sKPbLe*;QUA!LN>`5xLD6{vL36OI5 z-{VT+IZzeePnAim(IE#ki#yHFOvODFBy!s6i;<}CucLTJ==%ZDmXqd;B`v** zlaxvAdg5`m6i^HM| zKgBi6*28YuqoK)MrF74k*VK=2@jd0LK{5Wu*bfAOS@61-T!rDvLX=lpF!JMC-&|Wh zcnR!&&zNyH=1Xum{Ep%F1~c6Q`4_w&HHmB}vH*Su=R%&P?7DykRKG})nWZid-8t)=PE^ExjCc-2sdoCMO23zGXbw?E$vbQUcOQKZnyxu zu@G7=(z0p zA>dfOQ%!XdM)Sg~*1cxqj!&gORoDGJ^GyXU|qU>3{76vJ^ zpVkN=KHHMC|38eqWl$Y!7p)sy5(w_@?!n#NU4y$rf;$9vcemi~1b2tv?gR<$aGLDB zKRc&x-70?ZgT-2_yWcs-eC8Ob6;WbGK7XWpeiiHTZy)|q?mqzifUcYo5&paqfop!X zp=kjFx2n379}6C^QO&|=Ud_#lAKl-bLTjIEwhcp3);0zaaiQQ)36eW}C;Q$euECKW zKF4yHZMk!fR7))FHF{pg^9AC$8VGw^;x03EaCN=^Nt*_@nLH{Ps;nFTcyknb*JWte zhzde6lnU?Pe+ROEK5SN^IkK@Bb~YQ~=bHmU9qcixIw4Pp=JFNZNQgYnO>~@1ZL-z5 z&-RJm^iV^22ZDz>+L3X)x-pBp7wU2{`{p-QU)eYIm}E8KhKfVpP;sSvfiL#O#EK1l zVY%I8D@qQ0y_J06jO>657vLE!^=$r$MJ{y)V;unulMF|1Nme;J4OA3o1e75iVjAUh zZy7>g6DHh-7cl<)__z-u{mxvOVPsLXBtXH2!t66_V7+sVO3St6j}J#gzBf>t`?I!( zo`u2OPa7!TsWlrA70nlH>8_{eD3po@blXnQ8OW|uV|C5f`*9@C7+dUK)kK6-5Iloo&6k8)d#s#&9tA*uh z{nX}r65^9Kpi}V?n@|&$*t&C;k5x>;*aXyV1<`S?0W6yLdmB{L4+YiKJ=i;l$Arw_ zwX%HeI7eDAzvsvT@>)#I^~gE76B&A{Pao%+6m$V>>vx?%m>AXfSNI0iKGLQwWOGo4 z>e8Sn#LSoq(Qe>#D*>Yc_G*Wl`tmsoI3mt1IihoHmf$J;SqR@GKT$+XbJ%>VuwD*e zMDdkrOBH)UlcOcX42;Wd>g4fV37oBrr&5v?9O`&(;tuX|)-ij+``X8Sd!Mj*KR^fn4>!n)SA4u$P&_Lw$7NNNGWjPHn zV1RWaEIE>NR)4J^a?rVNKV7!rainTh%dC4T3m^Y^^u@s~rInj)wKTH*{f*`m>gO7X zk$AMUHm`3dTa?Gs%x8eJ zxWDEbY+4_pIz$}Y<2#E1WieH_sca{=vMsSW<3jl)UDe!zN%`*d*GdEHPw2&z3I$6z z9>z5(B`9>xP|teAmSxH*T;Vosp$__M#4`bn6b?BbkMs1*;JzQG!G z>jPD9Sh1Ma*o~a4jLC*iD161;Zj{D|W zOy(e>BM2rRu6JI#-qEt`6Df+Q5X>cDs6E@tRkeQSYe%MSjn4Zf?H*178uL`-oAUwi zPG!&mtAqdY`BSug(unq~Ao#W4j4$9ANN*$5BY*p83MDG{g(0&j0#K(h(85@)gY8C} z@LB^%4@-V-Sg_BE5Ju zrkcQAyOvtP)1#%duI%wRc4J#_3cNxIvF1C!LHmyy2t0)Ej|*H33sL}!3^{f)zjACBtuyiEW{*6YhP&V$)}`vKz}k-&{to#XY(5gQbV zs8KF@qe=^jN|DmhyLEH@j=u8|3#YSuEliV?CMzw$`VeRSzZG(7$Vw%X|pO73b64Lo0_!{8luA}_b#W-ireYSHqT{iAG-q5#2ZKj53p zd#fr~0xSwz%KP(2&mFM=J2dw%C(05E=Ul2{i2_ROVTA`@rLTq=(n!D#8Jt74TVnE$ zH4u|v9=7qAR=)2T?erhnCNl3yvR2Va)A3>2e<$8H!+jE{SA7Zm)b=&s4VW4oL@K=5 zO#Z;4(Gy>7&ezY^T7o8TF**sc+7QKHAnuJln7^Kqkyu1a0Sywd=qi!O*mO4u%Ndqi zM57PILcKsw761FguO>LbR%+i?S@*{bI(3&K+=UWYLF(x6+U0(X8Du7>2m0^;%VHs@8S%JLpVz653A*ifYzGW}fR3C7`4!rbDk^ z+tEB{U{ojkWaqcCgG6Yl9IWM+6CcVcZyrq+3>eyv7`{KEjBZP7{JtTZ-r^^{Bd*|v z173?aTr-52IL=#DpOn54~M4NvKRzkI`d$H=|XKcfTuo z8{>o5kDQOSp>k6}qFb^}TSM5m@n>I*C|+?^o+gP%h>Kr-eE%VQM40~FKOY#OYi|hZ zktM!s>v`ES(>gn7w$7lRS>ab&T(JcWjSC&6pX;i2*-8f~U=)$UcR^}T#m?K!_PzHw zNeCuBdGab9Q*e4Pu>Qp5dDw?{QthRoQkn@mT1~#;SA_C?P_Z>S{yTYu1ZrQ$6VXKQ zV=N<&(U`VkPld1Wh`DFvD`ds52XW%u?`}Snk9gi3Au-{g6=Ve<+|aA)(~Mq1RZ6&S zi1=sc)PQTRLHg$s`|)@6sL8ly|JFq6@a2oLKj4Q2C#?rTAwkp`4AyG*<3R#M%wuz7 zF?bde)yjmY$aFH%nIDLn+98MBIF|_Vif3q&enN#1h0(R3y4R@O(-j0sE5waB{n`p$ zTN3l?gHf(3SiDA^ycI8!sYuf_e@A1vA;n0{i~X!u>>^ZmVscnyW&H{A{6eO8Dcq>j z5TRuFE#ix*l}7g(srHB|vJyNKB{w1xD7kbU=Y$Ln;`{EqQIpeG4qEPTR>45D!7(lo z$#PR*nxY9;r>U|in{z}J;)6LWT~%)k`ci1e8a5iVDxpTV&L-~}15xe&66j8J>wrX? zaz}KN;f7gEdpEox72iTKDM>Z2w;?j0YX4D2^OfK`$|6mcG0|-~aBAo~R%gYuRF)C_- zcdS5X4Kr%Ls&;)!b_xLlyE)^j&FUN&=ZYj?cUg=^9Kh^P3WRrMTG+59RG1MxT^${MW*H>t$4$@mfB767y+oSYjgT=}6$wrskiJI4@QNwh=qUt0IDSEG?=RFbkn zKUS!Ji3Y}UbbC%OQ*hoYL%wy=UwAsC1rpufPiqE#L#fJ5{?W*Gtc^5eQrPL|AJVMw zvbw^bK6KBTp%%{LK*qx+{y=H4qAl5M7s5&Ajrp+j*`J-vJu+EO<(lTYXigrVJ?M(e zyI%_{QuJ7F9hO?8XkXG^#vI463cQZ`N z5j|)}_++pF%vVZ!F~kaqmfZbC!$P{bdo%Zzeb8d{Fi-+;=7?~*IMNX7M`$x#5cfq4 z6Y@57u2cEb#a&Mi4Cx}chJg3wyB-aMfRplfc<0_^5;C957gA_P78VUneG3R^x|LE1 zE5XeBOKbJDiSR*)$nBhnR%s3)U@@~aD$WG4%FCcc^H17@&1=z&lhS7XP3%eUV&@PoMp28APfpu6!&x#K1VJL5d3xfHfe zKgi0+MdxWWHJ4HFUE?y0>ZD__4P8SyT0f$Cc;y< zSuDB{{`ML1P#`lkE>JI6;b0jJ`p-SV0!@xyS~88iu*ryf_Y^iEXpTh%s`b7xnHaZR zRMKJ^=3l|df`aGb%^-wv>tG~<&p#UgL*LLrf{t5gCj}%9m)xTFVo|Yy$|i+n^=R1z zmLla?7RLKi`wM5$;PY7N7!b`eHStDMOwXSg3W3{Z`EY8Dg?dn5XkbYA!_n-|u6=Rz9X$}!G-y_)} zFtUm7o#49Ls}wL!g*wuOEv|nixNW_GyQ&$L+Lu&)H6GeA0j=^pl4ULd5 zn-dDw>L1ryPl|%RZj3avA($!B{`P2`+|+c+LoWxHJ$N=Nifx2?{T=#oY}G~AMU+jL zJ2oj>-qhwBH87_o50sn5d+)8XDDS?KEaGOlqb)U|*35_=p;d~dF4SE5II+(WS{PF; zmz(-otUo9!)Nafrub{5(YQBlR7*hpqGfX@J>~X);eR=2GdiW&8(MlGJb>GFu!CiGR zFUqH3FwFUx;3TWCI8li`+&fDWQbN>py$t?g3Hb|d^)Ph#9w!tF*!!PYh7}A7sN>&|#?EOV{pITDJNlgF# zmID&=tR)c_HkpJ(>o!SkV4cv9UTClqFbAOOA}=Ru@;RSw{b3-59o6p}i4H(r>2*$2k2#k0DyPe1wvkDkXaF z94K?ruM2NssYylTjWN2_{Ff(Vm*r6@#v3lm>bc(pUFXrg*o?t0cCDA_C(affoL=M z7rQQakS5#;MI{ot?E7jnJQzu+uLQVa>5DbBJP~JzMD0o^Zpho(hso{^VMkEF;V?w3 zZLlcK$XFtVt1)Cy1FmIQ|j~2;F9 z;gAY{qLMV28#r_4PcD1%vw$kc#V4wWVOAfib+WS6o`~xYa=<+8k7a6V+kd431Zl|M z2^4z*d7cxIhefNNj6d(2_eXidUBQDp%+@q2^yN#VhE@BnVS*y&qyC58osw8$>uiVk zWV{;xWyg8^XRBMr_4&e=iL?}vS8aHp?E{bL8O@AO@_yR(Y#(1@$fy`hX%yoc>;U0|Qu1D$!1)9e)D-TfzNU4o#r_<4fs`hK0f+*Yhg?uM~ z&ES};{1iu2{tnxj5Dl^>5ShbWjfocF03A$ogVa?!-?p1lUcN4O3yv{w9A1OOt*~P+x3ae z$Ed}GXX1%2GiXYRp_ZgO09z^P3e}fvJdVmPPrz)kQZ*S*_XdAt09>9ESY*f zybNS`fhcA%xKm@x@$2uxeLR$mh;@fUARhgyX_r=);jSQ1;aG}gWZ0ftBWa7hG6^_c zeZY2%8wpI9u%M6pP(KC5eK5yG>Yf6GaK`8~IV_re`FF^cf zx1X{3oW#62^|&y)jT?LWoVaEhs^u+SnDx-?OkzQq7aGcqi@(Y)3SY4g?sC^f$;vqU zHVJS&xs1#hNm(D5!Mt$5*x&uxhICJyg5n5(>tcyYsL~mCqBlUvZ(EJ7!U^n?}_fjoqmo-%bzl`$KPYX~^$oGOn(QH4uO8DRccJoJJQ-0|il8BL1Kc9!_@m-*Ta zMBF!T>~qLszk}6Tz7up=ZGrF9^dxC4mS$=O@)}PYG zo63_-#X#+Ob7t_B%O2k`QLhT0dqp5hc9kMpq>kX$0FTolur=3|F`a1oXs{x8A>Apr zT7W& z0-1@$Mw^H|Q&RsmbHLFn-=_F+N=9@gMmZ4mc22B9-Z!DbC9$P70&_22M`sW|pVIOA zck1wPkq<>o)os2w9&UWU;-qN_hz|9J@zA`iQiF(rG$i;G1fT$l{H>KLrVOUq2z@3P zx9S>teb@x(Uce^ub<{;#_wlReDgQ+$kd^T)n2IN!rXLp+5hA@!GWe|GEb_F$_??XU zd%>H8zFj@iCp4)E8Ra4)vNj1kd_c6XufF;hkluYnBIWySN7vL5CP8FDUw6uG+|kJL zao=l%6ZDU+p49J`jn~AF>)f@3$RwLDw7Lll+o9RTOpb8NRI1A>O#^{U4~5&jw9#km zq~mX?xe6^reHD=T97;qT8hS(yKo1&W+LFske6>`s3N8Rm3jnwI&593qixcR>q7S_D zHxM5z06#<6Hxt4bNbyucmiMyHiMo(q@R;b;Mp#FK_Prd1>aA@v@Z8TE3e2R7YZXg9 zu^lPe2h1{_^%5QzK#4iimY1kVc2=luPIw%m#ScAzX&0KXKYUanUyTd=&PwEqwyZ+2 z+`twWdi|XY`Kv1?jz9UMI_c}s@YZwd0%Al0ZQN7(phA;IHX%)-D1M8!l>pC|l+Y6? znu7&nk1D-Gj+5nKQqx#+gyW*bOg%*t^Y2c2EGz*!_z9*E4c3Q87dUPQx3Pj|+d&?x zp}UQZ9#0JTf6H&+|I*N|fK-xvjZ2;GJis5aS)j0~9S<YUp@O`Z7(c#!89K zx1|Yej!&_1qA9{e8;KRb+{(nWV8)Owt15$jOv8@oAz2k_Gd&EGVeTW{Yhq8^^`)Ix zdLOP*@aS;UIuq9j2er01S;I610tl|-I1tf6Qn{|*>N5O1{y%DW0R9{n;T>XF14Q6?PHE%~QE-2?RWAmF_UN*r1L zCj)&f6>uX}eR=u}m{AVuoBWbf{cL9hUShe_@mUbX2!B5j|CrJY(q`}i_jj>HGY5Af zPdKaO5=o31b^f0iOu2@mN2Z|c) zzENrtw`#>Hm*QZv9-UR-;#`m_PrqkqQN(n_Q2U&HvqZ)HrJJ!l#qz$)$Ugdh<5KnX zl?tvtM=b#PFAWOZ93z}9z3*bdPtI_X8B8FWlbrMspU8wmtEXkCT=(|zV^9MSHu~QN znD5LH%mSb=i7ek!&`lX?VZnEP<_jF#z4{gc8iQBq9juReSWQl+1xceCp{K8YbfIw{ zfp0csT@d`Y0@@#W?nbmu;o{dCLnO|;`A?1qHVq94V(r=z(Ti>NTyN4SdB;`Ovd9GN zNNHw{5|bZBFdErsL!0TTR9T%@kDTMy0I@MLlp2NDHr}7sBE>BH!q-z6%Gt{^u3A6s zk~md~TaK#EsoCc@T5?V^6V_Mci?5O26ODbrcVH z6i)M;6r^7vFE&wnY8eTfXo!`y@X`3|B@ zeBB5A7~a%Zx6YSM&3M>qiBJIaPNnLY$#q9PTXA6k`yjCqK=InmL%qcWQ1@`N_RRL! z%c|?j}~B{7{ysb5}5Hj zo3~jX`4say7PMqkMvV^~^4PZfJAU+@#Q#9pJrcy==I|_b{C|yJq6-BCsx_4>wjQ0N zQ5lE@g>9Tu*z4lDt~~LVI(W ztd#69c9^Ks=@NX44R(zsklzqrfv)$8UM+sLIkCvO$~TIg2zhi7e)(4O1ci2m;D5`d zyX5mY?L2f+r_RX;sC)TNv?BDfbPB4ZZ%lG2>ibL7;x+U(c4C8EVvKfVis`GNwsg{S z2kSH0y~I0ONWEne8Fvg46Amb5?`1`v)JpT;4KjtOgUr-*pnXUrf!uQmC}<3?BeWr9 z(h*;z8EXx~YVmOKx-cjfGZZo2m&A9!f2~!tJhS*EIM{3bG48Bl)4|()}_LNLTiG?V!;z!nGSB zbar_}T}(RfMiY%q7krRvYLRc~BsMf4@fiFeGt^^$;E74Um);cKhiCQUIvj?5uR=$o z^p#FQQ-qGU$zW07CR!Y?@&RvDd}yW9e#q z>gKH~nZb;1$Evzx#3k3zGQH6sc@(Gv^h(Z)Xm?1??n7u)3^GP|?+L`g1k!>?e$kuH z$k$^0m}ecTE-Me5IHQmg_;2DN)!2TR=`ZYi`$ORCV!U+c~Z)U|3j*;7Jmi5f(b4%oPZ;>$ak4e?M%gL7T;$=)OM zzWcwop8#uyH?1m|==f&^!&%I)uv0s^O=bxTQ0feZrE1-~P@Jx=m}_*CP&X2ic~G|; z)>ttJNmnot_}F+J3#xFx@3_u{PQ=q6&k9yrIo{Jiq{|8q{o;QnN-!77AT0rpa|l2# z?e#WBch_93t_K~IF`(x6cV>WrfiZvS83Y>|Md&yHHRqe-|4u9W7^32;IDHlnyr4!Hl^GUlok4%UBpi~0lII`)f;z3K1XBrjw9zaru`J}Vw?0Yl%4MVyB-zZG&+`}wR3T58I}L#{ z3+>rhQ)W;(V$X~S6d;iP65h|nc@kRTS4%cw&v6sc!ZiRfI?2!sH`uF2LGhQ#hMq?5 zz5S7ZVS0)v@%^h^Uky)5pX7}^=2%dCB`9bMM&MMT5Y9Vunt!d;$&6iBUSKBYGDH%< zL1F5~n7q{$3|Cp)ofG~-QihH{eAqu|auA9xly4IQ(2iBK?m4RauOP-7lp@1I|a zHppn#>LyXoOQ>9z_?$+UJ|tP`eWhuCD60cqn+5rGzKff7yDPvmhfuW70-Y|3@VTQbQB}OP9d^&3!MP_2jl>&t`rw^M8z8i1NRgNAYVW`L8Zcznx%2==1g5YdYW^%%+qb&P;*FaH#Q~-mM!aqINL{1H1XndMfg4#-*O+tDS z8Mm^^L`-)mD~zC}|3$4jylL(nIX6G&-K-A1%4D-Dw4yC={DguvvK2Zd_#_tQ$ab&s zoS4YPiwteh^iP;?>WB9Y@-#CS+)`-R6qpQZhnZViZDqu>gnXy@o{F0{6(=_y933=X zJW@`Ru-tl8jgXF0rWl+v{06w$#NA-W71jlQ2D)r4T7NX5Qe9u-k`<+~u=SKZz7Ns; z0@zT9tBk|yjILeXkRJjv{97f$5=Cw=e;{}~nvh>`x~6%_xG5A*PBf3jdj+29N$wQsFtBx3$Ti5F#ya=n^rKLG^DbC(MQTXG`32_+QLKeryNz^8~If zt>3?bPIBfoyk4J=n_rW47CS-1F|Sv@AH8dlY*#P6n;gs2u5r@wseA`V4jZ2T9U89U zC{^jJgW?#&xaKFW|Dne`bxBqPBIUZTLh4&@>gV)&!MAFd16UFR z>TyLd#QUgGrVFg2DT~{{&c9zM`prq*E_$*zA7~Rifa=-Yf}5%)1Dm6Un1_bFt9fQv z5XXmQ*q7V*^Ujm+PUn_WN#1us6IY_Tj~UI~m7h48aJ9h5RbQA&&>55pxo`Z|?@)D^ zP7ou58H>%WAjU8pOr(5 zrsQ3`Y$P*5Rp?iMl|cSsdht{;E?G1}?4bNwh7uOI?88Fw2BBOu;}N&?&^X|uZY3h9 zsil3Sq8TfkL|vzRNI&DEO%;VuQCA)p^pC~^H|$}z44ftZrx`H+1RN-33|s1GlUD{r zN!(s7dicH8E+EH!oCU=I)bP<{;sf$oBV0erCJY6& z0ywjBb7&oI%C%~vo|6Y7D<4jl6Qq?~wpbYy4bM}4J5#ykXEdIq{5~)utLP?Bs5!Mq zZDP`4h7V*GU==Sfl=b4zS1BcEU0=i-LlFthV0yvRLf-e`9r^|cZ?gxQqsYH49f{n= zD9Q4BT)r8VHvWO(!k^=n2^P-OIXl-<^ERu0+gYH0g}E+9fF`3I)Wl!D|>s#*`X~+J@&y=|nc50EwPND*kdxB+>ypA5cwokC~FQ^QgH1Euz z)#tYlnN3ntQVwIiqm^#g)~y0liOj^vtm`;GGgy_l;O&v-fuFv%HdG zgUAsnC$V^s!P_9PpO#D_#}N;(Cii7vw7_i!Ec64$iq6`pYFs)y`=G?|fwkdl2y>?> zYAOjZaNuwhmesg`QI+6!&inisU5hB(FL!`Cr%4$6N98D|lXE={drrk?0F3`xz}A^R zS`#5ZbNIB-0Q{O9ZbsU2*H(&(pw8NpOMwc3+;k28+I+%hS0}wAYfKt>^{NeR_Y2MU zS6&zOm6mcIg=pUpM4U;5)SP2q=127-=nmNE_qs;j^oP}HJ%_L3yr<=3v|<&%<1!O_ zo@xe>;o;N~55EYC^d<5pzR7k)9}&uE2hUPR;=a;;S10LAgdMHCPSQ-3@Q=tHzB}E*GvLsXHC}R^2T`n2Y{XdTG1w(S9UE61E}*^ z;GXE~5D0ET3~8}HD+Ua#FQt*4?2PbrCSNog*t84fe-J?f4wbX7{XljUixO4{Hjo(; zM0&kZnn)X=R%#6*6Vq!2gIKFwuI0ft<)%1+9^4H9z-1u$_Az9lSXkyZZ;1+ja)s!} zx1UdJ)<1(@Xln7>hp4AgOvK)9AgbHFbZQURml?G3=8S(9$K#GqlAxAPw5P@8f7-AY zdleueQ$abJslFQ*F|Lm%+cV0H?aXKp^FNc1kQ`A2M#~j8 zOytUX(vv32gEgF_yP-C9%NLKHBY0$qBq$*HA;T?5?Wmxu24mjeB=NWg{Iq1- z(vZe%**_9Lu>As5RT#30YlJdELUEz$Lv@s6}~=;P4NM|DdQZ)qK})J z)_s@s>@Q#wgkZaNR9gqH$$#-Y>e^UV7ynoP&$uU*be|YJ9}EztfgPa86td5Hi^>9i zjHPv}szAW>_cv^&bB(7wduT2mZ^mSIoG%)d9zg5NpI1g~_JD8;AcUgGc*dy>$ehT& zz~qj8az8oU=3^SM!0$GgXJIplTzk4fn~nebR6YYhKbZ`;8LsGg;7s;^`GCji)YUEG zjb@b90Az0eqXke8KDeHWxn(RHPAFcuJ^QOnre{yFO0pa@DC0qhnu|pD2Mk=?4*eust4ZE{mv@)MozX4OKi#FPw z>$ePQrH9c~XcZMRMg+k2p81ym1JH*33-E3jE`{kAR`ng!>dfaQ;sdUP>I?Jm1@WyH z^*WpLSodG!xT9rbT3dC7L>Hl8*5B93U z23r$_z%=hh81ac50`5YTKc33HVKbN`vsjfL22O-N2~7JP+u^9a1hv9~5i;*&tfj9tckLpxgx|XmPNZ$7&}M0r6O8C;?W&%ixnrHCj-i={s|qKZ+7) z2VFTDT&=b7PlJi9*UXlitIhRdeB$6u^G834*NTaVD^&1c&B~{H0m&P_&p3elL-3Q4 zu?a)u&;S65yo-r#qsQ@#Kn(bO+qoM8vFJ04F6l=@YA2{%6{m3p zDBfbpqW9nxqLbw60E4sUmmNL|d}8rcfu(t0BThMHw!wIomX9|m7*UZQh9if_Bk?=EkZKl^u?}6`;+U}zU1MC}H7$2&_!afZf zO-}PBN~yS(N?I}*b*lEM;KFuV);~^bQ#VLA*U#Qo6GUMBm0|apz!mVqNc|^vC?By| zh)CS=^Yc)sbDUNTWYeh&;+wb#83_ME`zAc$R%^Mvp{Ofq5-L7Fe4d3V2;GD$w=zZ= zra<^O@K~2J;9oE~vH{Lj6pu6z7Qv25@id6+H*9sdUIU3Q$FDaORhc|hoWA$qJ&8Af z{VB)mx4kAWAP$pqy0+v`j@D=@tPA1tL>^{mnyN_w^S*&vDX@7BH+3P zou$w+L{JTek5&T;Sf(A?Q&eD4>8MCX?V{M&tVW63R%cBZd8h{=$SNM9Vd1^{hw=K9 zeoIwZbeDTH=K5Ux2*k`Og0FzUV#Wa9W+IHcC+i9(2JPYAGfo;~5RFdFD_)j(yvX!1 z8jtINletr1QHj(Xq;fhn?v%u^Q&Z1ZasIUgv$=Xe#Ak2?}3?7fX4WP7u zo|gW1XHquGe@bH)R>PYqDsgO`|!=W)efIBiMog{B?1f4Y{@8DC{+2XoD6+~z%) z_{S~e?7-R~E==|eNM1jy5F9XPXKy%qxTFbz3c0^N$?Iz+b;J66(9Rma;gB0|M zJSdp9?pzYebVY(~SsdyWBY+tTulWL2RSU2_@p}?)tei+?!Bm2JxoQg(;+MvT_P5UW$s+VH&cA)nX-$fU) zofo`Do45c9JqAcq^;^^-(k|)H+>Xp@iV-TSy=Y8gF`}qkwXlKtqHM7Q$=2k4HGw$p z=eJT4qwa8US$Wuz?MhmDbcIx-F!&f%(KlHs-$Iky7XyM!w4AsRP=liP zB!S^VV_0SmP`I%`|1%b35oY(gJ#kff#lG3TeFV~dihb>;|!5O(f0Z} zX|2RIJIPqLg|(dOvuLv8&BS-LUQ~BD?2`(4=sNSW3eDr+jB`2}T=NnrgQ#?2;WRyf z&@_|>%8&4NZHH^-iK#fV5z_z=ASib0kpp5JXslp=A+*>dmoFXg6h*>J8bG!AgP)C} zXnxL~H8OvJJ5r?qHSK zm%nJzLmS>EPj{^1q%i}qw&w^(lkw@gi-Rb5V@m81oeJQ4vQ;SZO{7`AlQne%LvVFv z;eGY5k_1_DW2O!lSj*xRv%a}wUFW^V_%-dmI;*$>ry{`No%^B4LfWVWa)rux0!>lq zyun35Sv|m*8(ZKLDPT4r{AG?~zii-D{f?SH)S%&#NR0zVpQu3Ykm!6%ctr}f1m^l6 zXvKpjNRS3vELCHDb?xr^aMY7&zuTLGFluBejrI>q>nRRZZ zZqdI|&4nPTIDN{laj}xr4W?X~;GxHB(!=x9Nr7oCj0us7`(H8#m8W9WaU#X@ZRro(92!{hJ3gGb7(BZUv(bJ`9(=37{;#SF zkXH=*4euX!uBUtZ+%ic2Bhqyo^5WrtQI;RcWN3;BJeVI&Se7HpoeG7t%@9t)KHwzK6?7)W%}4+2ulhy$!}=LJC5N#hx)B@Dxc7>Iu@2-A+`Cf zQO}_b#G{}1v2v7X2X8Cubq)75j$lCFAagjoE`_5LX z5dLOc&1r}m1jNeZ24-;l1OPDDl&DMmtNiBesxvb;=C)VXknmOq|I9@h2>^?~CNUHs zNf`3&UA`3t&fnO}dwXUk1{!t@j{O^QYy^Fr1_3AqjvN6w z=Jb0{?hHcuZ}!MGo?J1K#TPu}!N-aeiQm@_VAIHYfBOXM-o zJv-^sQ2mYWG0MtvV&@C|LENZOBBxm#luIamdLYH@09A@7&5}aw2IUf%=irrnY`M9+ z&fnwLYybIT2r@F-Y4}QzJRp%4I7!^P&w4WSBArUb7axYpbmUE?A2|0`*Fev!?|bs- zfF@0pf35mg<@jw=H5u-cUxB%#LP(gDu(C$bUoEDC2#IsKO0)p#N*wWk&=qfWy$GPf z&_Ua-0nI;R-ghL9rj7JeWw)coD_-K(Py=&rI5qKg`UI}AXZ88d&5n#QK4Y|6auFy) z6(a|$+$s>U%=@{TEBXK%;L_yZd=LTpOspv%vAh!Ns9w0U0Lo49%G`bHLLnO^2Z`^k zAv1kj%a)eznijWrFY&gJiQC-Wu8D61(X5S*;fa&HUu(dG!P)d?1Ne5^JpomCE|(}K z^`D9Kf!NMVz~=ZLL(wzRv$X{mi%5K(#AhI0fq0#xG~V$p=^N3byy*`uF|YxTZqgSe zT@YtUQp(QUTpWuKGpil3*XuJQS_W&0b=7#C=jzw=dZ0DLTWDmYA1CQHeyI2)iCuXg zp%$vWS$Vg(U9eq;Ea3QF?Squ@yKB6@nb5w_FQ- zI!#%CjfzVDL^*L?Vo+?h9U|=&KcP0{H|AM9)e~ooI(CPS<_4MHS69M`-Gf&RL}OH} z)A2Cc7nd}54j!`-r=F8Fp3M3)3&G$ChJV`l=>Dj`7M?yJffdXA+SmQUe72NRU%tUd zs|7giLFW8t%Pxs$*hDAzC#~ z*0w6I6%aRX<+UvKj-WY>+DlMrbU7sz*#CnRFtP^e`-(xk;M&%S%MX*b=FJo#R?YJB zi_OlDbMrJ69nTU;EjONz(Dh|={$d0M{^A?zgG~Qu0nrY@ z7;b4?w5ECnU7>W_{uxpeBKyU&0kGLhO|=+NI&;4+R1Rz@ zA!T;V>2e`_fSqYzTNqU%9c``p?K_evcuf^Y0v^6oJO0QGTm=C^zAK=k}`@C z>CYCX4=lI$;Yg^sCyJyYt}cZWr(acKT+cC;qp>L+=bs;f-n%DX_iSkV%25UPWT{=g z9_wMByEE!_X&mFL%t~PRiu~uA^JDP>0@>bN$`gs0VxmlMEcyE9Q*riDc-qNJJZ?S! z?LKrN?T?AVPs<%QMJU$1o|UYM7ysJr5i%6{{|smYond02_;*B`ZCfV_IHM#@aIh=Y z&6aiNJ65~ilbtK3uMZ&3unp4dvc#bKWl_#rjK*GCEl{`=1HvY4W^uRHjhP$?Qu%AG z;g;4+Jk(Avq4G!a9-yLbgB21w$zT9wy@@z@5qTo_C4IOTMb!LS_7Sng$+9__M za~2fV4?8`Iy}SADyF(X@p&XY8Mp06XI8g2Kco^>jBj84yV4K1*Xh}nWc*KLRrDPvp z*j=6n&fnEG^oO5H^KBXJZbW_b*cpI)d3o$-IIYvLG#scox?GpgR@{jF%3a~j@scmN zndaU0dz%X%!unx{1o%1*0{3uC8z57+@ zsi$XN5%onEn{l*Zf9)bRSWT2~Cj%mIO( zg~e?%S>fT~#zf{cp5OJzrBbPy-Oqm?-CN25fq0_%`i{nD^J3)G_zaUL+wnrWgL3)q zlO^csc&!}Vs*k%ri@~j%JcV8sDFZDj(fZVABzQt4j=XV-n@fQFiQ{f_AMD{7p22$N(7E>K#4^h3`L8Rm zscF<1PftErKRH=Q&#P^9_F3-*6V`MHDfw(D?e(%;w5+F{orNs1F~7L6F3d$>_?#@G zxUc<1px|(To+~m%v{X(f> zQdNTf#C4(Ugyx3V4j}dIy$B1a4iR2mw>_?2WqL{fHdgbd|8}7z0c4EH>QRFFTK;}2 z9*X`e!3)b~Ycs*{Do;6q=z&)AaSGcrLq^J$tRGKGnx3`(96GbpL_yCL{_(~3v8Vp7 zb{=1~4kfsMTyH6@(1tx;T^*)-Bcbz5uiC7*RWRfxjs$1QPOD?Alo-EgIEd{#M84;l0a1NdmLI)mt-ck>%}mP$DYSbQSRVC@hx8UZVr z1Eggj)PEM+A_B+`LRJc|9TkLd;~Yl#*dL((QPP$#a#ZsVBOb}}Qi8-~m)ux+T*?Mb zr{%?YVSC8$p#P*#b#2%`-J8PQ)Itu1>9I3C(0H!9@5UrNb5R(VG&z{vG<-N6J4~3l z*>mJ&YR`-ESd}|Cz)VHyV1i-oI9`VtuF1Uoo}&OHJU67;OV&kPNOQ6@gAE)38k;dVDfJC zaBcoqG*w}x5z@ZZtFy(|iENE3Y#~Xa;1@dubXZwo{GRM~=x!+F@owiCb6Yka7cvo2 zc8A}?94}wJu^65|Y==lI`K2#iq8U#m4vup#AMgy8-?m>DD^=0wpK2@Wt<{4t3j$16 zg-ux+bm+&I_!cGT`u@r{sNPnJt^ip*|1ZUHpguNPIFo=v`Bb-$VJjjU0D`tN((7i zaM@s3lQqsD5!b2L%Pj!I+z356NlvR1*>%XTLx8%8ruB8h`R3}Nld;9Jmuq8aB6n4j z5A!DPUO4s56sV}s#d$^7n2KnVO#~)$;X+6(RtQLMO2hD1oTk&#hVsJBl%%dkc zk>}oGY+?e`IR+dVSuruOICH+R2eQac>hFK%WQVsrYrw5%=uzHCny3%`muk5D_595I z<>#j*qe=xR;lvtRGFR!wj@+a*9kZopS8=|^1T!2WI9F@k|HIZhhUc+{ZKFw}Hnwfs zwr$&X8aK8Zt1+82wi+jmZCi~S?@8BM@ArOt@8fuW1@jy;bI(23bzUdF2|J1v#jeah zh*m)poI{5!Fyypm<6Jh`sFg?bkuXIUY!!fkEWt4fDMg~eY(&u8(FNq2FkZ^ zMl9J?r$jz`=qJsVJyJS1#{# z`}~LF@fnYi9OmKAyc_R}N3CTK{|XM#npowI4%}fi{PhmivGfE32TC67jyv#!LR$JH zka`FeZ^;n`+#}N&AWx|mp%SVFNK{fC@63F{@#K5?Vaoq;nP&oYQLuX7bVhrJ%)R8G z>*R#~CT9kNcw!`pVVw?pb$D6I)Yt8d9uLj^y=5uGIyvYV$Fs|vi@W82<#V3AkoGji zIZ0sP7&3*}8e<8QG9h+I-IodHW@o<3jti9cBSwj_u*U}-44rEWrL}c{__mHxpj<7> zE+^H6GToNl`00q3zT4^D0W^_v4-4C+wZW#KTnj1^mz2tTw%scN5^tB%mxRid1mSCw z=)L{;2N@yF;F3yaUNp;@obx3^J1QxzpJfy8&^_Px(;?7nc8-Yef&juUNlrvWH0Hk; zbsnD%xaUGl`$;QNU)USL-G{v^R{@T#3XH{6gUCr%e#!fG3HQOu>m9@_tqy+%=7?#j zwukap#eFxM3B)R;&Cdt!-?{o%P-dXGBScB=aha1Hk!tYz~&k-tY4-CwygfA{%( z3?&Xd@&8T$np3_eo5$tmaa7uXf9d3HU`eMM8RhrC|NVQdr1x2z`(bRYwd-T!-fiy~SiBX&(HZyF?8-gRT;scj+PCLh2+%|AO5IgGkK_7u`q2X6<+F9OxwV2wDV**pwx&vkDIRQk z(J%N@X_#EDVBpU$PQb;mYAW_=w^fhQl6A~D=32JnM25|@>5;F;ZyIVA2&A=H#5`A; zs7K@|gr6<8a&eor|7cDQHyIp$+w9aEBiP*q&=~*~6`&v)h>MX_6sWQoe@-EGf=x#+ z&GKqe++MU{t!SO59O&mtU3vxCvHWEld!VQ{3|QMjsOG2>1I$osV_`3V7^>Lv!TxOd zOJV$_CVK3hhw`7`^7WKR`zc_U^I>DlL>0Hl7cBiFj!YT>Js?k0@ghp>VP@ciU`=Xy z^T{c&T(0Qsk8moNpTx5TkBekXfiPzM8;}ZczL>}}GANu!(SM&}0qBcjJmg|+nVftX zG9Njk_Un1r(u=?1v+I)%7^-i)#{@R*hW&ZckDS~g0c3)3d>27|<=uOS$)fCZYe0qe zVr|lRU;ZB!Ab+R^zfO-KG!$c z=_7KWm2<9K((8HsMq#rfLveac6z&jzw@_sgK#gz_3}JMt?gl>amP$IH5S};?zRBxJ z^cUPWSK*Pe*3D33ID=3@V&^8M<_1UQm?yb%{Xq$$jQW$f#CFAst{S9n1JC;$m~(pB zTSxYZ;7>w!~c7M16j$(Z6i>`!sOZ* zZsn?AR=?AMMiyP1R?_A{PEn+B;#={k)!S|zD!n@*je`WTx=UZ#b9S-*l$~tUy#frH ztT-hBzovPn>wCV*dcfPo)6*6dGEo9Bt5R0|^YNMc=_B7^W@K&Hk^kv4=`rHJU&@~L z1GxZm!LSsEc{{dnJckl)u;;WrHGkp(5XV?eT3F#CaM_!La#?8LK!FKiL)_bEkyN|u z2eceGY6cL(eXNW8*5yry*9701L##`Hwd! z)zvi~*(Kxh%6s}}`ghH$LP%mqfPxE zPmgKK8j{5o;I>G92YOX|64Zd#)#wfVhl!-|Aah__fCIPsQizo}@R6+>JQ^!HV%65u zWT%YFZ@)$2`R61C@?T;PxD~&Tp5x{88{PZ~>eMvXE*pz=R0^+w&n+5o8CfaO{8RP$ ze^lPjs;v!*uktoxy(AzN@-v~-lR?)u*u)i{T`cs$EgcFWrS}m1Sqm{D@#6j@o8FX+ zw!NVa8y61s7k7MT5CLz%$HMY`n`MI4c0q93iv^Nrb3dnp2@=fZ4?@SF!~`pz5Zs-t z?N_-Je@A<4#Q!nT#E{Y_d7S0`I9jNDzWS}MjwSbbeYw2>;9LF!=>9DMS;)y7W)o04 zv!=N|TbFcS?jBq~dAw551(&~bwtASXrbd45L+D62r(GXxWnQk>>;&l&cSaBc$Net? zs84YL7JDw0a!4#s~eH4(m1et>3dk^%qM%C!FEq-&U z-BrWlXTTHN`PYaR&Um@H@zm>XPxc7&`2N!UEu!Fi+&xVQfDPraELI17@c|GXZ)+D$ z9h$D;sf*Zaq=tecMWPn8uG_UzkxFES4D=3H-7fi*8{sCurv}9%WbE=DZ<6=3Qk3}X zx$*avO(;6unuILl_byDG-;_laoDSWn(&)ZkD}HsfoJiz&#zhqZ2Q%}l!@5IZZcF8%b$OmE$wy@re*5a6etXcJkX2HQ*UDmNk?$6@Fd*!L?160EDIeU$NU| z_viI*o%^=X9tsS&_`_aM2ya)4rUWJ5J~@cyzUDm!{*%1j@uftbU~(9hlnRG@{80!$ zufSs$(*U%PAcND2E{W53@Mw6goz( z1BN9zBtC>k+YT}Zr{kj*3Wn2XXKj#ReG^zmCSZiI2E4q)5q|Rwk3+|P!d2Ka6=k76 zV+bYn91AuhmYOx194+$-HT$0I{Hd$Nlb_2zu}msn|Im7#pdfxM)0yM;1nx ztxjNLRtEuFAD?nCp!DN~sBP-E;^Dym1p;2i229nGlNA6J;d_QsD4B`B>f-AK)I|E@ zz+!5)XmwaM8x792)N~rYe6i@~4%#>dCsHa4zud+THWYf$$-5@Ni3Jgh4k~S8!=Gq*Z5Ar2zbSD(73uV~Q9DjF&v9IM86SCU%j8;aGsJ7hF6Y64m_ew~rgFUSG0rWu+Ya^FN%_|Tsjb1pQh`-AC&3@ul^S701y%A@vH z46w3!WS(OJXbrbtfppuBib?OOz_23>d4u6mt{L~=(}vz5D%2y_e?}^d*nkcP ztn?)QuNZT&36zCc^oWxj5GrhL7)My?L>qv1m|4k#UE_g(CJS08z^IuXgZB8AC@3kX2Ns$uNgG=Djs%MGsz5N{+b0 z8RO<+Ra?Y!9YBgnfK#Oa21Jn^@ri>~!ZnJzvZJT_v_pJlRfMAosp^6$+kiS!>@D?y zmgbLfQa)#xr5Kc*mU)P%=l3*tO~51%t$++Q-c~f@03SjvCcCboMSw={1pdA&(3|a% z_I=)XT-HEgY7S$V(zFY7_%s+G4{TLyCGhh=~O67o66^R>xY1{>+mw>q+$L zd%7amnxN?N4Ae`OkI?jvxChizvvUF=k=a=xAmwlA-TzTA5t`y(okKl_)B z7Vvhg`VG$xD*`MCeKHCX71K|D#Sa#7ol(88;8u<~tK$^J&NdWAf4NeYL~@vE`>=9p z|HOI7Mc*aMbSyhUUnwl1v~VOTrAV}uFzUg}Oc-YVg$S&zfPfeC56NCceoM&IQ`3DA zCeWOp)czGTvJlaJ0DRes##`P^vLpS8T!_@Y;GGHrD~)UFKd@Rr;RF=MThv7ZNhr-$ zZqzO_<$yOG@Rwncvcq+F)Wob4zxjG8$7LC8cCf6#VdaHVJnfRQ0G+JV$tfZ&v4SB^ zrf0O{?SFYKzhLF}KN2F(?}u-fM@({ejd``@p!IO^*b4I` zcK&c>oQL)x`E3u|#$OWlWyk=t{mP>$>Y!Tcmsk$oxv#);Ho{YG6$Qa{RTXblNGcRfh~{9EX6)PDabx9LFI+?0%dqwF;=8Vo;`o1FXg;3yn-@6>nLnxeHs< zhWf1pYJ8)=-tZ9R!-M>4chl!x=|FmqT4~Ql1PWL(DhW=wRPui~Y2Q`l$HH~+$}XAh z*Q-7RkR>;AIS)Ui2M|L^jo|O0X<+k~Epd)^))E0Wx#h>;;aS~N5vx4+`pSpmRYh46 z-Lo6xBg_m+%l*>-IK^etr~{cE?!)7@S_Ax%fUK*xBP5=P{Qg6kfX(^qCj(a{N_BC7 z%0*~uTu<{z_cv>3CE~rQFi{EOllS)No6)4T8WM_(U zLm*t;FL;LVsA)*KA_S^n3`UzzA_d;en@*l<*6;H9d$P~#|eEORT6lm@E=Ow;~KU5=`BEg z(>|T)5gwFapYEjA(j~^ABO{TOnl}j+F70p``vC?@v$^WP^m`0?XWbj3S1TTW;Jxv* zzOkBK*4mk;ZciofN1ne%nOofkn?gY-O-{h*y88Yn`CE3f$+K!*p+gjI9o*;gn|aB8 z>a*BHy#1LmumV*6Cu@=0xy?2tz=IqAZUQV#(4>1@Sy9(V<4zr*>2WrpFb^JA2n`eF zy-Ta3>3>zVtquX0Ugs~-2z6Z!=vfT%Y!65cC40mo+REZ1$Zaj zhbrP0E(XqS5euD+rhf*r>N^W!GXoEX?xA795AEsc#LIQKG$Hfm^uylBfBjAK6OVBY zA-H(@X zmCxMNr2Mrc`5QeK&~!x{hDv5 zTTd9O-S-_y^^Y(Jon(Byy85oi#V^uy(j>t@IGgPT?S$#8EiU_^JT3<=6MPhnUK>iY zKpWO2h-x}*7bY|C`ne<2)f98_cWo0W!}lz0HD)^fgsgH9Un%>tCsq+qCqxeQsY?^< z2S&00rz0~{gWDY1xLav^l3ObK!Kf52TpgB7O$9f@x3q@K0Vkw3%nI6j%TG-;+zb`0 zKK$2M30if8QdZFtJ)CD5rxa)@>QXT|Lwl^+POLgz>nm=`P&tp9PbQTV3hj z@G-v=lP!?q1loQyyJn#^V8cC@{TrP z`1N&s*F^6dOBCf0U6XwhJuDc$2&{ojxM7Alw{JmGwPT85fFTU)u2Yp6*NRj87y?Cm zzjJ|g$rc*M{LQyV<96jUL;%u&!Po=E_1o!oy7QiuJOUvKh;>S~cw z3d*K)YG`Oi7-+wX94Z0x>c^bi`VE7%=bpF)n{dM0jS0G2HG0h3#fev&8_HCo7ZP%W zKv-nSR8qVoSA}UQDGf8FGRZ??$6X1O@X&xMF55Jc>Yuxj&v+pqD5d!|GW_mw3g$AmTF z=7z=~Eu`k4!wEA5UO7XXeV$^w67Bc^85t7B?nO$XvF4|eC0l9tPleV~r9(EHsfn)^ z6QZW3h^6;Cj}qWc^x=eV*SQv}B?8>UoQ8k=z28zbcbUO4J`;u42?p=H9o~4n8rZo= zdG0JZK&g=P4I+L7H)7()68xBBD^Y{9`Qg2lfdqRew#IiJjO}P!#6Hc~l-}pJiE_vsL@2r8(sB+D zIH{;Y-K9oD?}RAj)R1FexpUXA{)>$W#lJ=;4T^~&nmPtn`2$B;6HG;yyrP;|Z;81_ zUyvt$Kk(nr5Fz2`DKCsvq9TdLAeH|-1^(yPk_OBp65=f4T>o&+{{5$qfv~Kr_}E-` z2}cD!_@0J^6y_k}@j2hlUSD$U7dX@mI-MSDM-iHg1b5nrulBRJ-=j_c=a?V25k>iP zN;Qw}`#xMfhFw&I4~f;&1H;x8CoE>;=&xrsXa)o=iA^3LGQ;CE#0FJuLBzlkM%K?J zadN_{cra z=6IV)L$NOLfGOk!HN8MRfluzjj&i^&!ELy;zTu%P;qGLOhh?(O2?y$AqcXEV4V%XUp_IU{ar_legvWtN}R+#KZO=7O;6pGKg^F9eEXkCE81JlfZHU z>8K^~wDQW35z!8AMOr)v$xJvnE;H_B!^K!Oi3L713dOiXVL5@g#Fb1oI*sV9DEUGX zPma}RD+&`Dy74s<5Wb)r@}=V^G%SdP6`J}q#HmH03l|%SI`Dj=JbbCY%KF-JO{z`! zl>8;&rI$sEd*t7}?|~1jEcZz$uTstQZNQd>d^^BIzCkJk3+2C;n8uQcI|qTH;<{<9 zsJNDHpd`LWT^7+{hW}(qmOMw833}`kKYD0`vbV%DQ=6xucp-IE`G|tqp-&Z-SXpn6 zUhMr*z0E?DqrVob9DnvCK}dYCL^=2<{S&@WD|f2M+=?O7leWP$54#Rh3oU>15$VOx z?ZBdN;GUxKf>0o18j4jVO+1nohhSRqoQ{y2(kq{fh76}f)|2pJ(n8yC5+PGNGQOj<2dq-)v`Uqz9d6wH`=x~;V^E- zRSV0Tlg{~dTX67YQx#j2Bp3ACk{SoQ4|ahqb5+j`Wlphi)&bs}oMwqw0ZCq*usbuE z2|^p6d}8kRJx?YSwj?GsH$pkG&9kAD!WB<+rRLsi!`FQI@~VEHas!wMrtY*S z4`}Yvp>+WZU-n(NW_-%W&uY#Usnx8SnVFfJa3$U~ckiDMz)tzF^+fF|A|B5l<|;Cc zC6y|QA0Jazmbh`V4~m8KX=SKX~Bmw9Cr+qIH_M)nRRLTqH0u(xw=XE6~zPn`|9 z;jqpQB#Ng!ZfFhW3(EH!GHUf&rU!~EJS(9R7}p`{VNDO`;w*g?zpZlq#U$M5oMwIB zuS%&S4(vUj<3O|%OErYuTxUV@r2^s@DlC`|rYV_nQjsn(YmscV#X3_Nmd-+(am_?! zGTtl+0nIZ$9>PF}q@2)csu}X8EoQeyb#ryHYX8$9Bclj8gn^Z{SH02OBZ!VV5&Z9c zr|&EbON=6)sY6@>CKx5_N7Mj04oXD}B@PNNxYBRdp%F&aKJJzE>vNceBrvD8DxIQlE*D4z>m#u87im>bK$vD>vt7E9_k^k@1Kj)xCf5}d zOeYmI2Q4c}?N*>Fi)?cqL&ETd+=-KbU+$jDV-X?>rc8zJT^>0#Io_5^T^p|vl|`d<758i|TH1Xvy(Bhq{Sf41j`M430Zg~5ZN@;<6i0<>hu4$- zx3!#_>mufz|DrYh$T)ah9whZ^pdKs=@2?PH^E87a%mCpeBiGQ%+DYMKG77Vb#B$w6 zj9IqhMJoE^Z*<5m9cju^-y^?1#E+)N2c8y4#!_Twdzg`^iBTo0n4zzaxTeTxkk~?G zZ#e%q*uc_cwW@X%7InvB246l0I=*qi72jntD0E8Vx{;8c}d2WzTR}(#Y7D zwpp06qHwZpcu2jX^}yChYWQ@G9ai zM1;#6Un0l-O#XBZxN-Zsex(8j4~vJ&2U7<{TN_s^8C4%kJceejoMmk&H`VrG%$!i1 z1i`M*_DEs^o*l_P`dtfDm-*p@P;{g^rUa~P7?&G3b6dHh8=6Xr-VY&>=@2Lvg5P6< zBdW(or$cBK2IsImw(_?{=?ijR_0p?E{^x}r|0?l2|CXL!L^{qEssxY@h_`UIkS}ct zYIrjD$2#Y&C+*ef4~sFQxt6naN6||cBrwz0eCY2pJ_fUlEb4B?XPGn4ohiaWFkJv^ zrEU5_%@uh|Q>mPKOz~SjMzUhc_7uWIpPyb04<#-c&G*W~EJjY*nF#yTn=f_Gel+k_ z+Jy7tfa(e)V?K*FdZo&byU7&YnjTVE$zoOz#;g7)7jdUqy&1dRv%**jk zS{!gg-OxcXUq1!xfHJ_QOKATD-X>MNhyvj`g)jsv5Fl#uZ%eE zer2-RiBV#19>wsc717ZmpJ(xr7SWtdFS;FNxn(|nWx)ND21&nwwn+=%5C2_(8&;r3 zbr>%@5rzF05Crr(j0_C=eS&?wL$q+c%dWjst|8GE7FOugEt1uQoaueJHd{ZKFvxz- zi{qLGaNF6cs(HZwhXvH9K`SpcH2OZtVOQN&e%eqSU06zKfX0jBDC>PvwVV$Bp7QY= zlEHyLAwlTddK~CTLZ|6rIFtk*xwm8cgyMYyeHA@0G-R2YUv{s)yo*~X&+MF9wRuJQ z=ShgC-vo+*y;hzeZBh1qWkbt_ni_!uzMHa)0}QEbGQ0>**|mCK$j>NnpZ`~_-3L7& zqup{Es`ZEPfx5qRx)KbN1M7$++~>4|bL6^=@#@iDuD4^ry!oOJW&&t9w)dHRRxulM z=lzv5y$?yHffwKnMj$$AW5XK5gDVc-Fp;*X#3;E;(wh9+Wy$X?;Y zf06eIr%Thss*%-naqo?0oBDDsE}6?@Ed6D}B`OS-VE*^g(h5ZC#nBCLnsv;owAfg$ zLx+ch{c&%V+b<$x7*3B!Lh*jxlEb2wIL+5H)n?hb6mu0UxjwIi!FEHhJGyUe{pocm z_si=h58i_d!(t0@VgnuUADQEw-i!FZ@mGCEK%X>x-}?jaBfgLqc0M6g`&A|Zz1r%k zKlGjnxRalxKu5Am&aeCEh+H{thll;Hyx3@9=oT7hyf!)l5ho*N|7&F){VY|-IO)1s zfr3uf)5>TTwYOxfiS=#7i|_VF=+o!nag|SbB-C-C)H8W7_4W0Die(%g&^G^FSa`<3 zU46;Xkwx9j@DO-!zY#>G$=BRhVXmbxUg;kuFc*N1(xT*kHNIE5k)r>xr6*fwp_+XL zu9r`o_pry8(?!I6K-Zs`py~L820LSom()aJbieXx(;|Rj(2fsmO=xSrIsE%%9i{SJ zCnA{*Kh_jm5sr{oPNIlITPt_9Y823My)SXeu;2Q)Y^HyHWIibXfb$FDv}aK@9R*&? zvBMxRQ0+^1aylZz^Bm9rm`})oS~y6G*YHehd*W42c*21H+&dyG&zw{5cv2j^`gJD2K3ec}bR|^+ z>LDt95o@eLOro*+MJ3P8H?ahTo%|OZ9tCl=j%#wsy2A3GPnm^lJmgNj%br4sNcEv9 z`!WOs>!wJ+Godzl|wHWAQ7UCH}D6}UbmDeH&@?Fo@!ODkj#jdJOf5p3bL5?{tJRtU3j zoPM^GDz&)oZ*R|PmuVvSIzVbKWfV#0PjMY%MwDpucyc_$<=MZi-h`w9BcDat>=)+q z`GmOauI~sLXvdTOdO1GkaKzbUdk#Ya>tU8TA zx$e=k@CGvek`jM}eH4+I3S_W|WG zNrJc16Fo2vKEZL+f*>Q!C-Kk&dgt(d-A~Yve7!9va#7& z7?Sh5ho2u9BV{_2spWRTJ7hG!^6(I*H=f!~tmGT$ht{(}|F6CFpp|*stl1u8AejH* zP|bFz6QRf&UA3_2(oSjb94zs9+na`7;EhPba5~Z>@h2I`^3S+Cx8I1VI5*dY#9i^4 zoI~ML>p0}lZA1*YTnUx~3tif+cW&CWWZv0|WkDGk!HVfkw3VJBCjg5Xev)2|0 zpZK9zDnNxENeX;S$FrEA#Y_!XA5Alo5XYcJcn{RcJ3+5UN)r*eM2d$YAllY`iOgf? z%;wYVWv$7&VY`=D+aj`bpeN_xLhx!uUia=@j7QaU{|q(Qv}4VXj>U&LpdOH5NoHY- zCP53FGA|Disxonw(gKRLQ<}4;1y`YS{oRnUpY!VJmQoHl;XfF` ze@G9l@VNXn#8c-f9XsF5#ZxT&)iVlB+Y$>VJJZK*u@z(5FQn`K zt+FKFM0VeHTVG0&7i=;>SMl$z(Oae`QKFZR9$4M>Q@4>Qr^+CYe`7nALap zq3JZ@9OU+4n%9vC%*=4sb>D#3$xbhU~JX`MaVtt6~l~a7D2FI zXFMF_DsahR5b!8$FJn>qVN8h}fU8za8&@-51j{GRhR*9!jH8sN&wcabw%wnd2&VkgzvNDTi zs@n~7FO%*~25_#D+!3=*QLXcO5K>Rq_U6{^yYrvwc#6$cj5_&YEdnryHj-&%a_<0mAX5P;-JIl49e+C& z)Cm@OfEEB;1hnAbw1@lIF

oS`%tQ%NTMT4w$;w{N45=v#8WTPt}=8MySXLO9IL3g^M`hjT^R_^-Q_T*?Z(ZV65WVB zO*ozSn;`m@9JQAndD-cs?R<5#$)O!t19MATIYz!T7dJ8y#O4024QGV5p^**udOL+R zby-bXqp4;mldqD;3@^`ke7$zDUi2B}wWe!OH^<@^%+x;KDT*g0 z-d&6^sOp_=zXKXmv%&l7Ry##e;O6jeC(P!vMtQhGJO0a(#h+)8`dE^@svP`EIK08S zeM_!}6z4TkP+dQ1rIpeERIpCAtdJo%jP#2y8tM+X_i_+BwL_^>(|#tR^h9>nr%i+5 z7z^*bJM~Qr^+m#)l##Wafg{6@pKDG}UlQ^~J5S-z{b7p!z3~1x8*ao#6#yk`7L==k z58<;ud~ZphPa#9&b2f>N#YB-m!sUG!F;OkPOOE`~9mJ-4c_q8~RvL_eQF;e32$5oQ zZXTqVqi0$SS*3xS6Fk|#i6d{ROT=oaxgrnAKX+E0*+Oz2WDVD99yj;4LZvE{9R|^b z?PL43*?`=REa8?jyk?Usk-oq5ErlFl#GhZboI>k#c!8~)cJ?oX!&jMB#H+7&zB!7R zuV=KB-?~#jR3N%QzxZl&KWI`$D&+#H8(w~t%cg~f4307NFcDL)yrggKxz@1#>be_c zV3--G-DVCiN1o!Y>h9%2CL7LDh7W1CqB#U$?IP(-$?c3Q&tc9&&O4Bv&ELR8Mysvo zbgxR}Ze?UY&0gNjcSenZ=Y6mg~>*58XH0`tcG|z z#io$hFvH0S~?r)wnat7=SFX)DZ3ck-=1bxp28oOacFHxGMq!T`gC7cRGF#s7z;X&iumGd2mVE?X@8bLAGI$UAXMQL} z;Z4GceEJ=$?@rN(swNU?{OF*C_m6C-ukWbiX|^=Ti#Zi=rk~olFTh=;`vX;Mh-P<) zt?}EoVbr4*W&D&?ooQpuO)0+Zc&d0BB?@BwbqBx!HoP^SS^F)0%Y6yJn%yn}+9@2T z-sao8S9d#Bu8@oyxOc+#u0kx{!-G-@bVgeqrF}WnCMIhr;S1F^+B!{rP164eygn8Z zF0^&LzolW9>r`W6^Ak8l#5 z8HK;&LM5_&d){{S-hp^RGIj0#ol@BGb7S%goFg%N_)b^7ScD3;-lW_ zbem?WP?d634E9X(yNGCu*~)pzgeE$%tAjRl8am;m`7;jbm^b1WkfSG+)NU0{o5tCa z15J^-r0)BZnF%vOr0@q9#8+p|BStWMB*Dbi#q0MBU(wDhcP9j39ETJItGpb3Z)B#t z7_N@G2;Urq9u-CZF7B^5iDU!L98mvJHj0&qE@Jj$k`;t+OUc{#VPP}JOGqw&XTLTGhBH*AhGSUjgSGnQd}~0 zz;W&RZ3&p@kPN%=TQf-3Pr)Py4DeF|ush{I4Hq!2FRV z*u>pMNj0p2MHr9v9kvq(QlaBhz>TvXHZ+DM!OD_jxNFfD>=_nYh9 zwWU1HpN9&!DcW8o6ppXS@Jrw4-&YDBp*fK^Z2wFvq?X?gN?(j$yo^C^56lBRRFGM4 z&?kyd;(vK0?qFDLP4pul5GL#jU96G_pjacwu{`R^E}k?HRKmcd_IUI z1sNcewBKAYj}_nT&+!BdJleBW${4RR7npSaqKgXq*zmLq2D7BbP6|*440L|AsY+DG zdb}n(h{6A9xS}Xd3eWGx4t0J;fygu~zhPMbUU|b+JM~lU^{*iR^(wYk#&d1*I>kPgybT3L>d%pgog|DByN@9)^0Jzt zg|Tj``0Z#EXzzs{iedCKpXb{_`6^A=?pY0*_mB4vy`XZvTQK8D?qS2kX!gL4 zS$On$zY|;Fd<#^XUKs%Jx=$yhnqzq~a&Pf)ONr01fl6$9OU{)HCteaE>Km>SLj+ zy2`l=ja4f;FzwX~8eoVJiJ1C#kV{9_DR}4#*2+%OXw=`>%!eq!clbE&Q`l|~C}v7k z*cVe;^)(A4BF-NZZA+r7qB^VF(ibKXGuP58Ag*`v+H0u9Z(GY6>q-RSRuQ}mi!fQf zFuCQ@+DCWn)CEEJBmynWGFu(4IHI5A%_Lputs)pt;J)7&PbU_lF$7&y%#N<);@IVU?+4>HBvC z{2`cl??ugxfXkQ0@A2T2u zq&udlP`tw0`)OI?P=9%8d764QSdGC-UuZGTx73|uVY=rY$II#P&_a08Nk3hK$%scP zf#$}fKi;R1ie%anl?*=>F#d%qKaYdrsS`AX!?^il{&+laaVkdvwa}+lDP6;sm?{13 zY;SZ0$_1-Y6|{WI45eWw)lc~TYRb*s@||^^{p^8K9c~x$_PuLl!tY<-`i+QX;+#=0 zlI*@~w|z2z#u{%}i?9o4_Af@gUR^tRMIH63ul<1tYi`b`kte&T2y00gln))Q`O-tD zd~-lYRJI0fPjao%ATxdrlH#lTqDc<6xv&{Y{`n;Ip~=Yd6zXno*sXPZhro-L_QO0| zXvDx~GxJiYIL``In0PIapyr16sWwB-1WW0RMMK?c|Y5sDeJ zmKtT@opc6ksbe$cgfU=fC-+@U`sEqF9Zx*Jh8$UjPN08GWEYa9hZ-V;jlUF;LUyKH=o+K?&jQ#zqUoP$q7kYSN+*u3?1oh8)Ml zwOHb_z2{=q7y@OzC0dt$(Z9qM^wVDNW9rHCAD#)^QrO18ucK8llw)3x>e*+MzJvn&!o* zi;U@LUf1tipNouGUC_jNZx?glRboFJYstT2bE4jQAvx06dVSkdbAR!&oo}%8v_@S^ zUB!i{O|!85&CAFiWzyaTYD4ZF(Q|FBiH=({_b`2`D}H4SsGUxs``nF|^eX>zk!jmL za3HP|5H(68BN8TLG)rH0L2Ku=R!N5DuvEWcvG10}NMAX%On1b8V|m-mvrq?kAp*Rq z;cnBtzE;@#)j`rf*=W|DQZ`!IP(;c)HTSOrbyF4L`8?tq%zvnvw9zI`L1uQOY< zr>|0*GkcKFJHM$nE#X;IWtmeGeW-OAI>BJoI9q9&rej#BROmOx9%fVjl98K(D}0T< z_ywKp^BC^=c=n{F7K7U^iSv!k_*|sXjOB~~i;)O(hKSsH?_v``OX67`7Vi6vEYhzN zLmKbG&H-NZHAgXVw>OT1qM_bBu2E=by$gsDsERdv=@$21UFD4B7X@>56G2U1etzdr z^Px*8TBE7Cs!oeVy@@66&hum5b%20so&>Xg8h%5O+#6G?iutavX!-$|Tww=v<3#e+ zcWPFVJ|65~d@@C9Ng8bnVcmez-s=TGC)xaluX1CH@Nutz0k0;G2K5{K8eTXUC z-B8#?40;L^UT4PK_4mA0xqT7YIk~K;2NsclSlX>$ykDZLZU?rBM&Uk^K8`8wj!3-A zSQ_S$QyXAM@|EKF;?KhA+#`yyy+NKhqj>h9mwn!Lb=_g$WNlhVYAcWpwE5z5gv@q6 zxO&^*CT_)?i%Wa!OO#!;oQHO?fj<%_A}Ajhik)&XxCjn*)T9HzB^^i$9EdS8QIAyL z3Z@Z;*_t0cMC6~m$wQ5{*bw7yd^92l8>+5V-B?qHcEhN$&+p!IRm?7OoNig|S?qP% zF89~ofb-(3q3&J|{hn?Cj@^Y_EUyI&>el)zd?^{{F@-;&57B(A z_)Ts^-TVP`wgfUHjyaH*na5T=k==W%ucN$^Frhej#z*eW*x6Dy$=jH7{n!~oXRh#z zUH!wG4x+Cja$=3)9C`^lwJr64Qj0knToH0iBuRJh4JYVP&9ZmNnwJ9ygXV`ythbaB zTFQVvUuH_es?ZJbt0+ySXL&QW#a~s_H0jg*STb?zQ+JRqy?{-mvyjkJC-ith6UWW@>%W1c+Yez4_s#d;jO0UVz3vHY6KA!MKIm@TKBOwz;6fT#X&`d)xV0kcEFtjEI z-i&j)Vi8pem#NXQTi2$*K2J{ph@Y=X4-z0xoBVLK$R!#;afa2{Tzb zkZFelzY70E+;zF05mSEm`<*}%Lx1~zJ|0Twb|M~`4o+9tt}t#4u)947z$mbx&=e8k z#C<|`QI7h!Vk`Wy7BlxuT2_YYgn!q!3m1o^pK;i>7AuH?WmzC-CA^%E6z4o8a{Mab~p z;oZE=fEZa%hiPxbyUFv9C)oLD-Nl{Ag1;j=Pbt$yitXmLoV`<6--oT!kn@iQ8mQQJrx9f+8FBg|7s}YGhuOL`X%k{h$MUGqD0|pGV@BJ zhO3|cP%h90gS-2ngS*UY@}2M8bMAfbzWdgC zf52i5J>AvSUAy-FNtKsMX=4j(IOSKFmEl=RCj{vVIv3Vpt$cxx;SD_lkBx9ShLuxf zgrzwj$3wgmS5wS44pQ86pvJvNv6fzE%z48#n7=R)n(OP3*Ff!k$D8tGLyW(*WueP^ zBgvMGn4cCJedcvpD5X{cMZecO?PsTWt+`;>h7lk>mZ*zPN|YK5K+-2j&g9Wv?2HI| zIgd2Hp?02GWb_2pOH#g8%g8Wf=xw*VN@`$f*bfT(5u++SgXf)_FKxNZUrdbJ)nHNc z5QSPhDj^Mu1i5{6Jbo7-l`t4KgTA44G#2Ek`#imOSh@~Ee#QTEldZGzPC`q!`Rs>D z-8Y={Cbpl`O7z^5Gw2<68;=8;vId)6yNj10EfC$VJ>=ZmD;x1f*Sre}G#d$XwcnRC zFVV5fme017SDWKjlwidlI&eHweEcYB9Gf|1(B#;$GdDk~_HG879DNB7 zRXV1uJQYN?exK(zZ>K20I`mA|63?GKIq_mreY1%pndiW&+Z~P{<~~lO%D3{K3d9p< zW8hBayBf4H>Cf{?nm`@I96q;cq$aRAl zH=w??k-A4+Vw;TjI0nn4?7RnfX=Ic`W;;hht0gL#Z%b0G5!@*%Q|6`;o^MLq*gWa; zwsM<}iiAShIr-%!&o+8NyIW`9;q%D0X6zGjilpmaTwUshRV)r;3WzaP z-A_o^XPIn~Jl)sc(HNY?|2tUry9jFGs^c0 zGxL0_li>Yy2^68F(S4=;!&;084g)=Ms70aBj9YDmU`~k3kw2aY+Q`;=8jrZ9BP1`; zi~)8Igzr_@cm`57ybeZ8Iyz_qoHc9Fqk4r~9*_P>K@j6axXtNNJfxIcr)mKnz?323@=gDGHf0oven#CpWCh78?>BQJj zr!3R7a@Y5LGJafG$(-L2h9MbiOlaUUJ$}@Lfdoe3uL?z*5tEp{F1RKthVGA0pHOT` z+H^^KPNBSqOd*gPbh|trd=|H_f-$_tDlUj$h=p3bI~3jSXTx2e079<M z`y%-+@oLN6ZMa_nf>WPvxsUa(33=HklP4h*Snx>S0TF927v}_{l@+j|+Q6pqDapM< z+1`~+Je75fkQ@48wCzSo91f+T3HKwizr5HgLy_0mn&`x|jjVt_{o(Rcy$b)nZ6_tY z^v-2islkE01#XL+ZN`FhR~i+iD9jZyU~W4wc!^><%1r)Z?a`$@$+zo!DuE+2L&Zmt zFcho=*3BZs25&0l8oH{-o!AIQ{7B;-?8Gs4Kcc%v9__A;t&F|KgDXnK@`wrC9hsqb zf&zJBJiSG`ZCOeAS!nTp{9Z)TPf*_?ETm{*^3FJyPN*6kaeP^l!aHx>Eql#v6n!6> z%f*P08RmS((WSa6UBa9Q?d=S;Trpkedl6sgu1n)@cVxVy{x2F+pHz(LVh?&)8A6*fQwN&cv{#@dH%yh_s7 zafF}1?fFQL61$ZO*m6Pf7rmXcCpZ8LD*WYMjb9$lnMhRDH*l)$PH0;o2A0-)lFQ$x&N zO8V6q@}$O#i zWM)d6mVRu0Z5Tdk#?F;G@Q7TnDNO;YP2N6DM8k4bhEKAUO$I-H#zzM$rtq3K2CFaj zRsuV6IJ(`L5RMsz<;EkXG(<8bN;s~L8AQsHSP~B;x=pLB$ky_bk&)S08y7T*Tx7l;qIw*To+jq2%kI3F7POYxG?9&-Zr_+vFBHOoJ*6<9qr9OLX z0Xz_3K;$n@N)=t|?a^9P;)%>Zjq~2J8xoLcaNMt+0sakp!AEq#$yF2Vurl69q7=!g zpWZhP6e)as#->^g)1_s})V2PC9Tq&Q--4L4UzDwXQ z>{y01A>Uw`ZRNW`=r`xbMLWRSB-WG93SD@~)X8a^W!rAJ>zC&n6MRFO{hpaGT~+jc zF;5@#y^xQg_=R8(El`pe)}6zOvtaxr2HpAI9%~-9(RE+rV@9AyBk*xB-cChBXPG!C zp;au8fUu(N1wI~ zt^|BaY-)N#Hn-3XA?0GF;Ld`Wz7teGrVwxw{5YhEYf|+bn<(<;aGzix%0sBaVXW}y zsn_e*CuP5e`WV!rgom0%;+F%j%KGEs7uwUoqV4~{@_Q#@5nGDKBy--NbuU5 zO2B?^$qv&uD}Sk^Z!ol!FVc~{s&`aU>CgNS#Cxr09(MZTnaDW31?~ubV&bjacK5?i z3b2v!#V9Z!Oml=$#o_doNPghD$v<-k>39!T_Zf)BFPN? zcxR<{*2uWnq|(Dgy8rq8z5v z0ZzS3&D5V$+ntz3HaM+qasOlL{CgHScRhD6A8Dr8dh(ITON#<)r&M4M0Yw|>s3BO$Fr*jrf@~d)E7l)Fn;cn_Qdx|D&lvMc5 zo_rCayuUW#eQ`1!Vpk1{hc~!%2d-)t07rmC{$_LP7_l2xyg_=BUWt>M*fT ztV!BR9z9FlR)_ShTqYlw0UE@m4Iq1N>N?hAv@ef&R+&yi&5 zHD&ilt()4$a73hSAr{{y+;sgjrE~z7!hGM6(Z+wC3425)y$&Qu?D|7<}O=O1Vdq-46f5ywZxz;64;FsA=RB=SzW zy}Mbsp_J+sghcfIN`7|F^gGTSHB58=%Jrydw%8TAvD^Lq#E7oZ*=)A)HOpHcIVu=B zfP9F&Gkm~5oHVq*8MT5o95CL3&w1;Y_x2FDFzL9}t{A#ElqcEK@>-2i?76FtTH8vA z^`r$Ssr~qn{g1uPm|FI^2Nt-pz=-M19@Z`8(c|qL+@7S{>P;+Li1NanP_1`pgPGJM zwo9KTJwfCJ`x9Zv;zeAR^P#i9#Jek8BvrvKHYxkkav%pFgjZMvVY`G1e7; zIAVGUUr&{*wdYp0w>^d`PSJ9^k^AMbgMVVNcY>gF%?=Gpz4rxEcJIy}4arhaJ`EDS z8*_HXE~+z{OVc9pLmhObeyl`h?U-v${<|&5UDYouMRK9jJS_k0P08~qTuY0l9IA3; z@gefLPlG;E{Nr#cgj)02o;XNyb!fZ}dYI*GRM()?QarNwoYR)K)=k80=S;nhM`8;# zN-S=gYFq`f_?c2)U8j*D1A^ULe`9%XA@ zN_>AZzKH6DOJr}v$pTGR>toPCqwMz=Zr%zBW#O{v@R2Xp_}NoN)W@J$aai$%avhWA zYeIB2Ihw(G80{@j=d4Qbt!Yg@R1e>!^o}*I2~N(VxLxN5JuC4>R2)Lc9I<=k9;Y=Lac91O({B_atx(KDDVd9{rp4=VSzuEle%g;?_ z8p&k3!dE_Wh|8-sl+CHWJe3DDwU(+m5T@>@cs2QDt;S_S5Zxfhj=oj-Vy(EI!6dE9 zA_b3`6_HE-UV9#}cgNX+i)Gh~+jo)#Mix}t>7=s1xz37y-=%%LlW*b+JB%E&@P?SzEMTJ(49|O*9);^i^sw2dFKUp*Thyy! z^-Sg@A7!7lJtbZM>WSH6(;C~0&B&vEOHu7=p&@9=5Ww6_w6JB4C z(oOp&h+!!cQlu%n#{f2%Pde}}4@VZ4>?M0BLx5NsbZyWF9nXA+qw<`iA5EtRK?|dn zSsSKiml$8}Ep^q*f2~15@bqeLQ9)U7B5D1zH(V=L%Xn33iJIG zfd^t8NhSuxoV)VW`cM@%O<@;ZC{P`YCBctKu^19xCNXYp(FofWs| z^)}5$%?Fn=O5(OtlBz8s?F>h`5o2I@`0X=z6*&sap|y5PCBNxN)jOD;XOT}yL*K!K&kP$x$(ce zvAg>j<(uv~VmRd;g_+pRlj=eyT8DPaGZ@#@$NwcH0$FHJ7&b}ClY%Nc5MJAa#+fp~ zpcGu&cIw1pJDUAi2|y9^=0yE@3FYUHQB;yItOL%m++7|~Q{d3N<4+mRh$Fk%Kr@*A zJcq5TzgNXi!IR*lp28XiA=`uwx8Gq+EPbQSa? z;-x^>^w9`ES?QkEC-Ad&Kk6V7a>!Z~5*j*2#`kURZrgD)kEM|>$+@*kwN$!4sAC!sQxT48+)*nU?xyE^M+AMi{Ie~$r)hGwP&({$X$9i#zs5M|T zHgP6Oa+f*;LYVz>M8bgU&P@Wf!*Ki;`x z4cU$DZCHm|f#r?&V7%rKJSw4^0w7j#N3uobBknv?=;PX})`jkAaW4)eGcK^%a8k4C zvu+PHk!$T^%u>=HlQDRnDBvvHZ`z2Hl zkS7G4QWUX3h;sRVff+#7I8!C9$jM=0?Mg-tGIB>d#z@AMIXW0f zkRW1_JPWare91RBk1#u;r;Wt4c(nGnMJP)8EB=knWS_hdQ=7 z|J*U*5xOH&X}L5Y%J7k~dEPKeUwWVzkNdv=o?l=f6iLfNxa)O|<_IDi=fO&tK_Z*U z1L%MbtFp_3vs1g=Mj;}j*1myH`;1oBeQ6SrKhLIXXm=aQbYLafo=F%VVSHQWR9(FS zR*w_Cq$FN(NPsiIQ|XN(bNDav(Zfhbi1Ml$xE&F#_lJ7o#Yc)Z(7#4%R%R~h-Yt-e z8JE4_CzrMZc06X>c{Y1?z%&TE7y&_FRLcB{c6e3u>TbO^I)%UB)>%4@`xB6hZ3(eF z^2Vo$dGc)T>S|=LOaL_^@_S%ZD&!o4FK=vK_ElCcjs`s-P@STVNZP9TznmwLn$Jl@OiiEPg)vs!I$R5}lS zf%oh7o`3!ks*w2f5E;JWL;X#kuXAdbc_2|F3Xw7rMPw}{Z`ZRYzTH$6X z&4)R1L)b{dH$b2}S`-EE>3VfD#RxDTn}cwb@`eLSm16NkkjHn@8WUc%QGP;ED>55Nmf07QA@wHxsOQvy)2UUzSw zLzbEtfoGZ~w^Q(H)S$G!L9Pi=u?Qka+=&0KHLr#pL{v;^&)*rbA^Axi9HT%3;pTo< zm&AqL8tT)2vzFKp;ZkM90!@{y~T_7_`2bUKtVHup_IF?lgwD~)=Rxh?d$^rF;N08q$16SQ# z`rvb;J-bZPNT#A&uK~kBp~)W`bR{iLu^-T62q`A9*)xb|!qG_=YS-E0OEKOA6}W~BJdKWB0-IqljWRjzpvR7RT_FJ{VVTiH&cEY z|9x2{(Sd7LyO%L7ERl3t|HL^NfWr8<#=5_!$G>KQ9xVM>nNH11+bN^)Qlfy1dSL~| zzdZ~2PN(m3ZlvFPgvz2nfRC&(CpmgcGH|EQyg4X?SER{TSnBl6OM^*tDpleJWM_QB z3&HE*CF5_hW^w_=jECFGH!n|xQk;^UBMa6K?`|VrYAyC36OisQ`cONUw)oujKfGY( zgvI@BRq(3Msl5TNA0~FW7@P`GwjZo}_KKW%l3BzKZ+nvu# z<#TJ1(jSR$UwIt})a^}QMTfSS!$`LpaIKZ+fA?_h1G4l~lDd{v2c zy!fIQnfxXyE#PY>hZeuLyHL(soI-iXy^;rZLFqjvj$hhFV#Jku3yUyFh&v-2YTbV) zY(mayv#n`MPG&P4Go(;f)(juj&_`5pi_wz-hFQ(d$zz2BFL^Sul=>DWn#!+FRjS{8 z|Du5)BcMlOlZkS6f>EZE^#*5urbnhoaQqF|M{Hge*V8?Nf3QG7u19W<%nf zRAY-!7X(7>G3<_S=;`f!b8_RMzYT|27&Xp&*yg)R`?!s^vIfe46noE_Z6ByDcJhmK73W(`NS@}BGS{fvNnYEpQq9&V0L3wOG2ig^g+{UpUC zdf+;w?d$GeFw9RfvSB5UrG zytsC+JPT9jBQvV!9aYZWM-1Ho{ zqr3NaJd-JpR+;-tmk#Fm5;N0dmbjN(cM>?M3Xwt_q^UCAuD8mjzsXx=en-OLUY=RRH9{iD^N<|f;U`ynXTrz-20@CDU%PBn|Xnk(gmOQ+ntAjY5sNQ zUdai&Aom`V`<)#^6Q%{Q^g z@1=S_E@f*B_c_M30OB50gepY1L>C(mQ`(fKM|i=IeWyx;GdVYxlfSQ>avf?Cq7l(9 z2`^-$;k#lR)r%NG52Tp9tlet(BC#q$8dAdbj@b@9!O-fL4%Nw$i0tH;M0V?-qoq7nyI$MOoX_y*beAxCbNc4-Jc>a z=pQRmncH?h$|C&VklNa!Gp6ED^54;GYc7mTX$57?UDS(;b0xsQ7+zs-6uQr=rA3qp zFDCD6W>h`qNgm|L>CcEdH8r7|E2JgeZS+E+IK23sViz(7WItR&o!gkpQ=n$I_J(1z zk205xon3(@eQcV2Om}NZUkB+k=YTnp30rR$k)La&?6bDZDZ;I4et)ff2O^DcmgLY< zk2G(HRNqWdf;SU@(YssXdhkMy$U z4RsLwm{BY5*&2`bEBw3;_B$&$#14sfW)Z)rba$z<8gWz{@Y{fRziujFDj&|2vY8m3 z)yYdcwBRS#9t^gA%HEz}OD8X?i*dU|3L};*%rLA+pQdloo!eQOC{Vd;+V98s>Plq2<;L$A@7gq=i4nS-KiR$_9-oe3$0p24fsmA|XUNF;Ws z-}y0n1hING<}9CN<<$!Isg9>c!*(y1!bym;*ZCFy-)gT#tZXeAZX7#Fr6A3b-)vSMBWQw>;Uz@a8+Q%_`rig zZp3vL{n%ZK?P-(HXLrYCXv>mU6FMo=vgX4E+S4(mxPe76p&4EVSHqn9goBCUI!T z_X)M8J-Sl11TXHj{gE%dIK*9fg4-_5my@DCeW+ALdwh(tB~CWQEkSe}mD`?ysHzRg zFI)p(mBR@+I;v4OW|5|e@7r+E%JX>yzoN2h2R=HTtl}>k2nY%C-|_CkNIO~wrPnE>vNHy6(joT)?)J?BpGl>VV0mhMNKz94 zp%LVb!-?T!GD5AHr${MG)~&-`qxFf9`)q?LE-~cQb{Do$bIdPD*mgI3DB* z#Zyk1N<_>(0$rYJ&mGxhewsP{7E&g!h13Y)UaNXpf!WsqVd2$}P&Nh?!{xJ-=ri#%|E0<4Nm7|555^3+?ld?uutvXPpM?A>;x&J(6t&b%Nc1OZ8&b$0kyFgfX{xyx-W z!@o-R)s!6k@xn*`jj)d$LX%CzDN!39%;ur{gLEcI^c(s+*ysUEa*o=pi-fi)$u&j9k`bTBaSM24N(NNnY{Uzd;)M{m{G$Z_EFx)Pw- zG9;L!+i-D5w>6( zX&<{2#7A#8^i>9BtZ?l+4efD$ob(gD^31!hZSQ9Dz}o@+T6eF~LFuo3#H;nYJIpXc z#>9$DPaNY@9B2tBFMu@WvkfcvR#_BiO+Kzf$#t3dcQ>2P4Q@GYUEm=@~JH`c??Y*;qZ8J1$Tu0V8GuozoP&P4RK@{@E-Wh=7-VzpXeV|A!_ zz-k_#{1y#t50^K;<_|8R89&H1L!;It`%Hw&-pTdei2wIgy5}I3WJVh=B{@!K;MMh- z%+=kaOMZdaX_&O)K~djqYl@d8;_ith)9){=H!?}+#{+ULq&ZhOn=c36b!L8gf;v?M zm5CX-KZ`fnT>Esb1+~uZqoSwZoQ4TnF4hMhwLY3 z;;D2)R{zl@Uhkp9f(}QnsOdygHxjT|ts- z;OBYIJdgfEqIalb;{YIFs`0igc#WghdFdTs<&=Ws&$?lm|v zz67d#qu!HXIw*aMqv~K8ledpv`s;b55UHu*ZF1K1rCw<8QY^ZN5*W@|>cwR|1>$P& zZt6{g&|h9PMApnqRMuI24usut16w|?bv_rsQO+rCLeZQ0h5n#z0~{)`4P0JfT3x}S zEtSXZ+!l@yWd5${EkWn6Vm8saKdM51<&%29cb)+ zgTvhv_}&yHl_ISB>&I*R_{zJ_eNjJo%^h|{6Yb1hXxXU^P|^?b4dD}ReDJj#gRJwD z(ppc=4UWAfPreqbaQG>1D>q-wQ!5HQhB*YB={`&18jxM)$jV>wAXpL{?JC zPJTFHgI<}a-GiE&+1Q)}FP;NL^y=c*sn)H3XZ>HXy!Fp#{=-sBXpcMcDlwCl0>u^A z{Yc5$?X>=zS{b9Zo5}F(1(%(PC)Lx6wMSXDykxW7I)ce%SXIzvU!Ply2+=@MQlcI{ z8JqBf269$sUfGoFW(lcnEba!bPUV&BvvN2u(x9cR^&~i$$N!0!8*y`T_iy6mmgMgqcy?NL5Rb|T?cjyohn-;n{t2`fyH%1?*%eS zeMW{ey*d_(1hkkv6@X`H>7ZTx1n0|(e+V? z^hqZ*B2&W2U<~4qcoZMB9II+eh2;wA=1f7rV)sLb_m}CjU!`OT&XSa* zW=uI1pAzKg--VRrg@C8rs;3N!sC@gGoNp9V4$6mNEe>&vyP*LYrAsF}lVwbRf&a5_ zi0s96omAl^`E*v{UD?ia-O$zBxpL2E(E8Gc?!zEG?i2uQKmQH>J`}-(ZKvTa?AWFl^giBNK`$b=R!L zY65gY7w^c)RN7Y~rzXrP4#eA|eD~8Ujjxz2#IebAy5ku4@~^ZV>eAX>9Nob|R-RG2 znRp7k&CzFSj+0A`r*&$gOI}nkkq4>AA3l0sG^FRs0BN`7~jkB{t>|4+3N{Sa* z;)A(SNtG!ZJD|JUu?_QuwA=wJRooCyrFS#?HByoj}s(pI_)#!VZ`GI zt9&?qX91uyisxu>b^qnN4ef0ULOm0XqebjclJ{#AKD2n_<5-KV*E zQ6-F{I|XPED7h?>@Xm_%(!YtV;sq%r^OxE5y8&3+x zokZ7zo0>UE5a0pbh-YK-7RMWC+<4?@g&(UgUQ zY`Y<0{q&DCyB`Yycc4_#^x9}w{H)&gobK+9=s4D0YiO54<3 zB*EEV%z%`VOlb#%kwAhZvWbZa#nkdb$$ENn)-lXuDJcm3(?f&x0;^aNA7mhbqlyO- zm+T|_g2o7KJ}8x6BdvQYeSl?!uBOG_h=J|);;D%xYq^qIchz`)OaN@1jCM!+ivPJq z;GzM7%)JgR(v72Nd_&+sqvOOnJ)gF-H8N{W8|rXG2+_)|`3S_u9g-#vO^`7bqGN)$2|em`#A9 zN8WyeEw`v{(``J%EezwI@AbowU6n@H?n$6Q1J}lPI2p2BM889O-qvF+`u*zUK02aw z4xj%bp)9Gay{#=&Qy6l&(GRd+Aku6#N^vo@&8A}48(vgYGP0y>`FO=zXpkSygQtu6 zwap#fKmdX-|M}6CQ1rBO+xNpv(W7Ec%0n;T=@kvuyf?K1vM`_Qh7g_mh*~9LDOvFvgsc@Bk1~I;FI#e1xOPs*d^jdQbxJJ z`$-4R@#_mH^K??qtJq@re`X@rcge_Dko|`lX})(-zOLB$6~PoKC`rko71R**N3j1N z(GE?{4pCrhwDQL|1aM!WoNQy83w#S|D0@!w;DDOEl90b#0T6SI8|~o7kMwgq+YSrTS$6c6<<~(Bu2+|?B%cBYd1h&?5gtoR3RVNpE1WU#tqBU2`OF*&QWzT8R z)Ai*Eh_v>Hsr_%EVOPs3b4PbjXiADXTg#FJVFY)>TOL6HUX#fs@xl@ErNN!T2FFTx z2xhA2BLwTM1@v_K#JvS2@LAt8oq^um7kOyx+RZ6a!V$-8gh<3&6YWGAlT3NMM0rLeDnD3WITV+X{+|%ma|Db!SRHqUKap|@R*2$n3sfh&Itse~E z6F9K~QoDb9y|Ok{`Si9)O{1sXJz#w-G zHn&f_d^nqH^Q^=|ja#H!A7E`3u?K454;HJ(S3EhNHq8S8``3PdI>w2FZHnXTrf1)B!=F1%{yh}P%|Go&P4CEFWJ=cielqIsYngm&d&D`mqX+qEW z&(!lLuH+}V(m%>Ydi?EB`l)t8_x5|e*+=bfe>d~5T!6!A87=V*&8-`)Xj>L$hSfnS z_gsO_P!nno3uu_J`khdI6aHqgeRb+b`frePJIv6wWl<#|@{D$96wJUiYjWZsP)5fQ z`b}Ac?IBRive+My+a#e9?c6=&?1|i9l3L2$aE#WaJvxC~IkU{!6B0 z_V0?C-ZMgby#0{gbKXR5W}oZ}Gdmb-iHf}tOrfEMD1%*Ljz8l(q(_t53F=5Yf~<)+ zr~&^F;OAZT1lO<+S9jgeb}VEA3i8M_4qJdO{cC2|gEHBj7XSOq{ud$DUDI0^??TkC zwD+t2#5lLRscT#*Jx8+5F|Mq?lm%W>KVlDRj5g5P8?h|VoiJ~4UgPy-w9Xt#aLyeH zGSct=g7duXATt>2{9wqA-H_G8-YLZ7>%bE^<;}b1a=??pHpSWYu8U6&WZwAQs@`nunR z3>>L=4!82!=zZ~;=At3T-D5P7Dta~XInmI%H4@2^m+JW)U;%uzcjLQZ^5GKu8eeq$ zy(R2|e%gLeLj^wJoGVnc2ft=0I_ngmV!q)983BS3(pUe*d{US$S6-xRgetXfMl^Tk zVE4+!meP@|*GTnP*rFCMD=P6qT}gzOm;LyFkH7B9-?iFYT$zZg zmSUjYy=fEaB$9kQ&FK2-a7t!uCQ)Z>nOY=ByzpP8-)tv7)0XcuP~9Rn?B13+0+Kzx z|2N~tb`p>M%vS0ELbsn=M3AA zhK+|d-r&GpGw|v|kuz>8q7fG@x7mbz^_v}VbNyOCq=%oQhUmv6_JG@y8q(6M>oYUi z$o;k0qvNx+-&a)xnvH<$rzZ66zlTJT`~jl3=&FVPXvPF!@|aN`5|;CR8TRZknT2BS zb(_qg`W~{b8|cTC-Dh&&tZ(-ew9sGD!VqvNbBBi5e~wakb#pU%WIQ(hB}GJ4=4Q>D zN@zBi^Hn}0q^4F{4R0I)qnQ}>IPylF=fd%t6|!Z1x2_S*MgraiujKdAC(mqFamvF2 z?Bq-7f#=-%HEtmws##3Wv7K!rSe^8HJ(NQZ2DXfg!3_6yWsur~&ET~ytk^&2tA+03IqzXV#Wl4=T>MXDJxlmlbzd?Z6McIA{ za$E3V!Wr)Ah3XmkwU{m{9kq4sceR@+EJ?W(IBD_7ZsyQdr#LsJxV2;;=k^+srT9$8Td3ORA=g*)y&e|{720(4GU zT5~u2aD+uvJ2?GWiRFn9u7vvn8KBJkVK!xU(Q^r4Ax4WQ3pKq?1lZo)H`D^{aQjT%OHMN%Dqi6PyXe|||Z zwHu2T`$&yb%xV79Rf@?jF7NY;i?z}3RVe~4ACng{$QK((yiqA!KLa;oEau98`U)ZN z!gADbAsV*bx~0S?9qo1XT;}ZfmzR>#Lri%>OuW^F-!yC8X1rQNo>w!2y*Mh>jhI0E zTIL+xrYz~!*ljO)`e=wU(yl44Y4v+K?R0G41iO)qD+?Yp(DlI)vRVqgm9a!%}|x+l8S4!MeiJ|hPCD5w}v1fWR7xFtic5qf)58v&4psR z>snF(2aZ^YzSo@qLRxl!}NfM z2uJ~)D5ZHj+zpH`Fk7R4G2(uIwD2V4i^C1%#-;iNUrU1I2g8RV@|407Fv#gNE8koh zmTf%K?t|oQAZf7LW430N;Bu8}cG}koh?yffBvD@F_xsjPy+QSt#y7MP zG;UCBrbG)+JzYG9E#B+@vM^dZ_Of0R-#-8)y{qWX31_~7O6!1KKO}3I{i`6=JT5-E zcOpyj(qZ4v>mtzFVl1wxGaeKZU-EF;_m}prKRhGvq6FobwT+CrH_H}{;1LJap%Xz? zt$y#EY*zrh1StDc0b5o`sd|@v=p&FL6U@SN>A4nSp5kD=YctMovmnX$>@qF{F01)vSK)o6QV}#8`e{DEm?>5OCrafN9ps`@ z@6C5}^Z)0vMt!vMSvs~y^|(vYZDcbS(VRtHNz+_8L(6i=61SPJ_WN?&0qdipArF;U zd#+hw_sFvP@kOaa`f+~%;Kh_D{qI*6>f*KEm4MUSW5!PkBB*m|t$o&Mc4P%G$x=`> z`#G-w!eeOV^zr`F2-s4tMsb%&;QbS)vUB&QO9czXrjx(t;$n}yoFbaLF&? z(>SpEJ(J4d}HokHp>50e*j-Gr~0obpIA1{ z|M?3HwcZAGh(pq$zLk!R6lBG<7w`HEKoo` zN4zx*{{9Eb(+Heo+?p2eBmMhUfNpTIzJPfrE4lrtFZlkzfLsv~qJU!azd)*ZwGWoD z|LBN6e`SpT4zmC6A6D7fSlK<;aHLC8Y|e~D&kMdhyt)_WKBb@VA7rGE!kQpU3V$b? zydoJ%b=7x&?8>%R@nunI3OzEYLscel7a;$2FkX0c5}vHTPF$l>vpG^pqN6cC?aG3C zaz;A4r)p!8Y1HFq;am>r>=wvPOqqvT&_)sx%9hqozrXy-j;WlS)D8+&stFyR^?r|g zPSy|orWwU4`R>KP+B!>k7M4izC^T%$5B!>QqJeyd3PwAYBN7r}Bv;Rz`g;lRU7JBF z7Chz~ZSdE8sAm~Qyz8Ck1FiD3Bld1S0nn!OUwMc{I~aC?dzA+nH4fh}RSs4jrYz5da!JTdcZE@XkA8 zq5aT&DsdBq2*^lQnVdS6-wy`+kRTvn4> zQ7TJ5BSNPqD>8nw@wXnG_Niu@*K_}iv9}Cpqx-gpky4~M55=K)aVt)s#hv2r?(Ptz z#flYoXmKg-?(XjHuEB!*r_cS}dtdwT@`YqFnaP|pv(MVH*48KQ$tb9<&k^H*#wUsicaV#9~C=Cq57od^7@qA8XMO%n!>dJ}JlZd8d-0eLZbOq^)a}JLji>a+&+|75nf^Unxq& z(!Wj*nD4IUlvwtamS)2-%TG#cayQ&G5leXG8HnW4KfFG0NIV>-Q3JeI>l9O~3W?FfB z?~BQf=r*nzEmFsTJCHAUJO(`lg93JDR&J zIRAvlDZrBfLs`Q`uGu0|^6!`cG{#}X^M3AI3>Y0=mk5B~)p-v!wA10V!~={Mf3Xw? zC>RRw050Sn<#&hKP?-WcZr{0|W^+;mO!ob5gUdl5&G5KSdHwBcfTWIc{a`z+PP|wj z6y9iF6Aq}O!p?J@B6&D`z(5TZFL&Q@I5L~_6D*hS^lt{SE`e?wNj-0IVWP+uI?wsr zws(H7>Tt_E>xTyS>CMO2!Ge%apdU%jFc1v7{a(h}Y)fjw=QNA~P9r}T_XwZ@DaHYW zJ+<*!^_n>OZAQn?0ibIcXKeyF#nTcl2pgpd5}HrlQu4dF&f8o*4FFZqeY;N%cngdV z^FDJ5()o$BhPiu`2@!9KcKzqgNH6C1TA!vT=URDbox_h*_GC$y!a6Z(4973SWi=DW z6abN0hdeu2Y(R0Uq~d@ND*wL9YMghei5GKj5lsOP5iDK4JX)J2@viDRyitX!m6VXZ zHH65`%`KVby;Zf%T#JbIlAH&rzca&J?De8_nOWBM9^`mQ41nNK&7RME>eb_pj^^~- zKjdnVZVFT3&v&P08mU*_Oh3H)`y}(!Sx7N(voNQZE+Y|rd$~X%X@+cT zEUC5WJ<%6Lo$MYZEa_VZVY=665919E-A8sea|4-}7{Jfyofyg(f#iiYei*93 zZ-I6xSaTq z2x}4tR^#qa2mRzx@2|fA%U{6|O2V(qnvemcQlNw)OCTjX>*IspF0z zt9hzOlOLY8q1!M|hnu$tgq!dmwr;IvOAw>l*6^p%W7{vy0YfUh0=eRTiC{bZ0Bo)~ zniWZ75{+C~LFFxs}=bmQ~=-(@)s0dzwp z&ulBX-k2!j&1Lb|L+khrB6{%McgIg(^{jYRUb0F1TfeZXozj&3+ZsqI0<~DbT~Fb0 zzctZ)nPqXG^F8|l-0R1~xUc}x8MWmx&?G;PlYu%4 z0(({==G~u@k#*jfHXL(zdo8HTl-atXt`{iqUw|;mcjtre->TjIH!46FvtIYH{nCFU z01)YPMUUF}ZMa=Ch^7QoS^%~{7pLi-$C17~bbJ2?pbir%zGE?lE3f}?51f@4SeD z3eQLqt&hQ^3*3|dA38?!dU{~M*!sZNXrig%c#~wFHUaWN!7Uv9|DKbIg*GnGS<|EL zG|*SMl?$5bJpNXERhtX`L-XgoZJaH=pSr(fXKemPW6qco!ZqfCv^kJ#0Zk&FgaUz` z-OU+~zcHCb&RN6tY~J~+_|MIl!k?!ykt1S0_S;Dwmqf;d2t@jk!(a1p!fn#H=P?$|E)_ z`vT(dcw^yEqNDc6!cZJ{{|qnP&3qkoAY$aNm6#B3D-4Jf<9Nf^t<}ub0a^OuwD|<5 zjxwojOZobENABDRoUJtnk%v<5N@XzguBuNt_J4D=|G}-I)c^ld%YQTH|6loVVvx7y z-|Y2&!@nL9$BTI~o*MI<|ALl!w~l~n%=F&`GCyn6Ib;8r)Gq*D9@|Rt_rI1}6CxtKTlcNEcTU>-=Pb})ymcEA!ylsKMQpY_Kee~TLbLkRCKED+g`>V*TvL0nR z-+UUFd}Bv3c;4P+aJ&=iwb%XqQPX^;oQ^C3B(XGqRUd4vUL;X}>3k7pFx4VD_guK9 zV5{4IJE~uKl*5zVY=u`AAk-66^C{-L#&J*7cfo98b(7c6$Ed*ke)Z~;ZFxs__0zj7 zxFB%+q^uLiN7F(FMINE}-Sg_4S7y6G@ttdW$J3nSL{3Uh7*(zIlIL z&7I?{87Fc5A*t9(8W0JHDd-!Y{SPHi9mzB1zyKa5{{~lc&46M)C9b-GPH88+#q8qu>GR_W!6YJ0s%UH|G|z~tjkl%u~K4H-$!=!mF~Tg)A0 z=#F!=r;crLd27n&JTtpc=HfP`)>a>L;|9lJMq6<~RI>AU6;iLsM;!=*0ZNWY=k5^=m4-8t$1qNaD~o2}VkLJ>oPc5vG1@j_+JoZ7MG@9yvk znJl_JlKUE{A5T?RH_HBl6Nvx%S^M}+WUUVme?Uq#PI6Xy@#qYa;gDaeq>IX%-*@w` z7VLWCh)SCr@2z8#dH?Q&ce%)HyT|lbrTRirn6n(~au%qe4w5sO3hxe!IFh@Ioz{pO z|9wlE^@Ivqg^^8PtO{KK_UzU-=<`E2cr#INx{%x8)CoBf!&#DQRA*pF7~f;)=X4&_ zvdsr7gsOo#tfjMTa~g+qO4}xiZ#V3->$1`9mlXzHTY74iJG}FV%o1^Gj6o_^-1Mw+ z(skDHf(I+&X!2~MyVX=Yfb~MB)}H|B&XI-WoO~iF8D=>@O3O5}R_*TNBHjo;!N7sS z4^Lwj&Lvw?Huc443WTA{zQD5y@5+!SbB+dPu2Ff__5iaqSI-Dd#c4J^z@$hh#HP4* zS|KQ8ZwnbDK$H#Ft-Y}m(7$}$v#g>cE9}Pb`8@d1n0gred1WJvs#LMQ@qrLjtTXCp$b_eHi8yG(sO31L=I95`3qdIe6#;!L!eazJd6x z6p0O%m7)@FEQhfC9>v7sTNFNPys3<%#V*mbQ)v)-JJWX@l4aJDTp5A?2qy)V8~VnM zA)r$QPJr&?^!OLz=l8UCHDKYMG%H}jXH(!~o)n!dhideJ$ix-MW_&XwEhKj1CYzHo z6^6a2{fspQ13MKHA*G4$uGPm~QI%=>3(wVy_p>s8nKSqkbB%ii3>5US8y#$m(voz% zIp7y#t(E;Cx`|7yw)w!IxNB~>snGuSoC;KRBbO@T4`ZlNKMWl5mj4l+;Wp09BR!Gs zFeq6|Cy$2x8C+)Y2c}kkG{1bG zmEH_}n5orf&adkUUxwGL$lmG*tlu`8+nHyod!wCov4~k4&_fq`d8}T2ILs>wACcUB z1u0p~JCCYoSNpCeGudkSUJnE`D8+ zc0nWedJ^r6tk~5?cQO_89jqsuWsg11Bfy4^|X1>yc_n^dy15My5@?O30CYZvC_|s}GNPJZ8P^eL&lct3_`P{5v01<+VqDRr_MJ zBSRdi#}}OUV^PK{p-90Xp%_hj-(R1YOLkRmWE@4^Ri&jx>MhKA``Heq{8^0(dNrTX zHU>B7$STgJ>!p&3yDK)?WK*8G7}|8d=En-q2}Ws7>l?mE=*Qfs5pRp5ny;MHKb#yB z><)+SWx6mI&Fo)n3caX%O0!=|v=pxh9dK)|Cf#df=LBQ&^F}?8(VZCKeY=X?nV?ubp>{`XW_j0ZUc<`2e`eg{SO&qAc_1WF!EemS zJ?k(C_+^TY%bO@CWCiW=U%*~(8;3;^jy2r#Qa#@zBQO1EN&q`LCkydMX$ZwUuZIHCbgg&9bGY_jGpT^)YEK_k6@3@tq^b*8=@)RiXXBn)oR~_u z%nYXBlLwxkz12)$3n8Y_ANr!vI+yiI-TAzx&L7nA5d>c5;A-0}izJF&g~~sO%XydP za!0Ny9%9-5cAsH@B20)X$VjwSMeLT*=l*!SIT4jEDLfMNLSPfL<5|@%ns$VFTaMW$ z=a<*lN(ljMLX;O=FtdTqYu$ldUaLG@W@^u|-&edF0tI26PoP%RWOnyTt6eQFGA_D) z&u2JGI)>7r0Te+*NM=v-UHH>!(dYESKYAF?TiC<%^QUO{kVvyRXyp)uDi_DjO5x*n z(esl|RozxMPU3k8gC2%Kt%7Y>I-tsgR7uS>8-;EcuVLB<5*n ztzBqZMa>OIB}8W!6pY#BR@=N!h|e$9xbIa|bJ-?%)@aj6TlAE<$F=GVGyzex_wCA$ z-ea|PLhS}tU1spxcS20d0l%7@{l^`!1@+~ov>jzV!AO&Ucg5+61Zz@3Z^)-S_wihH z^;u?jzX)5id4NIb)xJq5_5?D68awaT)wPHdMk4rqVUR=P{&1hsv*;$-=3zxTQClE{ zZXAQw?TR?I&9}zqYt_ngOKFw@WIuPs+(dH^iji!j*+-lDpr4reYB;Vywb`nmaW~n+ zj=>NX+&jnDzy$FO9y5H|LCVX7EOV)q>I~b-P|wA8-(t+1#d6iUGeCy1Z5=lndCdk1ASxomkZDCmKjy*=k0HohxS)jdQx(RoIF2L zMm(QJrmmkODP~czYN>0US6K}sxE7*=>`1FCV+80m`JFtMj} z#2OT%g==XMC!3M(`@?{2c&g_I1f#wP)1ItZTV4e0B9bu#e|$4^7%s~lV>79BRi>6{X;qlhVv7@QIRHoSX1+H`37JQnj6o9vv89wJn$4yFSU z%hNn=_7mLSGUw^IIQ3#2oLF_Jm~=JxA;gRqy0-fj(4&n-*&Y_HjriRr6=*gT)JXj6 zBavnFnX8=}F}cZ#K@Lr(e!%!@XZQ73%uVD7)hiZt3q8SQg3&cIWZK^Bi<b5)_yc=?B zANX3?ap)x|m~OSkrqQox(z7cr(^#K;1~G5-2udworNGFckJ3G^ z=!{`|8~hy~Q=vg={as3El^#_B>cMy1{UZmZ1LnxsW#7xHlMC7=Oz zC+}))xLVQ!s!JoMI>fK_fKzNYTz7khE9>PFrbPDkZ)+JsQmGw%wfHk>UZ5s8WwN%$ z_VP9pHA3+WS@@jea}2Wb9M_tMFfSP!OG&>!Ei5(U>QWSZ+NKSMvJE+PzdxraA9ypG z;7r|<0c5GXE*6g|3F-})nxpkPEI9jNtyucS&WjYx&`vb2Onlt54nM356=()yTvK&S zl@`B`SUkd+(`pWpw6BS)dH`^ec79g`$z8_MRdCQ+oQ<{?gf}F zDh23*=rf7-j7!~^r!6r&lqdd%4uRZZPgF^&+vE0jeZlvJDrKK}dc@1qY2mff>qJwA z$3HL^)DL_~Q_&umP7qnk4?t&ZzZ5rW)INf)7C5euOk>}iFDD`7#r{PB-Bu>v@V zXgLH015(O;N^XC=aaY(yK0Ud6NeZH)l_^J%FS`d}c2(@_lOOg^8LHB+w|76KSH-x} z4NGOVCa!7{{L%aQJ4)7|qg1E#60TV&vu|W^I-W?XWAK!oY)mk$%9`fNjhfZj4ZDQX zdZ|xxX-JbQ&&H(pL}8XZ|$5Q-+iI zRp5M%kIxqh!%fX{J}^fZuaG3OhvR%*Et`QUygzrOOsLjM&nSHqW{qC)dLRXO3lz&% zAKM0a`De-djFYiXw8DR*l%f?BWHkjj(>_eFyQr&Sr&wjEpW~U#8tXgm-9^vp#qu%= zQoKiE;B(^Y@vfH!%JpSM-H{UR;^2G2{1AW&W~r z6aHZ9w@z5(B2+UA!Nk@?zs{*+6jhqne)u_@X@3w?K+pH!5a)g0Tf+u$K$cK)AJy>0 zE!(Dx`=1@sfi|X9L?dXciR@2o8Ai8q=v8r5U%|R`94(@FQx~sv;!Y1I>g)X{ z7Dpg>cKMrKzbt9VTVZ_#nU>kv=z+qlZ(M1wOgFEkz{0~y8L|A2px#ePmGF8T|zbUr!aem);Bgk;4%O)gXQM1O$B9Yzp7 z@l;w-&li^M&-tMrx5=6ew(xA}xLxk)lbUaEOc&9gF!q$v05-e{*Q$!*;=mHiz^MIK3N*_UiyQ=%f&xQp#&U>m-hCIX`3hG z9WRDQCERO#rf~N%V)H}H61aeDy-e~aGe$OJvltbc?jBVcO2vemdqDe^JR$(LEDvW= zIqeQ&vY}B+0e*%HsgJkC*Kfa&0gtH>F$*kVV-ky-G#XL{H6)dqo_^tcJa{iTvw(M4 zG}h`s^X%=Gq*=$lUIi^gW}Ucx5Z&wVXY)I>Fg+^|C$M2j8J9I=tU*XoaHL{gj!OFe>GWRXF^T^yV613F^na*t1g z*IUWFEHQZB(EMXWwAB}UUX9|yhb}UK_();*bn}$`O~X`?k-#jD1#pYU#!&VpjpRhO zp5Db$FITl?4e@=+B>Ye*wwo0lxli?ghn^Ns8W^*^9j|yN`0OW3GMSW!7av0%>&8I& zH z4&ybSwd5qxS&U%S+qS%GWW#f*o!o{hky?{dgCUM+oaZD>p9UYwv^x_Aai)eui6<~ZRsIuh{wm~=|1w$Ha*i8SzO^bE2w zUuhRkiK`ts7E#j>!Y%&&#;E&1(@f9?Knzx!=aXr8d0R|9B(WpoA+zgc#vfvQXWurS zxWv|?y;nssG{951`86WtLLb6WfPQg3WMbCBT4(a(`w^PqOO1t>*Wu#kMAwNw zTi_*m#WD#mK6$t@aL-SCWPw1t5UTv9P1O2y$k22kpU>F7V{*EvuQ<;_K~lbJ&Wjru z_SF_wm|kwKyH;lOX>(N9eRx*Uy{SB|KRCoXTtRCX-`Or;6$S)5nh?}LysE0J?F(HV zH`=BjhbAcrYDG(TFj(Ny5N1(&U~>o?jujLwD)!iv%<8BuF_4I$Zvxz8Fkvi zYjSw;!(|QZJ4&tJD4X?6;tYV#sl}g7kr~{GKX5D;`T%(9VJ%&8Qbqr+Y=xtLU|6tB zuI=5dJ;_RmAK6nn>@)&9hck~e(Bu*5sAbeCgX*0Z7?ORQ*V-J6qO55I$;{D{+lchM z_N26A0-CihZik*&z$l}qCCSoq3B-OaDf&>XDrH{CSK`#J=vC782nR1APiZ$OJ!dPs zTd|ZwTK_@MN9=U7-4?4dS{xgCL=mfIx3j-4A8Rgr2n-srW8xjo7P zEl}W z<=d5Cl#{safP<4VZ6P8)p=5oO?d7#w_BN*u(DD|X&iJ5r;~9Qvme#`C8=bw=EXVx(Vo)s3!I~$XBbv=+1E@1k?6L*XlPg;AiB|? z@_4Q_Id=H`c`ZE4vj%rK0U59?3>`2ovs~0)k5?I0hT=*0o_1 z5Q4){c!DC(^tKO&3|}8lEDI)VOP8X4({7&HSo^9TKUMG3sDtaCRd?@dBQtJrtoMQU zA1y7o9}!jr>S133hF0)`ihKl|2p#QdD=GAwzxqy0PbuXl5nGGh9LUgs!~3*t)5iGS z@g+@{ks*&;(+Tb0H51{>3mCgi=60S+cPhN3xEnm3$W1AamAR?}+l97Ehh8BC3{Ck6 zms>l4k&HI~Zq~&$9Tw?hmg47d!L#+}RvF0=O7)yQ_G+;^@?vj$@|^wHjYPqBwD~Bd z^dgZlY&K$&-;$Aw>S$j5xn-umB2vQ7=!M?&Wq*p#)3$V@B2UoPp_Y)`0O>3dZJ)Td zRPpgWA;hM^sdVs3k;KxJk&Q)xmRym*sdCLmNK;Aw?zS>jo73h~IRG0Nz$lfzu!$9x z?T@DyUj9-n32V+CXi>%*+>o7J(+Wi#Np9fL^<7}AvmH#&FlSFNytm4of`(5W@1&q= zr!~ZQqmqlBuAtrNK)?!0?Eoevz&$J18LzHmI~$))Y5b4^$-koHQeg@#s>HhIO_hW; zeB!};7%?$pR`>d}!Au()_||ei2J(&JoSls{UdAsl#CZj{KPYPt(@S!tH%7M5QAj= z_1tbpE<<*NLxJOB6EC@429SOpo+yZ%{8uwq0TfN_Dl=nGBH(6$X?JTNE2nn;t`8fU zq&dR!5UB*cJp)v~)Rvjj1rBwpJQc*BlCCnV`_K@(kSHie_mH zhS<}x?(SBihS!hS78hu84||6O8$nb3E+Mh=|EY^w!)5Fi`bGO;55Dfzf}l*8WbPg7 z*@NDpcve-YYC9e7PyYhJj|4xA50#}q&$^i8ERDW?^gNmff3&02fLKxt>eFH`k19o8 z=R&nPl9lX)kA9TuHz+Pjc>4vnmb;89c=*9~^U-xjo$4i^?6AIi51Y&f^ne@vzu)uh zvG_*9l6F9d>&goY0}PRe-9HB9Yp**p`q{tb*xvss!1ji_Bl#AG3@Bz8O5L>)D*h*X z4dYd4mv~eUC<}N>Sc6S;*4(}W4feJFG4So8P?|X<3Y56^ilD6sm!&))(LLne#7P{9 zGB$fceGeuT^Vj3O9f3FHq7kLc{7l94@j3_(F{Pr__$<#1kn0#eO5~cIno!_3Eu-$Agek&a;Kk@<>g*#kJ}NiQPL;d zu>tw8-Pf*Unewnn4|)4%wVTn$onpJMKjxD>os6u6`|L07Y4OB!<*{gT4*`ZbAK|Mh z(Zq7i(POLg!Yk7_vsdXo8Z?uIw91~SXk)4LT`5P9oh~glf6d6zEb9g?_HctGPJ$v@ zoi_b+<_&ZiNQt!i?<&U|Bf#r@SvoyFB+VFjvMgu4&M}|)oyzRs5$-fy>lNVT+ZhLI zU#3>3t`F`tkDI+#chapM6&`jSDB*TIm#4aYP?wHcciF+qd-Ah~Z)Pc;ukp0!*nPmy zZ-M@}A{mf2_e==^MdtdZb#&DWsm@ITH3rA_Nf8$xzBfe1g6UJfNyl`*oZFs1!+kbG zZ=`W;_@hu-PB~O-^5DHa9EGG*^zNTx5h!63C}-yR&(V%S2SZC>Q(c^pV)eFUGAOe% zl`%Xhj3{9vRU1jUUFWm#6KbLlZw&aTn?i`C+^S*SiU)zM$mG6dYQDJsQ%<|ew!<6( zwdAT}13Pgg#PbPATWKNC98+0i(KRP{lRd+;G5YD3mMdT=m+@=1|7_9G{A76;ZITLu z`le@uuCYLijZ)Zhr+46t<$Xd{^c+KkCNQ(kw{VH=zs4-pT>Ry5Q4vp`JC;WxvP5+9 zx)&J@2cw5vS4z)CP5nyC<2;)+^g7)bXz%cak;31)jjM)x$1b~%tAgT8Eu_j3lX;nf z7Y3arqtwc=@GmlnhYPJ(Tn|n3Sn2LWSQ1;+%AkBP4}M`C!*ftPhS?0-Ue1Q!PvaZ8 zQ8Lzzh+T2u@{f0)9TTAj%5Q~e=Qo_8R(O_TFt%YH#&5U#6ihZ-Dq=&zkA@PRojNCR z+oNm=E^H{S3wp8nUvTpUi{xt^*s{8JrbkMF@}TZeO}>zZOSXtKu9$O(S-QZo95QZ)W8G`I_>Fx`+DD z&V|@S^TRX?eIP7GLw37Itn0giI0H-3m+?<#R0U4?NH=-`sG&c8=}W-t2n;qMOebp& zUYL~VXN-mLJF)6=13e(bO;nJ;qY1{Tm(p^X#cw64H|@!)&I++kW}W}R3QhBhGSe8v zP;b7I`};u&t~5NTe~g@&F(l~n$omL%NQwni=;4EWQNPA_kCTlvfh2@K@d1@!Gwd>X z?lbC-pH2bnpNM|w2>Q1Ajy_pCSE-TuIx+drKGHOi!!dqWp}Jlrxk17Ry(=laZ!@b= ze?O+xePqz^?1KcKI3@`e(&wo}c>HpBJ@Otwq1-EW=TVn@s~VLU;K&w zR$gqbuazp==VxsfunFzvUbU49TplmHKUW)a2uh&~Pqt9yQf=@X1YGc#!^>l`?uO(w zp3{e)xQA9t8=InQ`g=?aF(WjPFU03`xSryU87G%cXC@eN2y3?1X7Q7v{R-~vh^%x#t40_dl+!F z1BIAP^>s`BedE=P^Ny~5v!{f>q`eGJ4)1Ul%L@ycb|Kj5x_e zD<8{dWNTo{1l@h^mR^DY5n4^o`<1HL42>z`PV`aC<1yGQ2w+zJjiZ{Dm zx0n^H`8;L?9$rsYJ#}VWUpg-F__lt13-k{RB&I+irvUy!_H3;zO)Yop63@y`ZyA=I zxaW0jXGfRN>ucv$Yi3R>np-YED|29AUyFYJ@?}>^96uNq>^Fjnsvkgrj&bqqBe16k z7F@S(8@Y3CZC@WxQc_M+E4P?lS~gy<@s?QD=VuOE ze#O8{*`Ely24~xT?tHd08l!1Pk-25RU6Nt0>j4j=?v=^-!^Q?b+;T>FIIhd5^~m{3 zPi;zZam)=P40YxEr$}x)s4U;snc4oyQl4z$^^KLsY7F2@szovpeH&g^(1DTj#@Ude1DNr;G^ zS6xduPsvKZM%uPzy|&+FMo&06UT@3lStyoyjF~?7U%KoHtc#BuRPIXGvgww14Ih7E z4ct8Q3crdSy6Rl(t70^85LhDwdA2`wkZTu|6bCgDmo_sNJ={keC#o?_!@cd_R@%m;I%gmEgUz)}471>h(*tTfXnq}sTK&QYZuEk^xR~V%cBK`9PhUWl z)}|7djqipSOSBg2$uDY?=mtk@PHq&5D@I~vg$QzdJMn-3H zXhJ%v;D|pwL$L&`l!I&-t{qobX zfuiQ$tF^~95cs^_@Gp5aId27t#UCHPu4y8k+o{B>{{n%at1%46#H8lukl*Sw~aREaWg9gMRLH)BWzY4QhO*-|MW?*9@ld z;OxPb&Rn{1rbn)`maEHMoWvYIu&qIJ(Mj`)`-DPkqkk|Oz%Qo+jS^SOpgG3^B} z`A;*K-mJd*sfAgK_-n0iSziv@9X;mvPnItN*q4jvN|!h0%`uAxy(jfJ6(W{A0RB^r z)v9=k4WMMD>*=&cIF`XTzVNX>Z-3bh*UxsXd>4B)85nGv5r-Q1t#?ku&dg-BHtAFB2QM zufzjUyuxY{XGeeVtJ!kypFjVkI{373k?INV=r)RO&ZcqfF$GN0paBX=XDZnEtGe5x@wZ z0W?Dbn|WC2eapnz5yc^k#7mA{H-Z8kZSS5D5jTy>X+3v&%!a~RN+b<#(2d%}Cu3W{ zxW@`At<^c&C*{ec+fktX2e%>A_cECBq0-p+rBfA!TO?cDlu!6-^E#3MK|wSaayViy zeoq%C2J(4%FpG5_RbK2>2OB}%y4o%Tnv3OpY3)j!K=VH!2jR5`$7I!b!l7wKvwKud z!yRkRtP104!*JL8n#Nz_SEvNFVjHSCR=R&xpwZ7&{3MM-7X=ifh>CaPZb z6k@t3^$D^)&amn~HP!4tU_{eb6Q0<48v%6b>j9jcu0Pa5Y$%D-taK}|({?zHqe??|O^VMf*RKQdp>AzQq|37qMdgP{Bo0HgqhVMPL6n`n%4Wlot?&RN$ zg6RRS^?wsxL^{=thGl;M1MIMn<1xZ?kHT+ywv z3`Q+h{r}s(l5((<(Q1Aj5+y<#I$o(xj(=GLiq(Lw?Zd&xSNWem#DMXqrqbWj*H%gx z1cw?cM%b&JIxIQdf$)`4^r~p+h@nMVg2})AcPisaBti z>ZCG;ID7j4>qV`2+PQxQ$nAEuD}O#YF|VZ}hS+ia6*fDgUlnM*xP_EFebn5WtxYe_ zT1O+JHU-NK^E7$U$Ju>=Rr(*QdFb-^V3Bbgf@p%Vcx@yJ-LnF=H_%htL^y_-hsLGNInU8#jzQH$Z!q4J&ulx;}9a_74-~q zB+wTU%Ji{%|NI!D4n2`v&}t;{-+$$~3pRzBnvQ^4ud9h!Xz)bYWv0t0?5{L75l!rm z5hNWFEn%?O3q5+&_NTzr(d;{rD-`JHAUWb4=2n^;dEhCr=$fAw>6q5Z`K&C(C6sS# zRet%qEO>L{od9AUI!r2Nnx^wjoOYB(ADi>@4DJ+$Yyd8fT6&1+ucKe&=GGLrrn^Hn zB}+W6{wNdrgUV)MW<9@s5&4=te!3sFun*IKJHI(VE_N5%&Og$Rr~^0G8HIe=>cOlZ zzAj&Fud1;u)B=J?b+$A(8`#v}+3M?aam)6WULlJoxGfjSA8fcOaIw+N9YM0l-WH$f zmKgZt1N^WG>)F&W3|R`hgRjESAXPBT6w{hL;Sx&(u(5aq1NAI?ZS7l0BOb5o$4;YY z;n2iF^##`Sm8O{Zd)%gM>mi#o6_uHG-BnBgM15p#ifY^0i|~{&s6mFyDV8U7IXDQ_ zQidoJ)EuDsFA>nHo6j6f91g?c9x>}q@YHqwnl*gjP(l=)Y%hRsOle&46p1AipbXiV zKSNn;^fEQeQO)eW{nfm7v)tY~;^!<8z8(;xJo68YxZ8(okGFDuVTjn=z7&Fw-MRXT zd#k=90h9HD%l&I)ep~yLA@>ghKK99xN@-Z&cm6iKuPJ#xcQBuK9L)m66E&}qBGz`QN-K{P>sJ3Ea~dz6KlFOmxcl$f-(Z{F)~cH|GqQU+|hUzd;oj zBYLgt{C#E-H}+M1ytWdGQy={Xu(dh{Ge$NYZK%33*3LzrcwF7SlsdxbVg(NkpV8bc z5-t~Db&2nF94L6##d^{p+uPee2H(O@N1nK{0rGQMyyHPP@k1RJURqY8#u8i+EO~i^ z_D$S~;_pyUB>9|jmkNIU@`nrmW$Wcdcz9@;@IYrieK>i()`uvrN|?^){s(-zK9S%0 zK;NHqe$QUF_ww~yRD0ElYTc^#5l^!PDsqIH<(%?i6w+Z=g&+;$=PA2-^azE8wvLYTi~>o$V@USYpO zo9Q?R3zXl)W$AawjE@-f;cz5#9wX?1!Sr`7#cdh~Hwd!fWtnsFc1ySXy{x;vo+Bxo z;hOEKy(jzoZ0265`9#AL4c=Spt=7BkX^BLcYkD5^W4o(o!e(#xr<6*3CSYtn$xM3ZnuYfrntVj z9yw|l(r=ksQHhmi@6GKq4W`|}Pow|m*mY^k?RZG-cb`3!<?cR=KDX@AkeGK2e zqO@CL%#Unv9I8n@a2ktqV~(tG?&XrALpR9I)>nqG-Rh^-WtOz}o#}gb-q*_w zD{mgI;;Qe$p+v4`VG6@wvIY}4@3ZY&ZZGtonClsh{bBPcE@qiK5LC`cEgR3t4a*!$ zZ)9PG77>hiS#deP2t?COj^(f9A63t0SNal?6B^u|tv;R3wpa2ya}&Bo5!4`LVQiZAmG}lw9VRq$H)J6ph&7XDQ02J-l%gRF&ec zvrPMJd+M$Vlara!56r94T)7pC@j3C2m#@ZA>*oJV+ga473Cj7t5FPH#Ic}*rTwdh4 zzj`MOjw)TFN8FwU>hIqU?P#BE`0Z6&bZVhn{r=#|6#jn1e3UdC0rf5Fm|X+>eZ!^| z>&$(P>}~KaBXs|5RRV!@CQdS^Umse)Y_Rg?;-VHjXJp2!PS+3`w_0H&79W_gmA5lB z@116y3%N`pSQZ<0g}aYf@-0%wbnjM{+?ko}`nLv&;LUOcv(c#sMoob%u@dwg2x;Gr z;gpfR-Cu+l+BPfV#I39gI~n0D=S{LkDCdxIsy|$nXZ;|Svi1*L6XCFhqR_m2Tn%La znLCxi=^oJ2tAS`i>Hm=SR$*~9TiY%Y+=DxW0KwfgXc9cQySuvu3m)8^;O;KL9fCWd z@y1;m_-Ed=KHLAd_sKr#gYLd&UtP0m%uzM&=NZy~3EZbHGPE|r{1J^+|9>XHsYKhX z_k*J}jL7M5?rmRnrH!1)#J2|^naGi}PN2*E5o9g*s#g;qG&+!xmsv2_eG)t5yo01j z+0V2+R#FS=*cm^3IWSloE6MP7#F22g1&a)CjO$}gDKk&`6i7IfuFq%Y`I1crkJ)78 zS4ah~&iB`vFv6vetu>fTGUBj>a;5I~oq)N?z3>cPY4WXibr?B~PQpMDE{bN|lE$E! z0EWqBq3CYDd!ZDm|EO1T#_Aaop(uRD_^Y3^9#a}PehLt%o*X?R9Xf_CKE6=hrWAIk9)W%4$UWw$9A@%H77Z8o^s`JBk8*o6r3nN)?~=by zL#=>kKTU9>aY)FUPHKo;Z}fQ{1lFz*lg8+Wk;G5#z>r;&y_LWF&0(|YjwcjA8Ftm` z72!PP_~$EPf_~;`>H((UYU8gFj`;7rgP;SgrZ7-iz-Z*TboNf3V?isAA0pN>j75@v z;#KReXrkq_MgBKki+OL(Sa}jTmu|bAoX>k-YWccvPc}JjsIw92%nNmX;QHM%O;9PA#XjJcrUs z0CZ#PP`Nm^>`!ZYbF$W0tUeMzq}mmp&#FgWDTZzN_tuD$lyw23GO zbJXWE(8WbTI2lPq`wPwx%K>9m-%1sS^o~xBupN)2_6xhZl{9NcHvQX70k0tB( zI5O&+lElAVN=IS2>1dOl_$w5w;o8a6OGRc1`L6NnTjq$cTLjG2REsh}L3ZkBe;gGd z5im#0N$F_oqD;#qUv_4w!ih<~xDS<+D2%Yl?{3h2|Bgv}8GjcaZ{8+z!K1)7+Woz6 zsIE-z&Q5Nz#zW9Gf`8hI6u|?-c=S4qkjIGEF{lse(v9L-f=F_gdrm1oUiK%$_-Syd z%lgo`*#fM1_@$fFPq#ex!i$wkde|f8Rr*Qe;kLRq-d-fp06nhf}6Bx@>q@e1yliFTv zyG_Z28PwZ!+nrVy!&=(@Y}Kh(z0n;`{caA!_qKM_4UTsf97rh0?fCuF$N8 zNnfV@=htG2cVcrz!fB>nRA5)fp&z9#aPH$OZP3B!D7oVDZ&&>%*=ckYsIIK6t90e5 z1$frJVNKyq>R!)3(#ij0Uk-=S7Kz9c?u$aFi}e%^?2tNU=YF|BLU8JJ|V z$lLT6@AY9=p(97D?U4^!t%-v&n4*$s}IgkE@k%Z6Ca4 z5#GO^cVam4wg!DmMxeKA8kGMD#+{CJi`Ww-!Y+I$IA;Ksl+}N-dbJ`&lDUJ zwSj+HtTx*{n)mVE2+&%Ou~Hamk2P?L5?IzwuC?g(N%ryi_RWoGC3#1^vYiVyh$cX> zxxfdKqk25-OyhN|0U{+c;$l#@|5MQm-PjhrI}`Z&rYIJp|3^bSHA-^rUSY%wD2HMalG!y1-RMma zXNz%u;jdOh^EvD(0D%Di&5r>sG~0(W>kS`QBnAwNi*V0Wr590i>A8TsIj4YS z(Z@iECbA}vZ3U2Y^S6nL$0_EPgUYj!WBzzcP!eD~SP?*e&sETdSF>L~!8uR~m2c_!bidI=D>1J$n}!AY z^636*FjM}z)sB)4R4i+q+M=xy&a$CDC+7;ljen3N9DovqCw?6H31d9`qXM|XU&X; z-jh-bFMZ9_+Z=_C$wHHh=ca!c=_C^)SgBSIf zh(PcHbyPU^xH}p=>rpsTa59q#^Ab>md_!Nn&2FNluPqwwzX}y zR`nvoDlT!Ub3Ct`&dY`qhGHrfoItR<gPbc)<#9v{hTuXIVC|nXNBM;Ev0eY+Cjjzykzd;ssamN z=1PG&w49JS-0Q6RS$xPJ#S9WU=d#1 zQzcRG>Wrk}L(#3Y_K9QLPH#IK`Q^W_io+B%_nSH)lQ`a}{MkTqV%FXxoYo!9rx1t) zQzswN=w&&I$=og*fx4%DI(^;`P^K+g=v4a_C?@~%tMb9bFHSyMN?OeIpKR=vyYZGTNWAijx~sQA|a(aZ+grL-R*fkt&a z_WP*|XSvJOdlY zgx2|d4%dY@x5@B{o6V%g-k1nJV&wCuozw-a*W4B2B|d3v*+6~l6Qm|v;g28Ph`J%E z^3X(?0g2qCUaqk4 zwx<+cGuC0pB;<35wiAtZ)n_cbm0>RS1P*fNHa=^QHH$63Onmvq)7klOWQ{w`7)GZ`Akp=yS-I?T>0jOx+cHZDj)?I{!^T^1_t-^4?JI8sv z=$WDUZx+KHUAYLDdP>26750owLK-pI{$jDCQ6zNrmgRX<<@7KZ&9?~EtxCIhRNC#scjmEQ0U^|}^l#qchRhGOXP8*2QS#x}GR_yax$y>@bVxSa~>d0kNN2@+_4 zbdJ$>=O&?%nbk9@*PwX$v?p1sTAXW$VBtnGfI2}2qh?DmkI1r&83RxLS-F>1rTncP zYKuRO^WLQKWF!ll`S%C;f)!JHjmbJyB)L0z$}GGw^auhJu^gjMRIUPQbv6T`Rtmvo-V+A5tSqSJ&Gm;amHvp1 z8S9S$C28bUq`03K-)hvm@Ck50x8+-6z&+r>rR2?dWVzqgYm{2f-|^4mX>!tyO)TaO zz@V9)@Z0PMM8MQ3s8_~2SCY-DuXvEE`(Lh!At220zU41Hx-Ys`$(v0fx)KmruX(;^ z+1QzQB`rzbBYR?;ed3KzxV~ZP^4x~X;C^3lLNeB%34|Vq=Xf?zP1YS~bZ* z$;QGwt-k&oD)Zn$H@MSU)tjS8r511mTWS#p;}X!inqBoehc(MwU#Dk`W!ykVMEkWa z+NPPaNe?J!Y2hYxnhTM4*9TJr9k3&!i)Y*rw_cu%6Hyj=0?Mr4V4F{ZsArdkI(3uA zKK!h`%8}8iFA)A1)HYu|wHxRUVTH7-{+!d2(*iqlwJA{fqB}lmz<<&f9&?|aGFPpW zBp$VTdREgZ)@n$SlYmu$s*DavwLN)S8EOXWZ3Z^k=21*-n?+qi;NM7DZ+5FjlZdb^ z`KKnTLCZq=P^E-eLkJZl|L8e{8q=vXb%U)oQ5VD zUQR9_c9)Qt_x7EghUk+!nQ(yZvEcS2GQEt^b}KA&pHq*KRVNtfUt? z)#jE_qHQ@*@{67rEC;Lq!NwBz`F?|pXp04pT45xao|DBZRJCLiqxQcY9e==J&Fg4pp|rb}2=rqNSoKb^2t-i*isd1=BkcrHS=?NP>Rzq+l{>nDQcllS0**ee*I3Cs*jIsn0fyjDLQi0 zG9{{2rhi3K8e!wqi*1|jans0;GLwOg$5U@I`L(<0ba4CnFAPf*stiW9Lm0K&GKF|N z4+@Hem7{%`DT);lN|Z8+I1xl~C0t~2^3-5kxUXA*lkqs=uol?kjn0gpGJ481n?eJ4< z(D#Yv=T|eFmsiy_XHPtumkWzU2tO;D&j1$_%9H z@}&E7;h;i&?p!z5s^=oXkeCw^(Q0b-unS!jScp}L;+H%U&a4jEkUChaXj||5T^0CH z`HvFtsmU0Kz52!!&nk6JGem`P1+G2*J#Qcaga{-mS&x`UCy0FbrETHE25n=joI=%K zg0#@KLW@WVBCeumM@nG6qj1ahsYqMWO|1_#>YK}bJRuZg;Ti#%D`s&Y6e*zXH1CNVckFz+_f5E{ z7*e;uStKYLhj10Zs0@~_Dx6Y3II%5hl8N?#KT!6cn6g4ysjtc1gngNn;ISa(laJWQ zKURjrJOY2Gyb6|lnt6Pw++EyDnT@!$FQyQK6`Ur(fkll|PBZ4i+>o%b-qdz(A6;OK z`KfNNj_H8WyFs*)3NP;!oz`1HS8D+!u17>CQWyu@PJB{E$__ha zfMu3)ZG)`UibX~oR6fYby~P)`k=6Sn%Zd$FoynI*9gPUp;!2Ju7a{6hgUs%irt-=u zW8<9CL1A^|13$r!Ga=>1oU(U4h*$(8Y z<@_X+Ms+w)@@y0iHYCzNWeepMK|uZ;qm@`|cTHwO#B<6SW#B$6%M&xTves1{(s(n= zvL6)Yxh5B4uJU65oM0pL5^4px_rwsvP=RN%q>Z`=M}f2A)E)Pw1)q2qXdUrV+9fQY zpKw-xz$PxJ)QT4OMq9DwMmyw=Aa!_O82|wn(ZPcA{##+#WDE%f%mP_N@o@4%)CICY z>R_OF*Oqd!bKsc^91>TxzCWGb#^1*Qamh^1hW(qyn1 zGTq`~wEfG{-H>I-Ne~I?B+&oN8xakSnvpZ{`*+IzYs;aF96=9*mxL&c!*A(LE@*6% zn)Z4;)EHF8W~kSZalbu7seaX0Kej(rg{K0Ojgdp2?8!%`;|jvVNyKzSc#!=bgSo<62NPz6 zq6kDjExQtcgjP0!`yW28$j4+a#``rP-qrN>5A->k9hxLo|&ba{z@DL_2Y2(DUXl^Pg$ zyEuh*5_eOvnt*c9`ok?6{SMV+=?5^BM&ZaVd0XdR!W)ihPX9VBhI@Y^Fv%5#YcBB} z+jGSG^GLgtm7BKDJ0n6|p5X5X9=`;1&GsLscs(0QTBN+bV4cSG5%RboV$ERPt7rZh zW8qHF)^V*uE9Q2(_TlrSWtpw==VZ&PDkKw}Y;3pT55l6fa{(8cJ-?vf;4NN$FeJ_7 zx-B%Z=JWpi;9=U=XkVV+ZY?mmkE5)olsDc)KR%V_-N9eO#TSO+*|a{O2J8chhfl~x zd<(*`un1)e-T8*NeSTpq58C2o@kv&>C$fpqCtKvGWo|U2qZLOzxW2|rc$e<-D`$+R z@>eMv6A~*#84()ufxr2yAnum#Ba+-+xs_*na>qZ@51w)t~SeHml5uM#215ntoYWcsKQS*<$uRHAK*%R z2aWQ-BYQ>Xn!{2%YDv7ZG-<}5w%f5VdPfx=MJS~|1aoTxiAvOQrc{+CrKWfNy~M%R zfdW87^>>M-=vB1j&9^4I;Fg5|0QciNUe~FqukUkce3|uMX|#)8{Z24iWl6X6oWS0` z(V&j%a*8drgK4}>M7q;+NUo6A__b+%t|=xwlD2>M9fJJMhGLWCK&+3XkpbYgR8teq~YI;E4AQ8^XpS z9GmWvH#vI>u9j8a))|>oosU$i?MFS5dRVsgsh#lf1F!jK-?+(`w!bSxu>xM#7rZ(- zI*orF*zuZI#14>-cz$|QhEC+vMDtZi?R8RtxENg#`jO$LC18Mb=s!@85tTo43Hs+0 z0Pw`ubS4miY@0?{PNPZEy%rMyElxm$ZB0UqcqNrpP|G0=TydSaV*fNf{^e>*mq_#Jq54O}b?Fa8u)L9CwPOdb-9JqvF%1W}c`x&0w)-S(5 zweHEE2Uvg}*@BSV0JWaJfB%b2DDe@S-G{NtyH=w4;+d7-`oN0#z z6AH(h%2dl2c-=4Fy*j}^#=kRl3tBdmmKvSHu?BBSt2H|p>L0`EOmq&HiX)BMtjl>#==hx`|p#K4e)zQ;w~hlAM%BB zdcyRx*_t*h8~-`M?($QzBCGuNj*y=-`FjcFn-sGjRc?8zp^ncm88O{4&l&4AB%+|9 zsYro0m@BIp8@;}WC_uz^fx?Ot|9;JAWPb@){?9X|Wl-8rQs^3)I5a_n7WDY z+}F~q0&I2WOzM53GWPNjJ~x;AmIMCB_|RC-#evD?Mww@65O`~8CS2Z5cX;%Zx)~SW zzffU}POnzpn$zXLW08a@^2IEGTBQ{H?rnzmQoL!hYZeLbF|4w;cGfcq$G~UN>QSju z(#~ttl4zf)`Sm0!6hfruP$9NHKxkdUJ-C}M2pib_oge3`{c9Gixtkd1(%bR=n zo2c?3WR=K#t_?!-j9w;hS5lQW89DH9pQD?^XOH_l%n)i05n(Ikj(!ql!%F`IQa(7dYeol8yQMVF&Poyl}=p(eYdeu9Q z*+kmeZ@sC}K~ha-6tu>hnXVAZM$eK?C#R@zthKDatW53J7SZR#^b33a=ciSX zq-C_l;mb;9?j+lVTYIK%wS^;2+&>)I1Foy=++3cE!wa{aT+OvMjSE#4Hc!hdw_p?T zPBPZCWhH#gE~{#G={3s#&hTbzcI#oFZ>*Pms225xM6t_oQcc>epC1tprWK%hcW>5& zT2i}|)__GKDU-S*3!4z5fA z*B|y9IpyPBbVpq=wjJ`;3=CGp`Z3wPV|@m*-;zk?TJqR*N==^{6T7+V>-@5wszCP{ z65>NsRH=~tyc$D7aI6Yb1(|RLALVj|m#^=Q`4W*!5~oM#rY-W*(fz~HFs~SI~oz5Do*n!$^kD2Gy-uK2rGQLa9>}CqJH^Q z`xmr>@gN$gwiS3e)B|QN7sbLB$B?FvGJ*3q)~nKvH&U$&$zq)}z-9u~6L0@#@|C(R zA+>H-(wAQ3%A}Z(MjuKW!|8=*fyQL}gq7<0L5?{#R>jUt);S!U0hh`RR#Ahm`>sVF z!ie%GPqfX4%iBQ)Os}`>x$Pz=eAUIyL0g!OBLS2+rf7F)zWn|+ z6_3YW->1{};>In=c~$d)+V7b@mnYZ0 z&=URzTrAO;osPpKZb{11QE9w7fQ`!2Bnn zY7y9Aa;iXa*w9!U05nEMccn}N5R@^rNHkZvw89CnrhSY;h3-^#=LpWQ(6@X->xIA> zGTjx>Vr%C=glQ%K-CQ5M+l~HDm=?gdTikB?2Kv;~055sD$$>un#VWsl6R7?i`3QZF zE^0I=loAQV3&kAWgfr~Oe*a03BF;c0H}5zBPAF*v0IYYsLIyZ?x{CiYJ^%;%UkvVl z0Jcp3o8$e(|Nl4B@&B@XIbtR4>>%bvfCB_QZI#L5{~flL`MF8nR`0DsrL?y|oc91K zLQ#A_l~jm8w-Y(cM7wWq_Bg!Ym8B5q6Re9I%e!9X(MPG%H7Fcs%`x!{KXGT7M{C6a z^`GsSf+G_pdzEskGg)~h<c04m3}NVEz(2 zc*eUK9M>H8^CM&F<_!v5Pn6jpGn-@T^oiI23OGYFGHQ`@yDy}v zx&w;6@8134+eR`|APyVNU1H zEhbLjXPkC*WT|-=+WF zJLnFAmoPZl@UKOir7D21LWO4IBh0yrQc)x8(_)>?p_ZRgQ6x>m51%xDTT8-!+8;R1 z5~2d_J{%Cv9sA}DdUAD&l#uya+=TyfXfxlaeNz62p53$EGvK#tHjGqg8LYgPk1czR z#7)P6?EdKP_#Vuz2WEEoaIQZs)aYBEI9osPtm;k+12gaE_#5R;t{h1j5t@;$0pRi$ z$rvvfsiH1VCfv3Y>U*5}U1VG$l*G5JvyXi7X)T$O)d0c;T-{|k?Eopky2L-E<>uDs zNw^sOOp&G;YQok>hco6HBRxnt%)RDm^$u8M@mclODafWq#=Jmkl%S1Y$NoF%A2@rZ z!A`?wacm&abqd|&HO=Rx<+PB!-H$(n?IObbT-bX z2OznGWLw}_f%;f5N8a4%j6oz z4U*IoZvp)j?d}rpTA$!2&Gslcy1&F!7%kTNI$`u^z^NAA6Hk@VlqD?Yzz)9gc^;Zo zBl7jY!DzI$+-76&%~NHiy&U=A^QMN5LY3+KAgXeD==oti($v#8+|5jQfFx5e{t}EY zZ#9i;Qgc<#MIvbK=_A^UbZhM!e(M}+UHouc&`s_8jb72LLQ7 z^^M8YUMW)tm0ijExzGKc`~|>h?L)8@{NqJqpl3JT-%b|CPIG@i_NSf3Ev%dAIKwkr zZV-Lv#mlx!F5*O4eRQW!{?im!MTJv0?rj zOfywwc;bH3b<0kjwp{ppk^^M&l2co|dOuE)hVUxh(|N&9-KrTLm>IkRsmL47XBt*E znl}oxxCbb7p8)cZ79q-3e=1JazmWVWLBD2ubvA(Gj{+p!!j;{6V^uch)x=EUrql`p z=IBp-8g}09n7TUQs#p=XN=XJ4!R3fx^04}*RzoH1AnP+ALurCNNfIw4fPbqM*OUMH zK0hLT@|cqT7Gh0}P@R@c&YEOiUk%kbiX%YVx|x z{%v27kV73pR@0CAcJ-@WXy-2EDYp95wM8_v5R;cobl69OPpMPsLPe6_iq4ON?p9(&B1XEwF=}A(lHx#R@7QU6PGo;q;m}N;5A-H2jz6mZ7?=vrsc*c5AQ=*Giaf_OWuInx z7-H>ufzCRFNjln&1$Evf7KI$Sy)egi|X z-;dSMg>`)j1y`D(*|?x`=N4gbX-su-1{G>u3dscZin>ecs?!B1O^HO&II@BL*`e5E zGel4s!KDX;9fGLbX-B>tVQij0B9JT+CBg{07N6j1HG7&_ABbzT@0{Y9w{UJrBv22X zA6zfXA-`j*Pg{_NrAL^fhvm;I?U*Jroe<&4_1vK@a*-?bV-_o7-}l3if3YXr%!)C( zj5O4FrcAPfov@9>wt=|#?k>r#gF=;L{n*}|1d^=v!Ct5kuhM|k?Mehi8W+cw7VI$P zezh+`4T?PTpr1Wv`VtE(nrY9sxe`MjQQvWSvaFQz0rJiI`uYM@F!!iQhKtL|Xu{Qk z!|B||E>Hg|AXFVevR1c>VqyV6i5_*u=PSDZQYfetOSc7I4E}3riWk~mg&A65M zvkhZ?Dem0d;l+jNkjgdvd*-@pRzxu6Od(O~7wAz2^ZB&H-Cu8>s$=WwzFPdcxLEP< zc=+MI`!+<~elpd)FF)d!7{E%Widr0EsdWnHP?MAOPFusw4z2PtB_TM1j2)(yfBnP8 ztZ4Zy@aH;yu=^%PexB2FeGy0yuU2Q$`&NxC=-{dN?Z%%$h$%{P4f|*5gL{3BQF4l) zXTbKv$r>JB4l2)1`r%%rd?{{U_IjzdCF4r{KM}2?`3eAXuSwJy66}3mZ#>p( z45mJNpL>N-;)am(zpzxsLqs|9TZn*m7~scXO<>Z(qW_{pR@~IfbzNK|B<+aVy+P%t zZ?8$DT6j!~2oQe$2hkMjUjry%&Q(Jh3dpkLEfC6LSpF9#3knUEJX7@S*9oIOe~x5z z_4^U#V1yy{b$VB;f29?@V)1g}M`2l0DP-+q0;jwD5m$aJ@Mn2BR^=6PiLzQTFPO`#1=^Voft! z!xLK5Za!=2^Z{fMIp+n2b9HP?P>$`O69YP_?3v|NxPeOzF&yn}x*cY@RZg{!t?-zz zP$0br%7=11h1RutOtio>+E767XoSeK3lqP zWxMpHzJz4Qfq&K<<&7$R(In)yds)$ng>?t>;l?yvS9*jBn2Lx=Sfedr=L&u9h$3HV zld;JHd1S%+vqK0IKRNw`y%KRMsa`1ey2AcuWb)IUI zheg_+O`SV0EFM?JmzB;6A6vGu=9KBN?psYb&OT{Ozh|fnzcN4Z*0Rn40hLv4c@FSc z34+miM^g%J~G6q{pIHQnap5<>D#S3APVs@9%H=^c{EnmC(>WW->TOQeJraX@_1@~7*mTilWvQN4t8ko6PpK~2*;^L+k6~2uS!@f@Ry^?NE@qqQ z1B{PD>V>@5e}KxitTDsFAew5)JXvs9({3>Ul0oXlEp*}8J>xCQApBs{)fF_cHJ=5) z{e`uii;LV5*~l3On+bM2CXL|??K;O;01DiL*0l!hLwRh?9)19eX!NGPCp|U9CNT9& zvCmqx1ZJT=Wc7W`J4)h3XjliKj`SNkc*qkPdyKO-m|R0iGt9@lLcFVeYXmT{VHkr! zXbfUIuqgUVA#!)NFR$D;0Qb^0wbnlu={p6lHwVK@XN3J!`GL@*A-q;h?Ba|F=C;|7 z9t-0TREidH%Fpd@hi3x+h`|Ub;BL=RO%HCR<5!W2dOtK=pks@}Tk;K9a5i;;N-f#s+@vg=XgGFzlTM@aKCdP`B>W!?u>gx91@?VIiXJ5c(^q zO=|1jTN&yLsq^w%^`o@T*g`@bwy(P779te>~;0W{FoYrkxVL)&GOIVK?vkyZSGe>C|r*wxO8p5lsCTft1p;Tm(uDMG1nK0tw> zHw&4sNg$9U3tIQm>o!xFtRtNMo6;XEbh}j_@tR*8zqd)t;>hs$2#Fyu}4=E}UJtDaSw6(6!W03kq1^Z4}`YI#|+&|dHh z4LWby#;)}VKir3wAF{nKDs>~2&!TvA_{cq7Sr@vKYL5u6%2*_YCF1`^_T*sK76ywe zceP9V#sPS->ER9BvL)owm>jgWt5ea=aWiq+uIS3**veN(n zQ?NPNo=qU~Yc3+ikKv6;6EYcAp^0f+Iqx2!rpsow{ecf_t^))|sa#y%eNM@hUqq$x z1iC-J@3ci!&h|)7*<;!Isjon>)Px7;;wB+S$+wK>KmHYVE4X|hlJ)3O`JYCJoGSkX zow+rc%F`Y7V9CDr3F@Uky>-^}NmYzC6&4jnaL>%m4I-|R1cpkQFIt82a}JZ?@^5NJ zZ&yldc0mH+UBgqQwAFpMoz$Y$@6kcq8Z|<*mTh(V?8Agdpyx1c^|iM))A&g zi8o~%iXV@5kvl|u9kol{nXwTzb5VgK+27t+?ldl)Mh;K!Z`QRmr7DuE9|xfvLWGZ&RwmI zH)Y@^7$UPdeo{{vob+cloIq!4a2Az>Ud9t=;afyz!sk}{!zT>?8r45aXUp`HHg)q^ zC`S@0nWvFh(KXs&d)%|bgvh(Kxm>bI&X_#;3@>M@%p?+2|Mjd7n%nl1)GA-HWxd|k z?!#585Eov~ZNx6|XK$wRBl4ev-66=ULuhD4u8BugDYh0oh>i?P_X5@H6Qtz(#nlh& znql6@IA$Uuu_><*grb4licGetmHXDXyoHZ*ZPm)ebIT$%gBXi35rx)B6`E}w?B}bP zs-Rh}^3mq6VMvZ)R=^pgX>7>(Tuqq(i(Mo@8(l7UuW+)~yC5;R7bkeO<}Q3I;1~Jk z4G?qjvX{F)mkP8sN92(Y9`?_tvgL9g<1U2^^@2qNgLr2PUI3Eo18@JA*;*FuwshN4 zJ`?;Uw`(4C!cZrNBa;grw*?IsM3vx8k-$J(`I<`rchYVP-bJWZVX{^=UPKr+xT|GU z5jI4PBKuV1OC)l_{(6EubJ*)(=e%ogy!^YG&x3A1w6{5(FPDyQRQ*+FI3G^N!}-Pw z{ZQ~bu|S^H(MFp^BN8$K7Ety+mM%Ge}9L?<2z$WG?zN>de5iuc{ZzBT`r(0mc& zVSQpfb%DWA2FB%nF$QFh%!bQ%rFhSwMx&@(bG=3{=yOI^r)0*=sd#s_iwic=Y%DzGBqT}+XVVW=KFBn_S8nkQ z^ga;rida$|<_;IQ~7ljHdzw# z(8nF*WJbN=x}Rgq%C-kT^?ajubB^CB;|{lk*K*IR!buMB zELa2J-M1Ol&sGSg_bY{QpitMvzwtSLH-pz1Ny#`zg=SML8j`hneBsDR&wK*Gux|qA2c8s-E$6@*MQs zF}{gX>><9S^q`NRP$^A*xM;&smsJ##`ou&VR3dUQSXt*{c+~fG&G53yanDZtu;PF` zYT_a58zS1dsBwuxKn>35$JvpwxqBzk%37bCH<*A*h-H#yvPi8`lb(!O+@GE)GqCDvkT1qX9Vxuh*loIxKd9QAm^ScK}xEzj^H zd%Ww~$6|hEn`DksAT7a@9wKfe>yK~#%iCL2o_+|BSo}?xB&c*c#6IHl=v14iVnp#< zPv?S;a6wSPG+q>)IJWu|=cQ3cP9e6|DOdAxMm|R5RaOdVDxKDhKMw9iIW0!IZESLi z513Z=%b>)3N!d)YDc~rck@-9@ZP1(GQE4=4PewB~y(HMC=_0X3=D9u?QwVfpYBhc| z6@O|o7C@M`9+@|^$w50W=Vp!b<0BXRCcCNr7h00UN`s$=6+2(VN$~rciN-~hoapP&rYpr%PK(BhVvD0qG>R8eM6K0}_6p%m0;}9FEXw|MXvAQ?60mqym z&U5Zq;)QXoP9695kKeWX_clGLhK8)QH<61|OUGa4H9+@ux5$T2vGJ9#Np$Q+wvaU< zKq<+!0OTUGPM1EyEg56b)2M{9COlj}i{v#vDzUX7V3rj@uQm^UAvLRCdHS1z{9m>o zHl-WH=Ko#qTdPYGxTqU zGgqqR+O8J_hbQ9U#C++g!;}(tj@y?n3io>@mVNk>jsrr$_xEuU(&&_4Kk!32HtYSy z;I8{wZU2h^3#X-F&Q5J{SkYaQ`C_`ODC*|PBPQ1OJv3lX#DS;Z(^Zb<_t|~@HW<%ynb~1@K&^PN>czDEnCkKkhtF)zd zFKo=Uxa%BwLrAhFP6Nif2L1QqpzZK*LfL(=0Pw&CKdL3_=l`;2y=zHyaskt3HkDjO z{IC2h27GdkcvI-2>;wJ^VU%u0y2GK~xBxG?qRq1>lX7EjLU|arEl|2miU!a%W zs=K|ERB+2oWkUMR6N{pT4Oj?p3TJ z=BF1&J%H5<^Uks#OB!bW?teF1p)*d?(-*co85v!4tb{f#Y;j&{Z#6JC5Bp;!r@S>k zQRQmACe^R#CMVN-YIP+wuW-j|q~rW~EpLEjl6=x`-@wit(2$y(@{bRUNm_`#`~ICo ztUp)dmu@|u^F;LNdeDJR^6ZphJ3lB32sz%zmn^aJ<3fhpV&Ar^q3a09xIG!X z1E4Z&uHb_W_}|P+v#qL9jc#p}1-L-1;{d>66wq}_WUEU%3*FJ*y_-&Oll)BbdMHbJ zV%?JOOjGDjFcU--PQZ@1C{wEAd=jB=Pu}1i|4DA5KvnU*IG|a_;tJmSjrx0@xnXo^ zMCr*xD#V>a=mKnIJIP=rlK<$T=;$U=x>%Fvt0(Q3^$h!u3^%bO1?SuM12oz@(y*Qn zVx-JJJa#fKATEqzi?##>0KNy_54|1sS~#g?%&@Y~=}VF&1$eyZlf@DZBY4*W$T4@< zl9`~t*wX0#fSg*_{_ST|+X~BYDvEw0Fg-I6)FE?2_^EUoUl36pLqF!^DJEZRx8bx2 zOoSIJVmW`5rl$8N`1U}oyyBTeL}O?Px~QwO)g~&Yr+XV!-_MZDy{!H0?4gIm?{)eU zL;#8uA`3#l=)X@Pjx7eF0`)o~InGokFht1iAc!Z^DUq$JF8(D2#?6TN?26FNx&xml z_S<~~OFWS3df)epphVrU){=?;7mg~ZICeqkcIMy8>hi&aviX69BVSvh32FA{*J7#~1BT|TVy4Hezw zI9*zK?%wfOs7h=Iq?asx@G76GJuG11)3+IG`4hcgy>j+*(u*+S$cl&a*qp`c&(2D; z-g8)->*<>`^Z3MNwMMIXqJ3t)9sNmB{jY`Nb~XQ@8#aE>(_1ZB<@?QY3thFRw=80OJIg1}#GWNpY6w?@I%SBt| z6+c%@cT;-}pbzdu3U{V7Uo>QP!=M^emj2xZ%53~CQsS0ii54-L`L5af{arMfl!OUm z+{P}+>N%2#3Goo})&GWyb~j_J*!a^datYhILRLoao9o6033%0#!LBzL60?W6+^Dq8 zcJHbELaIt$3SRY~pDp$e#z%ry{~AEm7GMuZF3zN^WwO}|m@hKlkVjVs8B5dhx4Kd? zfcClrFK>p#KXtlf`O_BC)mL0PIU20j!!G;NqZ+Qx$aLIB?<#9HMWQ)@9IS*1=e?`)(`Me>^jG~K(%w2Qs;=uB zClo|N5a|+??v_?kr9nES8|h|1Ku}7gyFt1gdT6ADp}QM~p>yUvc-_~1U-$Dqzt8WF z_xv^Iz}frkT5IjKzw5i=sJb?b&^&)yn=O8BoixaQ!IM%0xm8P=KjF2v#L<+uv%us{ zdo}AX)WGigbNG&fB2sp^i02Qt#0_h(iuVzsl$9Nl(5YYuY`QO=hRRQ3K(oRxs6jS1$&)!D?+ZZ0WB z;hOXu4-_tP^k`yFWn(<>5Bh3*>irMl!jX0LMFW!_v7M50m8^r6&pXz3m-Myi9WxVA zA7*Yod;`Yyde>fe%-3gPOvPgK{XT{Y3CsYU6nV1vvT4`(%E|3N2OewFMg;14~5nB2p7L0F|9A6A^U4-BhGnOF7?UuBOPPg&gbhi;knn_BqkXWK<`mTRzr$xv2>`9IHVj4#DL&Pql2J_f{Iiyk~1 zfEjxJ`AGu-yhPrt_@rD67Ptof0?U(U=hfWTqZrVy@}0E6QV8M>xIzX*A~h^91FGx?OA z*mv4z!ZU~eBoC|mZg{vd8{ccTye-}Q{rS|!PM382KSM%GUjA2y+W%Hno^&&juzEVO(io1Gg>EIy(W!=m4BILX z&DYflM|}Or`hfgKnsio*%hz#W_yuD3USQH-A+{Hz_6G+(VTyjofk% zM$lYc4$z1v5%VPgdElmxODu?i*S75I_faQB1IEt&aa0-s%sYy{``aUVROxABl*3J* zvr0a=ha-h)BLr#a0FuGt7!qt$(xpk2B)Xuwew@7zIHpPc_i4#@uBMM`RKa<^Mvu(m z@g;aE_R+n6znhQWwHRip4*HQGM^=DxctBJJv~$j^0W%2oBhzXP_wLWDo#uq8>swG< zFFz9p&o|$Ldx~ZHj;Dv}09l^SH%0v>$DO^dyyF5!7P~2hQ-BDO!?5N2`yZmqfRXMQ zzddf0M|ivo@aI96N%W6SP>j5fh|Dyq-L8yjNshmgJy{wMO(zH1w8B0zu05FOm&P=a z;CN`;!mS(XRUhwygHkXR(8mhw>G%iC66@s&v$MDrX|if^6I%mvm@|lhiQzlfj1I^8 zx@LVK&2F)v4b&>+>Vb*k8!}gZH)v0NO{h_p#-h{uN*YJ?sYq{qkoTP)%GH&FEI!|F zWCq|HY^}|{dStWZJPGCThnEO>Gq-B6*Q6jyI@vl)c@W_W`{|w~4GJRW+d}q24y%1s zF=mu%YErPto2NsLk12^Hc*Tbt(H(KHa$oX<0e=IE{E=|PjqWYrhw^IR7IN292N-T{ zo$|Z}NO61Z;sLv}?{1)Ae&oZ~=(2?P{O^<4f-F(dBv!E^DLYP!>j8ybIMBi}KOUgc zoRtc0Q9?CB9Ge+VoN;RI}iK>RxX+7Ui!k2E?c4=BCKEp^d@$IV^-31`-Je&4O?!@z9HXRrb-n zxCF8(c$Y9rc&boBW@z+Xt%F)i{>$M$drByK7b{eKI9}_|-1GjgP^0?*<*0=PW~naU z0YTvji}Vvc4l`vAlFugxK@e}@uP5t3XoICE?GsuZF?ANwE!cp=XKx>M&D&Q}!|il8cT;i3BqnO^=jh;=Z>!i&Jf$9t$@k36Vq% z3iXg3*&+m+L;QD4rhO#k%b1>&#f+@n8oheU=8L*o1{TAfT4N^RWS+mdCG9S*&E<~E zvmvpPA-}yXX~qaK#T}U6osu3s{K0552a^X*R&)R)kEMhcbiJ+eWseNXxB4BPnx0`s zlF|fzMXF4+6V0PPn%QzOAd;PWP2YO%+MH(?Az#2g4{x-<4BPh|!Tjed z71Np@kck8ks2OIfkvIShE5FW$7jm-+Oaw}GYP;*hNG=n2ekVL(s-_qK2s|4#O5=J@ zzq@aXNC$gf%=0{eEHTbCoMyf^9H^b!lkv3xLgq&L%82ICG4Vz1F>XUd(OQA>RxSeA z`lNSDvsWgnZcaZ$YY23wcrvw|_tVTx%0n;#TuHjo;blHHl6NGA8c4qzAd{A$WKRyRi70PP@re=gl! zAVb}pFHtw-C(glsdwNVgX|hqK=et-E2)T8tD&dmX7gS%X7-BfUeJ?*ggC~orz7CnB zPv7j2T8*8FAo?3c6&44WIR7@p*cq#L2HSeD&%Zpk+?;LZF5R0sweAPt0{asUft8OQ zDaX2{@0I^lPEA*_OZxorkDVl)gI=<7=JXFM$@*I;K@_*qR8Aa{ooQ6nfLeuont$Kl=LV?Kq%Aa9|6|o$`raY$mDWth#~)kjo6T zQc$Oldk;uy7zgSTyAK=|p5Jz&t{b0Du&`2-xYM|Sj?@3kNKqskD?!gG7pIok$=kgV zrUfW~jLy%tWDJ1KdIXEz{Bbx2SJ0T)jE9OyQArw;%?8^)V>Pe%81 zK2CVD>lPHd=i5_C9}yK-m-m32>a;Vf(|P0QMIS61<;->&%iY-af^&wO7bre6(AfeF zGqSx}_5q;9Eaw|`^7&|`jy260>40#R&0)&r&Hy$&4%?f-v$@VzfNnDWdrVyS0t{I^ zU|nHwyRuEdO?rON?D)~;j>PxiV^{E{X@+?3^jdSOP?LVryb<6P?i}w(;LvRP0Zco9 z{<&lD0f0c3Qb?9F0*G?S-=XEO{q?AG{>{ICOG)w;wEh6x_PKQTW!6=m{Ri3n*Dt9s zhC3SK|DdJ+xIzD4A5H=zhMaMV4_fQLJ2kL$!s<=9oQ!{OR`rjHz+LGMi<@K1RDb;o z00T8T2g6umQu55HNJYK3h5q1&0kVqTJcE`YfU;rXZUs~rWMlupvj5;^ zr?{ypfuorJt2IFt!2sjwKPmOaP@Vn` zaK1}KJpAoCU4vdgZcvDm`G1T;5dO~!XSaRTEO}d;o=8O(r(FpRnFe5OH2;$y8?aRh zfdiVA$vn|vkjHyqar`5`hLPotw6~M(|65Rymq;!?6*|GqO#?F&MN-91OFVGYB`Y(? z+0vt*^b>2V&4^+F#A#b21<^W@WmJaB9Y^qVDTRS($4h4yFkqF@+86$l^Lrz)Ki^xTBv8(sJ%l@P4BuG+$mhzM zzvB&^0*v>Xixo7gPrPpOkSW=c2c!zdNoIhul=mMKnv$AHIeM;VPm zljGh~zfs%wXV!N{4zQ%>kB;dcf9GHUh|d3B(vx>l6|_=eE4-N;JNEj&>8TjR;r`sZ z4`>fOF27&`_p^cXp^mDDnDQn;jqyC6jEceGhcE`6lHDyd*%=P z?8`o$`Kb@=P~drJ{LWgI-I{qGQ;Z!kJ~?LVa3jU}!E4&iv_YSU^Bs$(4T=>&cNSU&s@4{0{21q?YVJ9xCaZ(Om$8H zb9*j-b}YRRpY$6!GMBT5ug3FBxv$Hf|1HSwx!Jw1vEO`UwV?iIXCj>EscAJogxbi@ ze0TqV7DWM*1m)pzbsVsBBHq$zqeGc-=QX%?=ee8me`E>}{mO$-0vz1HYE7n5)7H@b zdG{UJ0xCAojDgnF7S=ouslq{u#tP5NsD=en3@`=De612)m*P!QI7 z^d?0^8GbfD&|a^z@9|^(e@)xFIWlW@^lZZ6HZpBo40hDSbbSfZ9KJr=mnEV=A?;cj zF5&?DZF{I0;5$BFCF9~wkB%07vO8~`C34*TZsPV>@Ut#`h!l^psmOB~Ji5Lf?woDh zWU&l&L%+5mech?Xo{Qy9lsP7 zfEo2~P}wekG%Ag8pV7l_n(CXlonyHR;36U1jM>6seg}0r`84w#@q@xsRi47$P&S_A ziOhPS1ty%R_T+oI=w6~IXH<oi}eU%iSt@L0s>s?}GsmHeY_c_Ty@ zTZVzW0kOT{g{4ic^Wo|62YW&Je9GqUz>`aaf?vcrd?R0fRX?CU-Gcze_6)EazdUMA zxKqggiP1XX%s@mFjKCa!{62v>r-Xh@Z!_*Pj-&h(dgXhXyIuL(vWP)@s`DC~~BI$7wty>Z3Af`|n9lV&g5DnAmfr+|x!!Rr^wy5$Y;3Ni7e2NBF+h-hW~U`CL}_K+=lr z=RtP%+$-%uJjufJBYNSAl=5tOrNSW>gE*SB$f_!*b!PWar@qr*;0?D9$K5hRX-z%V zs72ZPL}q!h+peTI3Kxx!2bDVSU3pU=H9>n6>DqC|eQ%ZMvHB?LSK=v#tMTXXiRQJ63zbmJT}qtoiA^{qZj3_~h1E8)yeYbJ*0eS&p*Y&Os2Sr67HJ*ZQFLGAsuPFW*BG^! z^{004Ld_%ZR%iNRVnyR`U<~hL$FzP)hw?UyHCqI~_*j$b{w$}`?zQDTF{;5$HVQkN z)iEn9N|BBwo5vRq);(JSX`mGL_{o8fX*MQymcOV{Mc?j7(< z&FPjr&wa_A82l>KdEJK2iQQJ!)P7~fO!UfF)GM%l-ypDUxN>NYo~^IUI?!A8@K?XX zQB7gFk&W*B8NaK7FLv0^vM!d_%;4?EeJ2*H+t!@5DY~;c+XTZr<;mc}_s3$?o~!y$ z_mLRyp&GUx#e3nYz8x%FD7+t5$wstoY%LN3nq?WIZnj|}#(gw3hp_JbTBtL;1U|Qx zon??@VFQWNI?t0u7xp$>3RO zBfA@CdQfZ-ach?Ftpe%*@)m)*AXvhO$l?6ro-K>dcNKLrY|IgVHJtGZ8HqBzmn_gl zT91Kv%xrJ=i>oN)!Jxg#uy!q|eiR+t+Oyi7L^SC0s-?~{clV~i${%Vz=GqV?} zeRCEPr(~@$PlmvVl5uFiX@H)!uTm!WPr zYxeN7=AVw!==Kmk8d@4*px*q#5ut=_d4*?Y9%LmAOfImDG@03FRj9ZZx5q$p_}6W# z8;hg`(lHjcw#-SDB95$q%pT>Uf|7w(()HVK6`Sz_TY>=LvPzgqWEM;Ij&YnwH5;la zN<{vo_XpQGx3&C>=0Phnp_41szW#njj&U^&9SLnKrggLR_sWXW+E{sf_qO^@Cz;qv zscqRfplTXL1p`JR{BlI=7^{^DPqJAW6s-!|>XanEaJ|&F&YV-O_4AY-=g0L*+g6iE z%Bndc?Z+`AzVrf0k5F2Sr(lGVs97`e`MN_rOV7A60s)C9^iE-xk%ceKFt7_+v%Vgi za;BS9@Zcdu#t%NEo&2swteX9Aim1l1lX>~gDP^5^+S-Ea`3!<`r`-jPED%urwgc9U z*EIPA$QIGmS3Y=M^5cg{F$E>EJz=|5)K_|0R^)| zWrtrHrtP~*5zp-}Uaq9;lnoT|-i~B-7kY$6Rqqt`MA#h`v@E-jEs=IUP%~_s_~}Jt zW4D9x?)I}R{lL;lp;}^!_WBw*sk_k8ACVbK^b?pZV-M%#py9&oZ7k@E~Nr!cp5-xf(v`JEglYOyU$;& zF)Xc^qeWqAeL$%u7Q-=q{DG`t0VxDwelOfiMLJ}~`i?)O zu^v2eNF>M)c=)hd@G4O76h*dLK1*SkI-HXsMek-E{S5Wnpm1US>->qcIGCa6$(e(v zahs6aZED2C(TI-sYD_Z7;41ah%61Ymu(rG!N`ZJuW4THu#-09_ho-CJWUNeq`eD~( zjSnsLK_-S?Xp)FpJoJlG)5y-t&7FEu z$0AeCl3kPgz_*fEUnKwz%<4#0U#_bao2?o^zj8>Gv;0hCo($}X2X$pVGF%;p46Khs zTDoU$4J%h1O%(J#)YKr#y`4tw8{ zHvpME)mn(NIG7~Ld8kr`Nb$!=4Ed-F%6`NbP%12{Y1aDnx~|4T04W@sj24o2Wqi`G z8`tw=l1OKbL`BslVr6&3Syk25AgL31>nc{RaXql{TFpL~sZOUz-~a+NqPYxz-jywn zZgK>`zjW3t7B!){(0TRY&}|J~^Oo!CSzZb2QWJ;ZYz><` z+6r?F>hJW+Xk^lrtTrtF{b|lkgi=ECH99+27+{UPThQfCs2{?-n1%HaoGnU(`%a7zSi$|1Z*mKRz%LCWpHX554P>O=MDFMtXoeLLDD9f7?olrjXpW;y`p^ z)`ePh=`mNzFc{P`v>!&gjI}jj7E7o9bB;1=8L| zcNfr7Nax8{l;sN_#8v$)Gd_S(TXg;=kiWX6D4Fp|)q`e+HoNmii52Nd_NKi()~GTA zawD>(^?&ko_C!>woH(kmTTmqU14LwrV(4>~91|27Q_)GMfQ1}i#0b^IpaxM8`1{(k z3+|(ligYyjceS6ISzN(jhU?(y$Xq+ER^RB3EgA$`M!KtMYdGY+n6ITljT?24i&_u^ zb2y8}*kjRRPEzzwW@dbAQJ{!H%t=nABtJFQ4^9{OR}jZUiJF^}Zv3kdC6g>CuDpln zpH+|53IvK6*(AzY^7G3lV_OiLXa$Ug9`rL=wZ$~-)Ga{nnBA)g4oC&>O_W2G{VA|x9JKCul6q4-^PLNQ ztB6~%!@G3qTmlgNkfL4*jG#0Sg_d+q=9CtT0=tRG3mjriu`H`WMg!#Pi!1M`j`3BWmXj#?uhg%{X7jh}4_*vI z`iDvvE1h>62ly0{Qd~Uoy=?pD1*=9{c~pJ$2(Q$T42NaTwwC06ab3kva*}u!8v%dU1>2|!!pY7 zqJ#ML$fj1x1rrP6igvVc@K|jZZNBvV)_QQNs7~Y+jOgd zcKvvga|WAIGQ-@^7`P$4*q}MrJa9OKEv}?cb)TN^U&InYz%NNfJhsYhrjjiCl;n*T zk)APU%SU}nrnRTw(&hcW)4^*cqs2%N{EQ-_${A^o2fPKQIY;@F1m+L$r5U#H??R;C z!erbK=YqCgjC*;=Dp)^QtERnXznS84BO+v5#}lE@192A!!2{CCeg?r;82iCax@#~c zqs`CAD$6QCq~L`81qq9o;?i$K#Lap6v38@KK**VfXyx^9pi)@JWz)s7%vsVI1W|sj zDN3f3Ql+{Y+tGL2Xs2*`%Rpbj0k^_<`3c4dt(TWK+LznF9dGdq&Pr63jEL48p-k)J zX-P)=c&gk&0Zlt>d3|xR6{#cu#bI{sowFZ_r|f~2Hw_i{8*HX2;qgZO^ip_KzG%B) z1qEphos2wBM65Vz_*Mt{if0JSb6Q0WPZcB;Ybk^_CjHy%2p-IeP={_OZ!6I(`AX}g zQ76m$Jo07~9^OFmv?y(kFr=?vLduPc2cdAZd&?iZR8_s*1j<1vQp8Im>d+sBKT+_^ zlNSnR6eTD7+(8Y{@sXPKIa|Es7&}MBu!S~iZ%oh?l7E5Mm(yR^(S9Rm{JVEPcaDrB zqT8R_)Oky3cgh{j=tWxMyV^9b*Qv2p0iTmLq-OX!`@Iv0Vm^;81h$T_`r4Cd8r zo@ZFL30;LSjiU}Gw#epej}oWX-ZMXsjb6oyikE1$W-QJQ*A7MBoN*|j93F@Cys14m zZ_P1L=nsL<(zexzfx&rH>Cyt&>aq5R zGV5-Up7I`#j~{MXuR>V63# zRd|~8=$Y_`WnK#1k_UCkh;!pBfmU0MB>Uu4mjj9guq%?rik~ej)3Z^6J+iy&L6ow zZAIC09*(Y{5fe&`T;TF=``S7c+zc|?W|*BiY0J1|BHT+;=(r_ws1VhTvdJLmHUj8r zf;_ebE8WdbakXZUSaQfC44Qq+i&W6f`f$_DOhW2CML*|nH4-0By-0tp6xhv9)hs{s zLb)a4x0~wD$DN(>`~5Nr;Q+(!t~?|)M6P=)I(;rj9?$D&sZVrF zbl2QxD6Pe$B}8rau*~E_mb2hFbD2;$3@r}dzfPAz_EAQTIKeLxe7p`RgM+mxZ`b4$ z^+zRi!|Gk~n9VnbXyNJa#-6utjOYEUY|s1<3LUdMu0nDL>Z{0UOr?$4S{Ci|^>_K0DGoOu3x-E2z*aOO zN#{tx49)mbQ@FC45zn#3gk4#SUMWAv=4?j9{-*!+qS<`a&Z0EP9J8Mltd3EKYu>ga z-R*W(UNp430Y+8*#m2qB>zPql4hJS&$<-=~iFXrcidqlrPlir?#lQ3G0;ev0>3Dd= zO5bXJ_AIouM4lr|t9J@wi$KKG7MsldJ`^=0IRed;a^)Uf(anLM%Ra;gjsO~+iEE67{sbul;WMOizpW9DBA?krlIAyp)8~p~vhCJ2j zgn?h^*GaSW0!Aw`UAG;HXOY8EU0}1=7EVeQ5cS-0_+aT&FpMn{@RfHCsw;XHww)lA zIO`HbgRKgT0M|r=N96nF&kL3Lp7shSpJx_#ZZjMn#jKLb-{6p*DvLL)Q*Z1Ftz=7y z4TOPwBYU2q*0_YF4rM!Tc%?r=4uB&=&AF0!)`8hUJFcZC_7lAT}03=F9a(RrCBhJ^=$+>hh9-mbDB`|+q-Q8e3)*J3@@{ zvQFgi)1NuM36YJ{#@+ZQ=aMJ*Gycli*MC8>>+^L#N>}G%?R?Q^5&k-?uMDKO?rXsf zC~ua3YtC;4x-Q_pjyxN{)b+?^Sfg(l@C6AOMXwI=APst-2b_JhCsa{etoT(NCLSo` zl{I2saf<2udm;@14{#aP&DYMIHQb!rKu`Vot5DtOUPmK1#Cp~t)xQnyVe<;Key>btLH6d9zxCv0wFB;B}`zuG&R$dH&~D|8byl-P?nAC3$o zw2voX=Fb+Pprgmob`wwUV>z6mw`-V_q&|LE=&OeOL$HfkB>kZc5_-e1A<$&AaJK4D zd3?S=dD}1L2Rk7a&cNPi52;SPIwLc}Kq~m4L00>}q?~@S3J%e(G~sVn(#|InZFCce zsz2Bqer{+)h;q&iTZF|Qs1lX#LLdTj-h)Y9@>`+d&xgh4-@z( z$sZX8EAccRj~)4hz-I&;xZXQBmNlCnp4r^iA1ym*)h!z=EQn;Z31<8T?KnX(B6epT zu7Q@4w_OERt~GHbYd3Wb6)cIvLH1F<#zda31cG&7BuI^7-WyZtQyGr-Q{I<=;} zmErLPd~Smoy)jq^)(vW+;2EE0job={W4x}+cl;r4j8N9mQaod=myZ6W0=5uGMPx-_ zX^Q)p*ehf68L(4*Nr!laYqjyxT4w-2FadR#`y74gGggs)w?Q3pBA@RuM*GMrGLW(? zx#fT;5k`+(z~O`lK#O3PTLr9JL>)#6tdV}2r=yMT=jk$w?1?J<2L%GJerpj&JnH5; z$HAgNahIiJdLc0H5!ki_=_Hscv+kGiVnpv@ZzjWerGybgQSkMacN?ef9uM3RI#$UQ zNcOuY<>!?l3MtV+gb9l%Nz_ei`mOk(nhqAb@3V*=G#{@m9Ivp`Mlot>2U*d=x2~c` zMTyVOQ;={qjFT3D$CKfP7dm85>?%6;-uB)f<|Iy{211<19$;M8bv(%9u@3>Kjr2+-RSc6k;bD}leQFC1w}B(?&=4e!^z)gRZiL;_pdnh zQJdOH@q>U_c{NsQ>Bb#1Ra>9FZ-U+JW@TrmursXzj0viQgK0aqDY7RW*gdOla0$yBUgugOrqnxvwJT_vGZ4_6g2znhtyyY6g@XKdewRPJ`VA9HKt1 ztue@Xd0eJ%5yAio9;J{fa%T>u@zR9K0MVK=_Cp*^J=mz9~1<9Xt!8 z?gm^1hx-o5jAr4eu%ZRzO9a~I501+C%CFwP@66+skZ@*@W9uhw-p`9G1vlJQrA&|8 z7_gPv)=!_BYj4|e=-N5<@~X)>)7I)ay_SA0Elqa68`TdqfP*6&kNR4BVYnwSl-R3 zX2`ep_8ss3G1Z1=swtLC^H*6}?0MJ+ax!F7a|s=fqd!Z?Z9l|yWSVbBJ36kaoGykr zv!M89Y!s7tf4iJPp=9}ewX`0m1n*gW!`^TrGH?yC=%S(fdG9H6C5N3NyYs{hw3wsh z0+=XX#rNP6=s`e}SXSJrX`R`Sv{&nS2A0AbW8@3s5-3w*3Tp^|YnM97reR zk>Ue~yPloG+h*5bTZB{hApR)wEiE3xHexLS7xyt%X{JcU#X<7)hCTcX^x$>PNv;DL zjw3iTOP1sQHtA$~q$5Fv9l@~J-b1uzIJ*~>{6*r;;VE`IM+HGknJ+m+exZv{TMb-g zHk-kUTzo=}NO0CDiV`0&R37CmtZ)2&GG>kd`>svK6=B0vq4cA_d#-mH_TH8 zQc`PrBjG9p1dpt`e}s-%By;JATLQmAJCX)S6+&Ts8#y6-CN7nhguP4^kaE-nPxCtV_y=)62v_ctGi^aDk(jw z=QnJT>$bP(6okJSA5{1HCe1tdQaDrx`^zX$_K#pZ0bL4MtK#Y9E_V#5pvM+Rz*~5OXbeEzIl_cmi-ABm_Jvv(A$6? zZ@QYAn&C%h-JHHPZih$h{U0;r%Zm!K3XYw`kk)f~p`oFv-VbG@|5{GDZOTuXp8?nC zM+$Z{x*vP`p}rZ&EnZ{Lx#}jrwI9MVLF89MCyC z2dH+IvrPm!NN}50QPp9$-^mVX-w|6#X4!3l3CB#<`Cy_xNSJC7t2xq(YHw&guOE^2tA^auk+7U;BEp0sc;ls{hL|}VXxepaR)4opt*x6<^sC4J zc6d2ql4QGBF{f4Z&EXYisn`7AL8;Bf&S@o3*BFnatf<1S;@g3Akxw}xXXO1vJA!M9 z?ayWpFKtZ1^Q=jxxj#P-!TaXR+OJhaeAjCtnMUuWc7*TeoCe);|$^Ge`k;brvl|1J=g!PFBEg6BH zlGt>s>~(uJulwlB)r_Z2KHWG{Ik7yrvKNO8AmWpeYmB(j*Wn{QG))=ZuV3SAE%r>6 zzHd}@KZZC!aw;mm&DG$TXzzTJiAZ@&P9WKx&er`s^q2PfzFoF<{~6_6W5is{xPS@o z_Gn74nAf+`-2`$uB5r|)Ssnf%LlT@bUl7l|+jg{BZ#%Svo!dkmOh1N+4j@hG)Wekf zXbe-5WsJFIdk00N%BVQCAK+nsf{JZwg&vHgRXi_#Gi~l_Q_lA6i^2iUEOB-75e*58 zQ9H!*CBh)~sNs!WmQjsX8lDWM{_oWJ5XGwFB`lM-Qn@;~TchEhK1mSYX!y!m0euxc zBh_Ykt6cnPqzrsGRqJ?>F*ndS4AP2NPWZBw^~h;Am^N|GZI7NX|5HxM<(B$}L$MSu zsWaaryyNvSl8x`?CF9n<^{(wG#_wwO%@Cd1B}^k4!ojXouNsw~{t)$Luc@y2+CwGo z^lZ~1qqQ;4kWZ2P!MRaQtJ_0=mFM>Nmky#M#rvz+&CsJ-ZX6zBKdeM#(P`~geLEzs zvCU}Bpz!1VGreeN3}@Wx6VoU(jYZ?FST(f^I{RPx7(EJsvWPqWc&qiqJrZ=zdM_su z&Jt#0zDMjkZxU`B)3?VQwcAf)$gsc{-8NF;J91A<59@@otFB*1V#b&rQa@mq!heF? z#qB(}3e{fJKx{hZGgH%?n_eWg9In3KKp2;#1=O*S@|6e^&mXbe5QoK}N|Y|B?HFdbh)jcOz%xbNS;Z>wGXORLC@k1E`Yf@0@;Fa;jt6Qby9 zV|aHU>)glRQ>?(UHwzUlr$fj_ekUXCxJ-)}=lS*#9c(}|TV~PBaMw^tx`{d%f|wt3;D)>>P zg6^Au6Wm^Ju*V}TiY&4$Ye9a8ran#I>xVU*8Q)Q0Jla;A$}(#bGJ6b7PwQNpjE^Rp zNn^p3X*fMzyXYezkq)X<7-}fpjju4WP?TlLlk`w6{{m4B5ABUY2amX_`+`ydu}y#OULZ?!-2c z!UFYM#MIlkVdgEe3*$;3O~%f?YXuDiM zY~!Q{axHNbF*=X;9NN=U(C=$Z_#vsoCTQ-rIxT2cGUaHhda=O(biT^-VR)whNE%jo6Sac8 zHlqP4eMZ(2Y1rVvN8Emv=|w5|C>h^IeLvc^8CW!TZ)%}Uxr}7jqbB@RxWrfp@#zbZ z!_j=vgSXkH`{{DMMtSn~Ge`4<^iT*luAO`|LU$Opemr^2+SpU*s;cWC< zEWgRvj}o$6!0-Uw86IQAe0(Nj{r%VuG=?VCvPG`lKb&tUX{dvG{bI*r@B3WLn{Q5y zb3KRFIMSWPzwPA3*v-QA;kuSn)z|Ec6GpIukOhOJ5sk^$Wp={>(#Yjw*SHHG-R=t^ zc-a>hQx2|65U1zayT>rE@Z;vE4IgVwx-AEY!W*(|Mx+z8DI$1e`mb;B_1ZSkh4Cdq zia$F}Uq;_cp`A<~jLg_LytLlDXv&5=M(KK#81`r9Fi{IMzKklDmHVV%G&s|HR4&F$ zE-tGFe=5L#8MD5js5irWZ%=l*#RJb-lji<+QbZJOXRcpw&G`h;K3B;uO186RbXBQM zSC57D+ZEi7t$F5qVZDk6a2LNzd;Wg#bXe59Fbarg%Go&;nSXXPUk!r`5A8dU$sm8= z-!$)==9R8>oD;_sE+egmT)G)w`5~BobRVyp{W0kPonNRcWcbQgj?b3M+J|(Gg zO~ErV3atiPx|B$ileNAqEG@U@{+cg@iDwY;N_WckIMFoy1mDgQLFYs;4@HS^l7IzX z^~S)5i8-@K<=B@PQFX}|YP(6d?f3E#4x(pxnmkUHsV0~N=wu55pTdKoi6V|@YT|S9 z8kzSYBC?F{_Qu00XRy5cSo0NAyBl@oT_}V;Ax2QVFU<}p&Dmzc<{!)YTwsX#IP=Rz z2L1N_h5zgqt(h}M{)GP4s#hEN`D*q{!fQWwZ0UZJhppnz0;v6pFAoPNJp||yLc_{d z-%XaxKkgBT(@x1euLXabs;g;$UnpFJ)G`<&Wd|0%^?UjXM_*nxvWh>E@KOD=x_o;J zkJ|6;k_Aqh3JSPwYU2`H1UO$#xcaXj!vw#p_;$RxJ)@%+zkO`Dhih^;`233lPDT2q z^9HBQkoBm4jq6axe5&Ap65P}~`a5snd%(06dJ>gkAu1kFh@5=@@2IZ?=Sh`sO?)Up zj`AUtmXeF9R*?^T7O4+t{PO|rr-ha;4k^-O)QQf~HVqCUd`(Hvrfo{;Se(aZasZT^ zT>Xve4CUq)_qA(%1`YF-56TH;x8Y+}+glIv7zRa?Gk)PseX#)mrL+R)VpWV+su`>{4!prw^kLZ)yiU5gt?Z<8^y@}5@HXK;K1oFlh29!uT-rg>f>*xB3s2$702Rp6Vn5dW1PF&q2ZXC^q5Z;3Mg+{_wo(-tK6C zOMIhP_ln^OjC+6wv=wAzgbEk!o<^*!h8@hbbi&Qp4PJlc3y-Iw zJUxn{c^VR~-}q}P^>sX7;C^A_8`1B9y~F!sfxaB#AaH*@(#*CBgJ5>L@id0vaqs8uKVUwm_C9;9z1LdLdiFkRXRSF0bs=ytE_e|OHB|U} z^0Y+pxJrB=dXE*ULUj?fp~+jL!Lqq6(s->GT;WWxqd@d5%n9G#;<5pw;b>CIkp2N8 z6D)agM)Js*_p@IHxhHMp+(8FNPiy@uGdZ0}n0J09e%9hSx(paFG2K~$sTP?SKFuY1 zmjR`?Ow)|uiIXHe5DIqK7E|2sM9#H)zCPGEWA{t$D7N<#@4Y23^$n4kdOxazEDKE+ z?AN~Q@l7Xdjy;+`Q*K%Bc`bMB#kNkZHq>CQd$M)6Ypj8TEFwm4h0~Rv&~~y3>z~N3 zc7S-ew0_k2Urz;^NpoCYVIEltzsW0bJUhsN*dZWX>FdoAuLl>|d09~^AL9cbhMIc6_|a)M z{HV`c9uCtX<#{&U&!QcTJE&) z+7FUmPPeokS>Z=Ie$Tj=?rTD{ZDHG*Mk5)B{k^M^tQR6K7-3N{hkcWM;Yo|%xQR@( z0`JR3Q=aYnt;=$de_cEhE1DIdm}2VG;}4etMX!gk0iB@FkH-Y@UL##zFZp6O)a}>H zc4a>?(QM2Ij27V}%mPKztI`+*HyjBSoM4EYoh6lRjSWZ-``OL?LBjK5FUPHTv~A-O6~guxJ0SU)M+Czxp|D2) z&Mz1Jsi4vAr{DV=$eUooPPuSE;%2Q&y9Y|b1MLwHruS&iU98FtSUCx_oC_3W{g58{cG}F~*;NnF> zb)aUnw;#xbihG}~)qbY`>WkELr3v?FX4dQQBfmmEPq%y3>28%&uw3UNT}+eG1qm|) zZ)|kx;R;LsZT}L0qAMHkI^sd*Er)GSDB-`31{-%o6kPfUCcZ9xw9`m{+k85jc^Sur z&feoPXJ4yh-q0M{(eUYTP$fJxbd0>YtRdv_{T|R-)wV~parzET$zIf~N4<5?>;o0f zt5XwXqI8a;omt<(6R!oIn)NIJ!P=E!WM9=_18z(o6j^o$|d9Ny>G0~XAB@No6h4xH_kiy zmuqB#N~uxYJ?iOHL+msTE$j5j36yy!(SlzU(KBVqB4R5G9xJ7t3mlzBSRZgLK>X#p zwiXhvlIq&-EKkz0<+fpA#Ze4d<9jm@@~M7>=YE`|ye8sWi8I zsq`x9QickAfgE=uBSPaC}P}&|DcQB&XhX2KjQiZnu2IBVw|fo?ybMu$a0n zTdJl^#vW~gGc5Qa@>mz7R8N0_coR$W{+2zrojY2Y?A)36 zMg0A56Kj@}!}g>09zJaN%qOLoZZub#>d%N}In_|}=xL{s*)Mza$UJ)M74sZb4N7lz zXF57Y&O{0E!LP~tlDQNEs~!qOeIzm&29GhjfFJ-~VPjk*swXV`0dq(0%8@}+gl|&y zTQ#qjUaa(TH8mq*4mv1+J|GU;k^KaFJYj>8ZH=rgw|+r#%G>+)eotp*gz2_BfvwTP z_G^_PUgAAXkW?`YZ*Ti2er=lZX6#tZRZq_`G(%co59)K-roJQi76o0=Xfst@DMt9Y z?rcnF`NJzga)n~G49mpUNRjk-mTw_DoZE3PV}ybGvgG#$Z`o+q;90i`D}BRCUpT_P z3oKPuY8CFpw^==^$KXvztw$SniL#fn0Uw3N#Lzw!&dkjfK89jDRAwXgn0T~W`HLf=HhVY_B-Apc?`qG^2Ia&In6U8-K9%Px=K%A6oa_FQ0uenVT6?a>*~huLKyd`(1^y$m%F% zDX4uJ?9`tMl)IoEt+QuI&Cux0=TrpD4`r?*c0LH7oq*{JYwMx)3;120V9|0i`iqzs zu5Vs`0dd}sTIs&E&O(Y6>)Z)}gnV3M;`4G~scnh}IVHHtHburyFrHr3&jXMjfRq!Q zHbxtVz^YuKM#j_5Xz;UT14>=E&LEzz*T=B}2E&xr{VVeq>5PlV2ZfFZ{kF5loJ#>v zR>-+)7X|6ibXzS#!wa_YO8!ccd*{H!&h&hFyYPca`(qZ<)v&>1TXb!|v!G{85#ZKN zAgB$oGcZ7wil?--da8p7Mc5)UM38N5P9h@WPIHzqx`#(?*|Sd?+~hvHmytg|sZTCP z72qSsmw-gGX(Sc!CY>;O`^fLTjci=Dx;V9$V|_9^aZXr)#;Dzz7VmGj>ad-u<9dz~ z=XLtUJ(35qHuOt-y;2}b;N-dR^N3Gb$VJ|lql3>~AFXu*!V`%bq(DavYb?Y}I)+Lu zTwQLY{3J&9t2QAA*`o;_we|7~Go-9(U}Ix%LaNY(yjJ`LMx2$kY=2~Zx44g= z%LoscjZZ-OjIr;E-VPLu9l8G!73_gM6b@aJ(7&P(m!D;#+#pt(IXSnY*}SJrB5#b z!d8V#{*WrW^|zc6|X3X+^jShT% z4x{JHgkC?%%l|FT)j$UunH*Sa#++J?7a#uAoC!rvxt#pP>OviLx1xPHLw%DW&Fzt>yF$N_mShgExXpBI=d~6{2T=S8D<)+-F+9^NKudXk~6) zq-5n%LP&M^KvjiB_t-Utt@1T}m433E zRoseKc&!`vxRJi>Xr)5B}a3iaSBYe!<9@@G7l?S4P_c26+d+J}|&LYzQg zJigI`D*to{ z5Ls!`f>5JxTT+|}eRSZei*AaE4SSk<)$}=7^sOLoOZXOE6-AK|UHI~q_-d9)Hp2MW z!uvX{QKV21lf_Vw17-L+DV%VQctb8wU-|}@vab5gr$g|~TRZQ*#jIu7K645edy8WI zBAadyT-d(OkM#_|?{EB&h}BMrz+?rPXYk7J1wWT3OEHhRlbSI;N}698NEb$~xL zVry3!qzVhBHuzXYZXu)YvGx8174-^V0k7GgVAljdih-Q5f&`VE0{@Zp~X)$apEFx*JMH~mkM zZvGqAUx)VBWviK=EK;HI8-qKhh4D?}fFB*zkOsoEKZz?K+Y@HJn;|>&)SVbO_Ei`7 z;RJ0~jv8=P0x}N&Tm_^qXOCjGWnkovXCxToomH3aImqs}$9pz5WqJJiZBft4C_z(t zP#}dt{?Z1+*!$dwnW8Hy$sNe;jamMaC%bOcO5`0d&CddD#x{^B*MSZ4mw26>jEl0d zH`6$0_|(MP4@3}#7k(?^c2HXz*U8|s+=x(nOB)t_4HY?~FYQ1c(KYqI^aS5CqJ@^; zY5@2Ru6Xm>$_weLP6{lpwRKDWR!XP&_dQbN&FIU6_A$V z*$7x86>FMcB#h8-F;T;|xJoPcqKaA1tx!6=U29k;j_c!A|6jcQEOZ3t8 zv#=&ItdPtSK4^2>2(5VTD*e~`z8CtFV}goFA0?6seJ9YKmocvI<)_vI$ynoiBX#T< zRl7x4LowjG;EE3Tbc0((h0cB34df)8&`-wR>vZ?^&>3h4aiUlvagzCvg5!hQ8le}` zlu-TghLor|hUE)?r`Z#Rh7%M!QcqO%$4!w}&Dfi>{O`sF8hJGrd#+xK>mQ)FKc;BU z^ToD=oZ-$hr7BRFT9z&wYg?TbQM^E4(NnWiQYXJtyM&huwKOGH2M?RBtnQx5 z<+kn1UBZ%A434P}V(@aO=Kqc#LTgG9?aC&5wKw;`Utqs zyWpXsA3w{s#;tFx`{>wyuiE6MZ6o+oN@!oQX_>|OrA!f{t4EydA()oZ-rurYf zD-T7Q5*-P!331yKxfV#|#+d zWtRK+wSIz8Jvaqy-f0;}%oLJ7FCdB;y#E1C&z@4oCFheQ!%wZ7(&FUA+s7ZCZjcvk zr-)nwz$G+~q_x*QGRy{2IuN+#NuTU9Y`1%kF@LnKV4@>4pAn7oM2?l|8y$|a9GtXR zls28D#+7Bu%{oXvwvbsHqNr#R|4y$|NB8epCzifE-kUO`B+^A;g~+;VL`irFHs(@U z>mw(7SA($>u6FmA{avIPG|bNK*0$tY>m~8tRr#bQi$C&Uy{4_&UI+lkMD6&C@$G;3 z`u0pk@MGD`^^bIIPW3!>K44cWdDZFWhE}#(>;QMoI^43Z(x`rh-u=|s`L5<2oL>(W zzBJYX)WiT>YeFW0?;^7NtvyEce%ou42(WZ&%iUIl#Eoqsunk2v4 zq+?GYWB*O@a>yyx(^qy2w`|O= zS_=+6!y1#OUQpKvV$5JqbpCYlVcQdi)6317f4BfUZa1Tb8WKdb%&e(BDN5UNVZE3V zAn`ChV7}pQcmqGkz0JP>_UKPpNk21M!zn?mF&R708C4p{>I11?ML^s`M_W=Nab{bc z36sYi3sW#oz7@#R?~WRd${)}oyki6fOum^;R4J#N?f~eEK**lESzPSubTbCSjIh_n zqleM_o$f5Jjkd0@7*LpnrMt3&p#tv`pU<;~Uy(BooECc#{Lm-Ndl z*;HD{1D8r5ehT-8xXaMs_pV$-i!IA(e#R!&`mFxwU?*6@sDi?2TeUX4)*hP>g_|M7 z`Ie}3Q=x6xq67T&;Z7pqk}d>%gBOrGZo7bX0JP7{G1~q$K0-*A<(tj4GcJ`*8HL8h zagG#*zk=y^*FK$|9d{9vQyxgStgdDpX=;uH7%f~mpMOp{nGv6-Sm5DzXZDjmcu?%QPDPb~i!&d3{H30Bh>vy~Ce6hb z60)?q8g!{qTcPrlg@U3T{gfK{J)tB^wq*M`mD2i$W104`DbckYQ`|Lx?16Xq$nIf~ zfq=?4bEZ{B42!X3n(SCozH+bzHeiODntktC&c}U0hZ*$=0O|9Tq$C1>i3u``IYYne zXxO^V>*gV%J;}_S$nZ3}?F+>k=zUKo4-D1Wc?k}Li2hXX;cAta($)>9_85(cm2Ky4 zqzND885=cf{dhFoV~6Q7uWo{5^5YGGcNhVc8dr0)HX=H2eqHgD@CCT9lj}IihycOU zFNCtU%n4?fciGXwDRk|PxO!YX`f#HZg_r&;Hau&A)%};S4X8jV^+B!3Clb*~yZgtx z67LYJ;F6rN$yC z1+98`a8Pij-#r&(#q*&!!Nt9^5Gaslj_-F&EGtAX?&&uw;}W!^WNuNQB~?Id>@jb|RLS z zpT(y+R)^OeG1_|r7X+45I7}eK${ofW7*}oebq5#OpElnFDZ!JD|(kXc?hHr7)wRo>3gOUd$xcNfAU5{MO*Y4 zgfLA=N#i5tyE^rZ=zHRwVtVt@$nBt@Xe2xVk9Zou@h{jMrT4sUTz}E*IfbjR__N@< zBmjZSHQL^S^_*5g$)aq;$m#l#ncI2=QDV;`sj-Se0#YS!f6Q?0*06H;%dT5V?9+s;%(Gc|mconcG9T1KYL(?i?d3g|SY0rvhg_*#Tr=lulUlga&r3*Gho! z(sX(qms!rm*tf(r*rBI#8a(HG@YzvGT}#M|cA%8( z-j7Tkkrk6IJ!5r1O~%?Uwy!`31_NmUp~hN`*J?i7K?|}ODkqjx#sAlU zN8AQFZKY&uk}us=Af^)L5Dj?!p?`su>j7W=ZbLd%pCeQf3c;WO|v zaCzC8)yTWwb=zrVMDDmK{M`f-9V1=m1vm>7H}>@DFD8p=;rAju2opfm>8irQE!9vb zHUXZ}PA|q$jnc|}eCYlmw}M=c)Ik*TR z`T(PNk4YKOyD^#vlG`C$4<&UOGiFA=%Sqif(lW9#V!H;IU%=TI_=>#wS{yMZ2F3}D z=mFLqp!U+XZ-F}jb2TsA0$St1hUD?sHXi4C7Dsbhk=iguNHe(~df=m<+L?j@ zRN*4|pctl-5Gswf1x`}{kckrz3zfOt>!t;hbzX&P4FqYRF#uzCE$w0a2%Gf*;`8Ky zbe$%|<+T`;Wc055a937{)AFJ#R(aKkjxa2KkDjYRShnCgTRk&keO^74Dk-qAV!NK_%~h2nQsQfUN<34ElqWOz-AF0?MSO6h z!$Vb4wtbJsDwct9%H~;5%bSoSX7ENCplYs^Jg^~c?^%|5(Oq^mq!SdRbuJ}trDyo+ z(zC4SsLdkHEQ;`FUdj!+RBbyEiXd*dXN-&`P31WNgSGoMfu0y`3ZsBOUMEb;dt7Or z^_h|dbKy+}3g9NtFZ|5MU0F38D&5r-a$+_in~{1W{T+DJN(n$;Zo?jlG){p>Bxx|`-89E~W3mDO%00-LN_c5d*?Elbw>(YQE{>q#&-n$N(`ORuLm$w-BizacPN$fMi`QkdJ*X zZ{M?=7`b*>8QQ6|l#}&enq4ph4|TcWgEPP7r3bucz+g8A&&q^+kc-UYFCIkyg5f8C z@*__-b+|Sq01c~17`1_yxd}yM!>6psE-2_@oL6L`;jIs7mNL3ne2uAhE=IU>ut?k~ zp6#hzYGUVQ_sPhCGktF()!qUYHB{wCt19m44$ew$9*LZi)U>VV&O&`yc_H^5PAg!x z2^}kxd@Xw44*&H`^n#U54P zB>J^bQDxqXva^bw3fScuIg+M3aNr3mm4s5d;%shovA&36z(!jakEFQG4&W$*z$ut zI8%pCfP%bT4}6>ci|(b(hoIU4vMNcu=x}LOfMQ%CPN$_@B{4_FpBXYe8KS@x(9J@( z;t@SJB>|*T#e>d9>rHXkfyHoj+l3X3CokVaaZha+ZZT;aSdogqvSSV^IBo>4kg>89oOV39QJK0#+(G;*L?JS9te>4E%3=A=KR8&NarM9id!j2dvF34%gF=DJfiH+ zlUO|0V*_4q*e`VZz-JysPl^}&6#(&dsBncKRT6lSUfo=2eR$Eez_@X39Nc9CKvj3x zJ8l3$9CBgEig)(lj@@hRUb|Cm z^~%q#lmk~)LoJI;AU&EukJBDc%HR#_V$fWowsD_!xO9gRdy3_3HQzuEWR$m>5AJ5O zxHxYMvf~2XG&@+b)^IOtJ8Di`ym#EWJEJ;WjuMdF1oL^akoF{T4>-F^^!~s|T}Jb1J)qPBz6iXEHkik@S6{v3I^hniDcFusY`uRvmNvx0ov_~t zPm6p#%kla3@Xq*1)72%j8?jl9-jg4;A@q0E z9eg%(=F|e}zcj>_2(Y64Rqo_}mc+$hQT}NCrTiuBdx>1bLtsq>WFa!C4)c0s?haA1 zEPt#>8#BJT^@`IPPYQ^>P;BY}jKcQaG9orq+SJ4IeM}F5ek%&VC%;Zl5_#7e)Q$Jw z_f>R}Z^GR-f;3R2>KR|45le7M@G2E;y z(X5Nd-ifz)<-Fj^`$;%%Tt|EkK)+Yy>Tr6(2(TZiB6}_bT*~@qMUOzLw z7h&T(VII~HVMEjrD~8@Jti};fv2Dt#mFPA=!UkZ-pfxH8W;z|$Lr(((ZJ<65{LGfx zLqI+_+6t7)GeQ{FOJ}_^Yc^#1smT%vsPcr=ZhniSdJS0Pu&~^#Nwy^2h{!7A znIDW01T5n9f`lgJAd~e>sv{5b)DQFJo^ZIKl}#IXfr2Ie86yFfrNsm2`O5m5k%(O$ zCC`vX;@!@^zBFJ)JHUL{zGBVXP_8GQ%l-fZgU__@8EnN-fmGynPz_8>9SC#)eA&!~ z*~tPKt<>>RSZt3AP?J%0j;Pl_pMC-c8t_JWb5-TSvs=IZ7$ql%YI6cAdJ}8>d0Or~G3#fFo*rKS^Z#>Y^SVM^tAMD| zzrDF4ByOZ^;ICAst&wk#pj57ZyHF!9=?D|I082I-`Nt*I}arDUa=11yAGGU z{nz7SgI~23a^A}u_#Y=O6x#~NyiNaaCszNL6R&|gtnvPDCo=p0XB~3D9om`yvsD+x zGl9ba2%n~=CZONUb&cvUG#lM6E%iM;Jy%v&GbUQxpk58G5j?v$%enr_WAHW$3(JGU zqo{##5X18l&uqYcC0|}z4u5@we!AD?xAm!l(P^;xa>D-wBje&=;}x*<>#vsx(4MsI zo6jkLD(jaA;ZfAF&Vwu4az}-UJCG_2ZM)J69j33oppBv?K|sma;OE!Q(Z5Rpa7wTTE<3rxQv=*Z@l(g!w^7+r ze4?oTB_F_hr=p2xx%K{idbhA0qiQkneaUa&7{^|-j*;h6fV=VU^1{dxyZ=SnqwND@ zkW$UpSm3DuF2PxO=NCmGwd-G({iY!+E?4iPEKKe%0T%$*7FQS?uiR$Bo|mv?2;@)jP> zPKud=3{?_En?Oqz-(Rk*t~{9+4F4hd>U!1FzX~Vi`>23>_T&a+pSz>$|GXmPHV7YmQ-o|STxJunZu&aT8_hnI?Ue>vN z)Be(~HwAa1xYdWcBpzWJoEAU7GdP+7ds@!`UmGwr_Vf*Kxs^GZa}l~89ASQrr4D`f zL-Oq6{MYKL=?;~Wai+JKU{DZRavi+S=T0a>;T_tN>6O(r!p3y2_2-0KhiobPr(X|LeI-zmFHfy;-~7mM z7c}_q`YVKL-$djI_65aBM*;LF$^|s_gExH zG3)o3`~CK@#HX$9^FK&f?3!&qRXiV|v`EeWEv9rlAo(}zkEGje_5Wm7^heg_8Gi#d zK8D@Kva_!)u+~4h-tL-D5uj=XlC@M%DDLg=*~|)jas5F0FASYWzjo_jFgFith7Ai2bMx!S&hLRHtEy`yd*W zt1B21b@6;%pFI)f>}1jvM00a<5e?FN{ZZ4w(gZ{jSS9%=05BMT0km|1y(SKfcMun< zn>lZ#1PGrWWh9Q^71_=*tMaO=Go=@UZvy{VXL*XdKtJS0|11ZR+@fuT(zeU0Tlz!v zES#0s6oS~9q2;H=-ry1LuZ4QVig4Ap+30ZCXSvmkV$7%4f?1`romDOyG*nyjR51lG zCy8q`W7Flg;j`k42`SknlY(L8q0TPF(c@uw8ZeA4hGR?sXSy3mhxfOc)AXP+Z+oxNDZq7x>Z(|rHujb7)`vynrQycA^`1K?3wHp^L$Okcyh zN9s)-@s|;e9_%#DmTN|8EuIk-_0pCnaqy^Y?<|vx=bgAbp8G!%h_sW|KvJ{^I~-m5 zEY0U=Bf-sLuBSYZsHpDLmftvY>Qy*=Ho%1d6BHK=&Pe!rlp zuqx(b$|hBk8IUBF9V2f7KtabG99IKRjpS8VFeE^Nqyqtk&&|i3K-lM0`t_)F`c|ph zi-2(y0j4#rGM}F8+%xir^f#kPa8;N*<80r7zvBHS5B2ajZ~KPXW8xPsi}|fr1K-po za=ugYcTa8P=kWi~AtxM5>!V1uX8cqY|58wv(z(3t~JibKy=+LY{|#T_vvMaIjDm?Q|~$x$Lv!W!sX{CO3Fla#?i9;RRU# zT+-e315mTFAalw&BL%WLJqmclVIX*Od21AD->>xs0l(}+UJbk@VGUzZZ|>++!;K2m zS-;H5@(QWRih4$F;n?xK1!3tqd@3iL-1+)jIA+`E0#wnjJUs?Xq%CA1c$Ku58=oD) zx24(QA_PNYaUMGFs!;+5+m&S>=xp`DTKCg=*U^0pBvgGNb4WT<|Ca2NBw(g z8jzRr%|qo`O3mOLkrk(${K{tAxMbwsrYaFoj6O zw@m;^1{UkUmhp!3WqNe+ku$jq?vf5EUVZqq{IPix2c($UdDau}0c`C{KK6cf@37rO zDobR?3;=rSDGa8UZ6|%pO1;v(OT($4Um7mQAfnjI1 zUJVgNq?pDf?qW+|hl`A9e z#8ZE?6kZ&+x$72`cyJ)%v4S%}f%)u;9>%o>tJQZQwZL4e-<$g|+i&tZ3Y322NTGklxpoWGuOr&qL&*;#>=jZRFu3}p?w$QQ(;|Ev-d6^ELZ0`XkoB2grgoN60>}I=t}J274&0Oh z1ui@zIwMo(g(K2`XLu<;K-BOK1Tl*_ED|u)+%=%3-&)r0JG&?vTFfj*0vRj0M zAJ14Tm0MQASZXWnPpl&^gvX)p8D@}beyKn!6=&Y5B+eo67Y4nfx?aF5eyN)$UH~3Y zkuvwU6^JJWZzkrBRu8H@<03@ekJ7aD5d!;Hn+fy7L9E!D#5)!_paJjppKP102ZB7ID7R%ouPy$yC91ha?Az&kumv1~s>h^3_ zppNibcD1XdtlF255Ae;wh40nHDFc1TKKeT53*8&vcf4rj%t>USg+CZ__t9raP~E`~ z203XCiDC0GUP%vj^0Ke8kDU65?EQJOR8!vSTp$RVj@BNw=r}PP&ib zN-TdYL*jWJ{Thbmw%RLYUOsW+xe`uTM+R(O^DyMJReiRaV-zYn5DwE%#UyVCwwsJ% zIY@a0%*=!GhMpWa56ED}z`A6vpG|CLkE`~`#Iit!)o5zazUOpnRJ)c05^gitZl*ya zOY`&9_aQ@r5-3Qs3ju3he6CL+ddY2wsA5m&S$*jT96MzXaax^p`fC7}qtqJJw#?DD z^}9!q&@d5(7?yhF0@a=gx z$y^|Z=IM-ypA*Y1gVmpclM65bkOU3bJu{>Dn>3%X92TE35M~U%aU=g&6MQh|J$mhZ zxn^}XF~NKKUD6c@+1e^x#?7ClpVaZZ@rgD=^*n!qyL1|H@RgOAz<+&z5cPN`PCSWO z#$}WJ-CH}ic8x=4muIBKnNN^s)WnW1!7?QB&Z$)=Cew2-?kSxOf_#BMB^7vHX- zrjPyCZ;0byKE) z!62cRUUKJRBZ&1f;_{68;d*v!0~0|~mM$mB;)#^7;icnb$p?8GZUZ>&;QD-;-Q0rF zbj#(Az?~;uol^GMF$=^X{p~`L@-bh=EU8Sg_OyDYC<3AtH}=Y$OCT)ho?cR&m(bp= z*%AC9dZub;i$WU{f8VE(Ibp}_tqw7n_{@ZV72iM%z8L=#UmuYd3}g(G=a=WqF{iRQ zeIL=!TGd;q2Fe(CE>HYo)B?};CBR)e#+jr;FmFoR>}_~Y)4`XDx_I~j1@Erg?%1yx znX0ox({-hKyh00`WY3v#XR(yCU2xM%!=u|Tm+5k1>D-&(l!PT_Lzi=xgG(y6Ym?4V^CyJg1^LArOJL!*yx zeMX&&PY1sHrap%!!*<6LGF9is*XO4<_*K{QgWk`0g~1E7y2}jntSgddT5T=kavqmZ zuG|d9Ojm~n*p&N}s{W50rj(zxNk>O_s=q&r-Gwrpk)dpedOUGJu(m=S8SlHhhppFq z$bdgBJfe=Y2wFapwTDeV#DgwytiE=*j&m_p){-_utfy=+o?WGCTfAR}`DaDHnsT!< z;r{{AmidC3p#?Qxa5}TGrx~_2T72i6xQteJ7-LSub`ebnt}L@}>e}KYNuC|DNVGjt z0_0b^>Ss%H^r<*++g;xI6ooQRE+_J!fXCnUSMF8LhAWk5}FV%0(3t229# zn9V`W1PY(V;Qqx-gf-M@qSQCKMXK(j&o-X1^slTX)UQW)*@@aKI-lW zY97)>&a&jpz-9^cG*btH2EkcK3-xHGOBN(7N4vl&(!;sJnJ*%{+SNOnVZo(UYE#>% zmKmqpWPIJezUC%$9?L~TEUeVqa8o-C9lK9`cjTf_Y>nTwSqU(Mw$%5e)*>2K#l5(rbsA^wahuycfe8QbpH?!jXli|Kfxx;Ee~ zm*BpGLpa5W9LQg#q*`bAivG9C-(e;e7;W>WpIZaL%#pzTg?c;^Hl6d!zDNU+A%r<| zS~Ct=OzSD%1^HYM+$Y!xw&!JA8g|R*`(VAzJSd8SW7wABw@$@|*Q5KFI(tsmyzrzr zjMC>znwmPYzuFs5AdNt#EyIvW{>!4U1{%9M~zXZU7d<1te| z7SiZs(WWh^Tg}OAFtqlQ++dS1^PCE=H5Z=3*5kXAd>;xv8lPFUNf!<^YiwCcVT-PS zSi*TSi-kHW^2GQ6%1+V8YqY4h$vkY#bFYOX;`Rd|ThJJlC(Cz9;DLUjgD1PzlwiX= z{P>9k>^&Q){zgg0LcDOvO2O+cA+6`sqo*rz*tQ9P5+%7@R3y2035AEupC@~#G8>LP znPa#KO1|!9xSQQ_fY^SC%sZ@+T^YgX-*Jw1SEQ)&VkeKzf&X}1p0^SiWP~@)VEu$6 zQ9ftXKIS7L)ETPt4&A)sbROh0x9Uanedgg%*@zaklcm&Dlfxspg1008PjiDO0tBUo z6hEnoXorayk&{_!r)R1HH(uCmnQLD&Y1^}T_NZQU);~|hcIV;`DEV#2Lnk*A z01qEU)3$AFwn{UU@t^)u%JWsV{4zRs+|OvYLn!HyMtSA2$}QRa4IXjh`;M#f5q`@V zRSZZ_dD|>c9#w~vaxw5Q_>SEoJZrSaxt4o(Dqi>(u!-&ZEpVLcNs3fi{57%?GP7@8 z*Hob@0al-{tqIm<)pB{S?OM3e<#@QljZDdB!O_Id>$CpVN zV9x(4=8BkMvVIs4(s7ritP-y3;-gBOlyG9eWeUag;n+i0ii9et3X@`HdDoZ49<}M} ze#LfBGbM>_kmQieYdbKx-i;3^W06%t#y51GT$*|_iwVhu6t1t+IU{eb*9&}-RhPfH z>gLaY;W0V-QQxEWm3-D!1hyO!x*sQ^Hi`v{tE-$jg5F7c2gK zTrq*?S#?vYwC`U=pZH+v&!71pg1NZ>_76~`AHA@iC^UXw?O2t;0{c7)G)cHPjv%wWWb?)ZO`__6EHn>GY{~bSH z8e3k{)0u6=v^6$9;5sMn1C~bhzJR}Qvs)M(>hoVp%2L1H$c~EYwvNl&+xE>T#3;HA z&x!i}z`6(o0;RjJkK-mcL@XHJ6M{~DC4$Ye_4uWo9dp4>K_rkQx5Cqp2X;2PLMb_S z#pBTPyx%CaFm;V_=eBw8HP>lWnUS$HAAh~Z>VecB|Fa5RW#du4Ix{eq@wFC9XPo-` z`v+j)G*mc)AGchZk*6WgZkP;eTVv;frG!p+wvZa*%Z8e!V}y$FHAWrFU#uEp2g;|f z-^jV+cj=l@Q7?OH!=#t0o?8mYh@62Tc?aLy%iE~&jj?hOw%l}Wi*{8rg4J~(x&xF? zWi22kyG_4k3#vD{2>TMa~X(BlQ0-t#9d$!*Y3+&*DCDT`rS%n<5|sTP&}mEn!*N1#zkNW@;CB5!(1j|e!53lRU-C;!)(q81v2 zbqmJ?0DlTN!fG74`G8N+zAW?avlpTtf%36?XVk^i^*m;Bbk&WOV@E_Gho`DUyKP_D zSY6S+g4~J(a3d$Mci)>^8k(yHC>R%RlA9&s8_|C+$%<^d?iU7=Z&u}750vCE-r{`* zrpK{Ietv#UY;fJvJm_jl5AvE2yY-0{d>M~F%vA5a@(swg0vZc6y6-!BT3%3w>V5vG zSGjEcoOe?|S4tv;9U~*{YU%!K=bbAfW| z-9-+{R&m~MtRAWqW9RSAH`I#1`8r*&9vm=iwIaNIH2(@30b#f&67{~IU7@I0=hav5 zP^}b{aUtGu8)ZP5tAGr1=osxyre;tWMwICrJg<8U3vX96UMwnC-lA8R-XhOYqj~9y zslWxPUZW%YMx}ml>%q%D3=!S70>{LH+lrQGOV=%J`G ztlVWS0D6-IRf)Tv7!_|w(kf+YWDTl>0S)M%ABafM5AiN@@}CBT##|~_$`|OEsmREY6|13u8BYGwC?*StM7j=wQ&I8ZK83B?)?BNYvRUGK@(9m^R z3~}K5Sy8GO^DizML&y9|KUc#)Ln3B$IApjYsx<;JCHKg1`+NtfoplWh1Kz|o>_z(x zQ=7%L)GjIpU9X*8y*3FaVL<<2VGRMiULHVpsCK`vWwBhe1+Kk19^ZZ-mEm*yHj^R| z3CRQ4o8w=6tsHPLCwt}bdzoBe>1p5e^H#Fki|xrY&q2Yr%hlwz0k!|sYr@WUoVwjD z?3bREcxiFXj(g#Iar(t^nWi7U#Bq3Z6lDrj%l08BYrrFN9An`reB&07d@c$+g--og z(k}mk1QG&4f;+)o zg9dkky99R)u7TiAa1HL>xNGC??(Wuj&mno9@0)w)t*M%-`^VJXRnUF9Pj5d<_xi0M zN7`J35k}@R!GC<20C`G>hYu%P_UHqc(`AGiho)D~pn6;$i{M+Gz&Q>y_paup&T^#2e>>eHU(-<8@xm;^E|z#HWThI99z8E{ z)xkn)DhoLBv4kbqAK}rfd0p5Bn>jxwGvc^5+}QM;uJ%v8<-h!_2JAugxHjJ;cdzHKIi2sQcU@O!_MhW{FzMQvJ z)Nj6MP5~V|#D~OQw|qXOvmtSOXRtHQTLC%gMR@@D#!G;SJLz3D97xRfGyQ@ZP%CR* zd==*9iT<)()2{1KJzIpcbV^mJo!p+A^!9-}@;lu))CGBKd3k+lJ2tvVlQ(u&U@o#* zA63t&0n4A^p}}<*{pCTb%&@u^bsccrm{TU~?h{Rhx-Fzfz4u900NTWAj5skqN;~85 ze5?++fhH+Tq%xmWXJ8s0X|(M&<6_U$tg`{Uc8F2=WY&FT7lZy_{uH|qy-|J*y0x5Z{C!?YGsGYCylgyYJZmES2N2q)Z`8xX%A~tH~E~%Wvdg z_n;zwX!B-aqz<#o$8n=2i5jw@<8zA{dc7+f53Bi(#!tcw237N{cmX(^!7~1!_IgXQ z7x<4L!NVM7p7r#X2>Hv18(S15Q%;sx?i{$!fB5V>n{M5s%qIs@9pg&qR+?ASI>jJS z-f#(K0Yn%eX%yP3ZF!v274_M#vH^}PJm(;?lL7S05a`#@a)X_3dPxg51pxA`14`DJ z&~%g3qcuN7G!lmw*F=pEQV6BGZ!ELdThqfal952*vw7rwAu7hbbqZzrXC#3-p!uIr zhKCBc*6!+XFvC?0>e!Fk$l=qyJSXbbRW$ip(~xgdMs-9V@0Oso1uWSwGrO1sBwlB- z5TDKUHu8K9{CY)y+$}39#&q>Q(oX1!=!xEN!3!@|89P3snX~t0^p?ZW)7dlq5P4SD zK7@W)kAk~@d$;?H+!C5QRt?=c^@V>U98B@?lKvx2sH+MV{&?2Fdsu-8`H8J>zlr7W zYnMx>x$m=*VWKJherNw>H_eG~!kU$At8lzwb&=&%8Zr9akK5?=Dg^WH91Ag=pBZPd+Ia9O<$R#)iu%W#Ks#|b0B>e`wws$eG_2|O-w`itss0%;6l#sc8_JSd@bhWFYLaa`m@hL8@+EclhTN-7 zWallF$rY;TMK{dp{ZhkNhrMMM+GQ4=e+4fW9xpdNF^b5uNkRHOwKH^>d1N`Zhwd?h zVAao#Eb}2Li*bupxPYaGMm$7k!Vj@mBw(N$ehW;2jW#8ycJO>Pyy4j5dJlZZ=e~33yD+g-j%{{}=Ww|6WiPFz^82jvePi9V z3P8sG=#~`Ab#TJmvPw>ST7}elC;EsA3_lCPj%ALC?L|zWN()dao7aT7kqj5#K`9R_ zl|+9iLa{+xUaq)$J*al;1|=tE70;_K8Vw{TK4Yq&{=Bj>AJkgG+?_^|{W5`;<%WrM z=|fFz)6v7hKO^4qmUS%GaGTfh({_Q!sIl|QGLiV`u#PHR?=8_9t4CFiR=bQ13gP=; zO8t=(pyp6Jx<&rrW3xRqTgb^?!ovAZZq|znmx$SQKNKUCJq+_Kk8?W9)U{+GAcjHu z?-OilM!%Oo7E5cNKLB)4R;^W0t!GacQ9V?AUfX?pcevx4DLa~AyldRurTvPd-O9_h>|Zu^f`%J;m}C&=Z`)ObH7=egtMD?41AMl?HPdhk$-l`qt177 zIyYOSMQ_5pnwq*j1_;b}S;*mL_=oj3w2Q}w@PcF)YxmYnDv%1}uB(a{QhDV%%x7EM zP7g~?{O2hUo>%7Js|uT0*Txl%H#3?VrRf*P_IS8zHN+*q*6}jS{GFRbRvko%RcwD*i0F1vxRw%o zLQskEmduZvx_13pP8LU!tCY@CWfL5!k1u3#NInTMDNr=@k1a*6+3#bBQf@_iE1;n{ zcCS=18yq&2oHAP%^%behKxn%DAk7!(fZE8Z286!o`8;U>0A*r94iVlSd25Y}-EjI{pHxLWd znTU;TA6vHAIK%}<&l;hJEM2S_aYb)v+)f*I#2$0G!KlBMFbZAIA6|D}?H8$I2VGxR zng~kja@nMBjT^QHi4Fq36M%~GyvlqlsI27**~bC)PaN z`tJI#bY2*vNVI%u+Cg8h&)>cqP~ zb0LEllsde#&0}CuU%Yb-`%c{UX9l?Xd^ebm67`P>DbmhxIc&M_9>qkISxd}*$+|Zwbpbi*tC_|`A%xguJg0iQ!%`5>(BUvU>MJ3y8L{gTCBFgg+BMq0I2g*srsvX0bI3WAf-PFm$j8YZWW2q=6)ul)b&k?rypvsZo{kaw;lHB5?P-VaMU$m6!I)udW@Xnh#+Mqm6QK> z1HdE#3jQhI%Q$WzYi7$$VoW~VW>&A?i5sr`Uiwa<%3v%qEb$AI zojl&_-jt?)Iu~dA4Y#s zi7bh_{VXT=(_16m3vBJm@1xVy$>-!4akA>S-9CQe0D_7l`a1XW;}_EbM`cs3%r`M_{iN4li_^6mMM!ern8?(unflKXiX^-kjDInpGWXW-;yP-(LzBf$ zr}uojO74PGBY7eyz8d)_rdgHP7`<>fe!o2A9jlVyP`tM+#c~8}9Y;O|Of?9t(Hg?% za)sAZ7MNCX`q3*Po)GCwL9549!IZ98NHg@E-&^sVZRgMsp-85!C)|bqMP>kqHj8JOD+q%T-LN%HW#&-%ERqpzL_HsjA?BNmi9Rx?b zhM=q@skhai*M8oHL8hkR`bRGpGf8u2keej)kKQA_Ivyj1`Q3zynukyN zwHdExjK5)#!_~z}Elh_0*~HJJ-PsZf^yyOrqq5V68t1JT(k$$|E*5((GM)d{SyuN* zv*WMh*rPc8883G6XI-ze0L`3eY~|+W=Cu33aW<|x&WH@!uTm1#*+4cH+0?lKZQGco z5^LON7=n23v`gsqwgvZl@5oA2j|8&%v2F zk2{E8To>B)9)fZM(W@|Q@U+8YUnaFCR{h%4cJEAF4ExTs76e;Y-kV0v=D0SdM5x2t zchzMV-*8L*9=tLt0N?MuA^-pbDK{hHxx*uaiOO-D!m`0-xUhuYmn0idbI%!Gil%3kWoX$lN6Jda8HdL!}sFyL1j*5$X!6 z)CJ~HgqPWU(-zGdw#Uz7F@o!p9V}-IN$nQ_ba>WZX+K;a&oSA+32lk(17_gUXM7i# z10HuMx-j!$&WMdxIePQ|NclZFZO~BNzMKqyn;@N;Yfr`sBPxEMAbcxm&Z_}8OS0fG zotaALdO9Am&~~#02WyY`w_EM*Q;O77SC5F<;MX7Yq`qx^qCW^D$#{;l-REBF?P1{$iKy2@%J;X3 zPXg(#5JRa`2TfTqd^5cCoPA)J483+-6Sz%2V$@XY4DqhbfRBi02GDh1bN4WP5Ks8^ z9(A}(BFUD^9_g;~hZkJ^;{D6 zobHQ+A&(M0E^|9tlY5PnCha=`eJ5U|ug{eUojZ$>x-YG29S&ff+@TCXnD4Vokpi)tu)H8gXrPOiY>~faZPC zwb9*;<~yK6_&61bXqniJ0i375S=Nr21JPw#9tL`z3jE>IqKbl_(yFsc8iK(?l%Ltm zrIrQu!Vrsi&~;r9_0~LvQQ!w8IM>*Daq%S92*3`JHTGmf&P#J+B_AN{5J&mrp`Wkg z8PW!H22Ik*HMn*;gpq->sjaaTSX5Z#B0G3=vI;@C+d^H&ixLLq|k&&$LW8BdOF)>!BSZj*iZsSa-Ok zgRYt`D_HZEkbVF6{J#9?<9x&n;miFggQB^(7gM2bO{P|-9@r3~@~XfI?d^(1Q^yI< z#o&B}oUnuIbYPzA?y|liNa&mj`*7)BLv1K)6HfFn|Ctub)u>C^ zA;LZ+q(= zeH|}>;mj3J2W?%{<8bj7JzvDS&1dE+o&NG~ix(6xplj^0tH>!4b=e*lCX5=# z&WibF?5>?{E&aZ0SSQz%NgIR~gyijwUq?DB-s?BV+Glhn^<6fPZgX^Dg_fBv%LnMy zK?;&UNN3r%PL8Ett1Ea~j;}*^k4tsUs$F@BhimS>4;)^!9Ejg8U*k=E>D6F0CWip< z`z=w6eQb_69dr#*k8z7b^Hch?*wF)Ju2y7>tuv!iSAgm2IQc4#^;a=$$|3S$ZN41X zzTHOgt<)}Lz-S=loRKcXGy1o-)<$Z736TD(6z1cO%NJYocsk7fzd}1g`3%nBmd-Q+ z*^|tlq#t4OmW-Oid!^$pe_{=~uN)PHCJk-C--y`G@sKs*x8DM#9tlP zg?&KWiq-gj*u=%^z!e_q>N}KHP!- zg7@GX$Z>e@%LG_WHaBa(qLLOigm(OZqw}BZb>ToIF20dTNgerXG=DW5IYy@~?&*yb zxLa<LfhZ-+f3l2#I8Kcf|m2LQW zyb@S7{lCa*tNB02`8HocI`ocIHDx(4%l z6-*0CVH$&e-;E-$7Q=p&80J1S7;(myJ!~Pd9?k&7vnlHZ-GV#m1F$MTekqSyePxV$ zTc0orFO|Y1zgRaEEMPqFmFSsh2EaniY3AX06z(+^7Se4SmAZv9Y9D z!+ez~7e+jjd`yG(`2pp&@{%-~H(Oux}d`*pb_4wQh{-+^pmrc?tk9PPk^d_>>1RK-eC%_WJ`$3fS_#S0EaHci$zS)H7E*#IGKWbLe z)q|A+9EvNUM|3(@xO>!)pbCHp&Q;1g+Yb`zZx}XMvabd<>Ej{g z=y4(>BpgAMi;bZ-GYAB=CP{1j0bS*5Z@ex^91hH(9#2^uJO=0a=6+! z&8nRRO+#YUn^5267-k<>#FuOJw0f}Y)G>!0kUEb?+?+;S?(~CXJ?9cJD@IdSM=xeJ+ zSJu=&bcQ*^V{z%bMxpoCdY@X4l6^6^uc!3emNS#Ab1F^1vXV3nQ8N>xx?)F?%>FaY zqfY1kX9q;<#_y2_?RsZ+m$XJ79V^(zrL7w*1(RU-aU_oGS%+tO!l*x#Yu0`0EFS>GAFqN?E)gu+`{!R z)zXwMu-6cg##zrN_PE1rT#c!TKj8H5&pG6r{FrZ+P&uU}ro->)VJKz1Dux%}t?}fVzFsJCbq3$<_Fx_HiJzAydjxZdW_W_(23Be@c9#&Zjvsu| zT3piLYr=KKM+X!!sLZtOT)B@wf`6LOom_xs2bdhtD1g&h6w?>@P-`s<dgZQAJ+7l4N=OZ9r0T{d7xvwBy@Om+hQ<$FkTF*82Y;ou&p0YozSR+x z?^|s80#yNH)9aUA^+Il(nOb$)`x?NeU#aTR2e)v9qtAb$!8)Fp7=)Q!AjZ<(CT3e9 zk=h@b=SX6Kn`#z%W(UI6`k%-e; zsq%YaZOat;NSm7Pgz*F@RtY;Z4()n~VXs7u25*~Y^!!dbXFfR{98V)RdSWQ4+3sqW zIxHEX>ag^Ca{p}EFNdBwme?#T!McRo9@)V#@D0Fh>n%)Tcnw%>pWLMCyyj(21#--6 zM=qgPdqVIesH(Keh}pYSy9hfRHjkG9Zz+0NR|5huIBdJT9BHhhe%vB|t%>?pm(=9j z3#SeIutSmRVclJf<>L)!lntW>Qt$f-9B#07JsNVb zYOY=uZ{h!JKv3wjv~aHF9SYB*k0u9v%Zm-7ITj}Hsx=p0tXu&sWG+vjEPXL~Qdy(RKr zT4$%@dBpRpv&YB}_!|zbCon?Uh^!rU*pHON=)5=18)`s-GCafatF9o0X`{b}BX*i4 ziQ$etv1s3racxxwcRw&ZrK^vy*y~AD4Wi)N$cwwt(KiCe;BzWf+H!VYN>aXIF9vQt zSshpAfS5PnO5SX6+zLHc5qsEFuHIzczSl|#w8g8q97dy7Ef1lnjIG5@3wetJqk!LT7>IFYLMm442#}#k+Q&hscZZhGCm$6Sd^Va#NDIa;w z1lQK4blFVy{c&=grs6B#!fb^~5rKBfW%z!@ewPR!ojAChLEV+-8YAcbP}wnWy}ptw1N{lgp#-t}~n;B~GGtCk9ij+*NQAav%B6(Pm7&UVh9Yqo=BpUlTMd}&C& za5Z(h*>K}#=vOa?C;eT8h|}V1VHQBYHZwJF)ZCWXlY~1-x<6o;`W>L=Xg{*!!?4v2 zS?C$eM`?>~!|{fs^=9lu4!Erw*1tqM&b&^p3AMlv*WIAPY|U(Jaia$u)@(28oP$nP zgcWIMP*O>DZhy3=x^})1)DpGlTt4i>4Vvj$X!OFsOYI## z!W3$bnb5}Fo@=<=)A;__NS+qK__p~F9~6E(YzF-}6!}6?m9p}2Bt$l!?~T0heK3;8 z?0O?`cE-+&33ae0s(xki&GPp#BPoO1=isYY2eBt7!bCZm+n$<#6Jn}GTsu`-sbS08 zR({EF{nu9(zL!Z9lG=xjG4^*_B#sU_`NwWxwP`eVv0lW3(F9SQ!vVS*j>XfVeWxNV zP;cV!Pt2i04{zPw=?1&gP>|!W(M7+6Xf_%KlDNaYcTY`)#gPKuZEY0za*#dMnb-G$ zqXaB%sDorgUVlTC>N$joRq7ErIvy9{GQgmIx%Q>^qTx5iRx;vLGp}NL9zbX&nEqqN z1XiXBB95*n=u*&ERFRf~pFqk5vxIbmsyrR&74ar;IVJE$=#R^xAmuKqp49zhM}bEr z?g!1XQf<{S zv>!9bMVGa*W%j5CWk6~-DtVw4VZhwY_072h`%sGr=|zS*;)+?vR-;F;KKYw17zA1P z4yUu(wy=;j{L)Hme>At`?YUo^ANBz0q+TZB>lYqE-7L&Zz?A}z{9WQmP@YL=u|ETC zJ`|VoeSepanl2A00e{;0RyfFio+CUo(_C#WJ{9TxG*!aXR+NR%Av(5*hjyDNQongrHM+^4j zwUW-NcVHwKDu2gg{q^C(-X0bQbYzMg%s<}p@sQEVE_>;R3%gO4cKfCHGrVDY$xky) ztw5Zky$Ll!$C>}>< zu z4sdS!0)oF?aM#W7Z7h1crJHxCtPp?OM=tI;`k&*BfA9M9f8${8uOsWRip2N3o_3_^ z6CB86OVb@vSViL<_aidKOe;!M1Rn~Dn9f_?&i4dAnSb8D|LGMFKu$g*7aX1(dBra8 z0?KtE55Eu%-<^=&#o_9X1zqJU2rA&A7T&`}6A=+SYY*pp4&VEnP&LYzy6hyG*8$^b zfrRAcNSSksp+k?>@_uvaqJg0ioTAOp)QT!Ye;AM0)ynSf@I6V=Da)^8`~68J zZR|f49pSvpe0pOUoF%etx@33GOxiUa!6zRd-gZQgU&p4!iv`CdBh=KX3Q!rlGBJ^*7*Ols1C$2Y0 z;c+BFe*Qh{3}hZ2q;n_!U482Ey0jWpTgZc&^de$gS4t9JC6VlsX3>kUFLjG&Of6fw zs44oys!i{<%{HU5zJwchUm$i(O!vCy2ur*A+2({368xEq-!Vt;_H^&E>&m%zma2ZG z;w*}gei}9rkgcjiBxu!No)$Po*1V{Z5>Zt59MyOY^Lw#Ug*uVTpkU9`tSVn8vu=TM zw4-9O*Ipx1spgGCj0vAhVtw)@xe8dZ>^u*ZP1Y{eRT3#Hmx_j4)2AMPI!r*i1C#pA z@MCjo!ed!?C~hWbLLnRJ-Pn>$tXVLzxa#omBnWmaCU#CFg`bIKGtfUe=Cev{0WKxC zyJ|%z#k?tLi>j%$_UHPlF8+|68fXEdIHaxKXE9et|IN<0kBW* z3QXDk9+p=<6Q*eWkAUImA$9( z!z|O5Z9{vI?9+nR$b`xdjlc;}eu*TM<_MW4iN!D$@@PIg;~oDd6Hzo3=9iJxW0M$* zOaV5kQv+H5KyOheF~UgtoK_mCkW)-`76S=qbT)?<51J`d$Ug!mxaK+yQPIz<)P_(Apha5kuTSj{G7T5o#OEb?-WKspA*j0aAAxYyk%b&MSw69 zPn{C^2Ylyse^(d|k0L@X+WG8Yfd}AfcSDTBhZt2aP_Z`otpInpgtSO@^1Hab&_j=M zZZx7e9Dn!8Op)*HxLFbOVrM<>8FE8&=Vy7+cTiOV|4c#cBW@zdW>|%|ZBhSpR85@O zG<^7mjfs^>3@Xm*d{DI(El^#Ui!+JykkT!lM7GA`tCS-w?W7tR0ansa-F2VLCE5pm#>&fOX0Cs$B_a!@dbSJo;b_Q1zzsnVigSl2assT9G5G|%;U@|KFop>t0`$-ZTD7{Uo z`Dw{U;w^(3KPlNnE)MvrlkhXR>jD@(z%cj^I+$H*o9q#Oui=e0rb(Y`;vhI7hdi$l z%nq9NOs50>GWN`-KLAyLv|%^d7o`{;_&4)Qm@!J}EPo$pnt~iFzV4}^Y#{yf=CKZY5G)BjIczavz2y@Mn2bZIgLV7k$k)n!jq-e8! z|G9p!=u;<+TgwdtZJ}jb-Lw^#!QrNwzk|FS9KT}B(+!3meu0#vTx{&{P;xj+V+Tpz zDXvaQq~zZ);mfJH)T6k3kKx{?kiw12?^LO|Lc^PWCH6#cc8xA3I^kUi>_8}Y4ik}0 zq@NvU9m9C^m=Ev5P0p@Q`>cFawz=vmdo!HCnTy5;IZbx#fxUFg%g{18Xrxsx#oKGs&aVKO=Mv& zkIwj08a@+vB{6>{mTzlYygV(4sRw|H>JVZ08ZahUREWEWzFr1qQaRX`|%fA<@a-JN=YAU&$Z_nh#<2OPNcm;ii4p=GpH;C+o(DIEHSU zrC8YTJr~cDT8MoqZ(z?a1Wnn{Jl$2cQA>-qw<$TUy^ZuX+I%L3<*P@XK;&s`J`r(k9i z$sE7875Q${gp*oV%{g|tJikFa=@lxj#(bu8RolBv!c2xT(^)o1lxEQ!NN`+NgqVBC zn7N1hk0rO2MgyA;zG#`(4hL$PjrVRA_OB@nclc6NlX6M%(Z#O(nJJ{A%*7K~)H%Lw zs$JeGSM4|!DqzK?b5xGGx%z{tj7O>2yS02a^=WPbxrW;{CZwE3BC49hv;tfqQ2 zEL+OeHCRD$<8Pk z3^KN|8?JV@Rnd(}-d%1U%!%wk(YHC6(lTs?f#LJAQ{W(-%r6*FD3cj)e1&j%=A9s3 zEk+0#p6EwF-!oWN5*d22X{(kI&GP`8c>dzp;;52tCwu!7r3PkVl}@@=s&fnfgj6pR zhD*36y*7t~Si!_qiosbYcUGLoW7HPZynvrld_%6Nn+F>3jVjw>?tYR&^+6B1W|SDU zL6tpxnqn8SM$#2}8hVkXwR$ijbEsE@5uC*+VZLZ>X{R|Ad*CdL>Y|ebJ7fyM1Z`pLz zOyW{ZG1@jYG5d~Mu5P^eWa+d*JSZ?MY7#5E_A$zPee%;naRG6Z{lItbrBdOv`$rZU zwfI7VJ8CSR8-|+}X;3u4xF_dE7Zwo5-L8%wb3PcaVO0wJS;eL?45Fo~!U!F3A7fsPtd=!qRu29R2PM%QJ zZi>q#J$9#24v$Q{n&A_R%G9ZY%n;j0?FaA-K~kx#gnn3U?(#{w5D8TK-1NEI|3H86 zDDER>VqOKk#b3+& ziR<1}-4-&c-Xe~{h{+|e-h<^fX_kp{MWW1-#0#?`;Fhrs7U zWyQuIyjE$?Ktb6l|3>G2FRs>%Q~CYPAD7i#AMpuMV{Lr$|kKw5!;fRKrjaF zNQ1h2_MmfUvS-^QxD@N~!K?)RVXvm|Zr!SCXHuhhM#*k-V3W$ueYh*v$v5^Rp7wPu z!=84NP}+eZk|wZISEdT$`eiA$Rl~CwEueX#(jw{A`0S)L-uf|8y@BNw(N^Po{Bau` z*)NU{6`XUO-bz*VxaUa@X_aIBCZBaxS2)U3)>mkp;1yr3ilKyvP7NvL%jX^KVpAnm zX({e6kB+e|Pve~n)ZtPS$LJ@@K;4uRintHoZWSga4_jLDH2ZGHk*MFA57ukXJw7s8 zuvrZ4a@0#~+|Yv=sGo4w9C^umKYDfM{PX@vYQJ-P%N8i9D22pq*cENO zpRDf9<+4w^ATFp#kn#wv@4F8bxaNiE8(F7?$(*Ew3d#qg$O3Z*(zj9kW5>!Nm7?m91Xn;kY%P?&#K=1N87MxC zml@eS@6(}cP|R)rqJ>VLZaRjpB}#*3DfqHPmM$(c$36|hMcaXKfc|Fa>(G0X{ki9h zGs5amgh#FaNm7b)C#=aZ*r(($LtLij!q|*N9jh!=P^0kwq;VQ}8ibfXmYSkiad}W! zJ7evPBeD^(aev_Mu4XxUB4W7ji-EDXnKn3@l^)r>nuy9|-)!ZW=$q*7SyR}sO|6or zy1TU$8h{WpII*FXmb;K`hIJr3z5z+v+FDn4@-TyPB(jkyrFQIJ?6#SS*rhO;Be~5& zoT#I`ee`4G6`O;FOUr!o3`&2;ZLd}7RS&Sb*XMjKNli#(ff%_sw^1dsG0EF#4nDjG zmSmK*cBq#)JczfY9*Y37zDUU{n^_s9-#ZfB+o8YMq7ba@s#w>el`)u$2-ZI7(R;(t z;K)z}cRBm$pr?F$2S$pbnV3b4xM0w2xbvJfm>2o!X^~d=1bNjvMcSzm z(NMxa_AVwBPOdZ^KIpqro|9*GflDVq3y2 zvml4hfKj8;jnR$T*_IrQvt9lYr^=`4Hl)%8?W)Cv2INosB~7i2HW`LosX1n8vc!o= z;a00R#CrS+jV*Gq9!Ms_i5%iFMJb#~NsU9U8!CuhSbTCD|P%`OC zp>K>%idcYB@nZ6A2_-N+##RfYa}AFI?q$?6$F%+94gE`#yi-=c))TFE90{iLv9%5U z*gw$oCsI3DDmuw1IR@r+`|+|qrpVc(OGJrup|QLP501+&kQ|yr;ovF{LMZ%|+1Vgo z4(;SfoUWnJn~dP#$)kj=3v|bz9C~%w033Fl>!4JfEfL zK%1T&uh1#4ySYYBT`s@0C{{_$MT4d5OXXPaGut%^6;tpmY}}R!k?*OqJ*0~T*`)>1 z{WP^swBlKg`ki%cc@~Y)+Zb?R=)@IOv>J55O&dIjvE*lD+-=7zC+*$U@M7eVhLTH_ zZgDaS%C)fjpG_{2ChKMkwhZ#y`=4Y&;<#1hthEjF4P33R`YC6e737M;*gsRzlFDH6YY>=+94qbKEQvpA zRd@D%QOE)ieQgE5A9*S|LFoWU&Mu~Fz+{^&4;fO=(;2wC67nyzWcI;yza&U|}b zwfzovWS7Up#i{*cc9sDpcZttCkHk_I*v8&0&b&0a*H>R}!ZzpR9n>^$ji}mO6kb79 z^GwBZ`xb*F-A6dnl4J8jjxVP+JJQTHx54&p_f93mPCALNcUohdw_{HSjqboQGLa80 z?{n!^4k#ICB7%c*Fys_@vlQNo*9_y zZs&(7E)*10>X@p%d$w-DL4{50h59}Y_~*?;fO*Y8Igrlk#5UzDZ~LJm`t#f!G>wN` z6t;znAg)j_tVFC2i}wp=e!lqN$4tV-yfzZP?Ze-2N)nO)(rA&;+%_E^5<}#`JPksgB+``C*w&L_R zw#wM0Af%|`>BYC6B74G`R`*G0tXg&+Tnb?jbTgI~y4zy4YaoUU_HtxH+d^S>*A7@( zm_x1Yp_I$13diq_jgnB;(Uzos7;BN6G&(5VWmw>3Wu?*X{muE`JBxq|afO1h#r%oBHT`B-J-IrY$Ld8CbY#}N8|7Q zRszU=`6H>G=*H)G(C~q07O-vPC&hURr?+$$o3Z>H*bVgm8m7IxR^)D`=s^1f53?)BF?FFHmVE)o_5__BnkNJu6$UVCv0i;CuDWMpjS<&BVrg8`E? zV3`W5n%7}&RR4rq>LB5`Xh8c6j?Zz~K2CZnbvlBb%mpATeJzhws!_}~V!_F;l^dE!Nb0oXms~uuNOong`x>E^)NGf}t#ieL5T<~RurSlAM1j5^5;oOU z&n&!!f%(`8+0VuBxaCE~!Xk5YZwd-V>UCeTIC>3Z5mZ&F=y2Cm@%mMMPL7(@{T4KD zei2&Y@tBM;6#6LDn!w}WOv zd%~F_3Wnodt0nQcNPyqDS?n$HAcx#?u`SVxy?s8Gl3g{Y=pw93N->giqwFVMkKz}^ zhYI7RPV|{fI0ath8OUavPUEW($MQU|1Byp~rCmO_vJh~4Tw}*zcoLUKp&F0hQwcnH z-XRCDhUtKiheKa>wa=R2@K*gj(^B4$;Epz1HUt%uv%$e%$nWl?MOY;gRW2m&?u zJjJEKh46xZyfuKkhS}w;vOX7TeYwlq_@#k?oZG{xCJg`T@$Jg|Iueg7L_#6NUDSs_ zp_yI*<;RZw`w-U5@|Y~)pV_U4dJE{vIg`IV=7XD7x0B3}oR2^F{qy|R2^KvTM+nLo(7RKG^CJ+wEXPJwhR<6Lh`25m5IFk_mk@9uk)8$;N?e|?E+?1Ms_W% zcn+8uOKvZgj5dwb`RM2T5<29qlGykZsQ{j*_q#xtUopX3k@xvjbr+6t-+R;Bgbt}H zq7~pbXUwpvORW4df7^gz1LLG(%{Lyhp=V$v_{iDiQOcLOR2|NFTpux`E8Gf5@!Hkq zlZBa4tW(O^t(ce#%nQ+D5!@3NNPa1(aWHG2*k^z(77q0-F%0!B8@!?ApX>JSiDiV1 zH4J?f@4@y#Un`tpL*c`h+?tfKf5yc(KS595gq5YGZss2E!aa>3iH)E+h=uSi$*zgP zDlQ2gb?`^Fs@}7l<-rhF?(PVAsh26;4liGO%QC|$PKuUM%GbJN7e{*3HVu5qiR7KQ zVDe8QT6y_*Kwhsy+VZjBl_o{;7rP_Sq?lk6gttz{a4ZWVd5qKGgl*L0JCrKB+oyAw z(fg_ux2!8DLVaTmf5!haAJ*F@)S@h=)NU5ycH5>BYFJ#X=xT{o7aFahME?(e>*x5t tPSZcXBN+MrKX3axLI0OC;D0*_#V0&I_xIid4L*G$CL}Fb!ms1~KLBqhPF4T_ diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index 91b421c143..bf7a425abc 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -1,52 +1,234 @@ -# Migrating from RavenDB to SQL Server or PostgreSQL - -This page covers what you can run today. How the migration works, and what is planned, is in the [migration overview](ravendb-to-sql-migration-overview.md) and the [system design diagram](migration-system-design-diagram.png). +# Migration Engine: Instructions > [!NOTE] -> Copying data is not built yet. The one migration command available is the source report. It sends RavenDB only reads, but loading a database lets RavenDB's own expiration, its automatic deletion of documents past their retention date, run against it. If you are keeping the RavenDB database as a fallback, back it up before you run the report, as [Goals](ravendb-to-sql-migration-overview.md#goals) explains. +> This build does not copy data yet. Only `--migration-source-report` works, and ServiceControl does not start if `ServiceControl/Migration/Enabled` is `true`. + +This page tells you how to move the data of a ServiceControl error instance from RavenDB to SQL Server or PostgreSQL. ServiceControl copies the data itself, so you do not need a separate tool. [The migration overview](ravendb-to-sql-migration-overview.md) explains how the migration works and why. + +The migration does not move the audit instance or the monitoring instance. The audit instance has no migration path and the monitoring instance keeps its data in memory, so it has nothing to move. + +## How the migration runs + +ServiceControl copies the data it needs to open while it is closed. **This part is your outage**. Then it opens and copies the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. + +> [!IMPORTANT] +> Until ServiceControl opens, you can go back to RavenDB and lose nothing. After it opens, you cannot go back. + +The migration copies the data in categories, and each category is Copying, Done, Failed or Abandoned. [The overview](ravendb-to-sql-migration-overview.md#when-something-goes-wrong-you-decide-what-happens-next) explains the categories and the states. ## Before you start -The source is a ServiceControl error instance on RavenDB. Keep its RavenDB settings in its configuration: the migration reads RavenDB through them, including after `PersistenceType` is switched to SQL Server or PostgreSQL. +Make sure that your setup is supported: + +- The RavenDB source is one of these: + - embedded on the host (Windows installation only) + - self-hosted on your network, or RavenDB Cloud +- The primary database and the throughput database of RavenDB are on the same server or cluster. +- The SQL target is SQL Server or PostgreSQL. +- A SQL Server target has Full-Text Search installed. Message search needs it, so `--setup` fails without it. The stock SQL Server container image does not have it. PostgreSQL needs nothing extra. +- The ServiceControl host can connect to RavenDB, to the SQL database, and to the message body storage at the same time. +- If ServiceControl runs in a container, RavenDB is an external server. The container image does not contain the RavenDB server, so it cannot open an embedded database. + +Get the data ready: + +- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. +- Run `--import-failed-errors` while the instance is still on RavenDB. A failed error import is a message that ServiceControl took off the error queue but failed to store. The migration copies these either way, but imported ones arrive as ordinary failed messages. + +Keep these settings as they are: + +- Keep `ServiceControl/RetryHistoryDepth` above 0. If it is 0 or less, the migration refuses to start. At that value, the first completed retry after the move deletes all the copied retry history. +- Do not change `ServiceControl/ErrorRetentionPeriod` or `ServiceControl/EventRetentionPeriod` until the migration is finished. The migration uses them to decide which rows are too old to copy. If one changes after the copy starts, ServiceControl refuses to start. + +## Keep the RavenDB settings + +The migration reads RavenDB through the RavenDB settings that the instance already has. You do not add new settings for the source. Keep these settings in the configuration of the instance, also after you change `ServiceControl/PersistenceType` to SQL Server or PostgreSQL. | Setting | Environment variable | What it is | | --- | --- | --- | -| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | An external RavenDB server. Leave unset for an embedded database | -| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The embedded database's data directory | -| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database, `primary` by default | -| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database, `throughput` by default | -| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | A secured external server's client certificate | -| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. Don't change it during the move | +| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | The address of an external RavenDB server. Leave it unset for an embedded database. | +| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The data directory of the embedded database. | +| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database. The default is `primary`. | +| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database. The default is `throughput`. | +| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | The client certificate for a secured external server. | +| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. ServiceControl does not start without it. | -## Report on the source +In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You cannot leave out the `LICENSINGCOMPONENT_` prefix. -Run the instance's executable with `--migration-source-report`: +## Step 1: Upgrade ServiceControl and make the source report + +> [!IMPORTANT] +> +> - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. +> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database without the migration turned on, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. + +Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: ```powershell -# Installed on Windows, from the instance's installation folder +# Windows installation. Start this from the installation folder of the instance. .\ServiceControl.exe --migration-source-report ``` ```shell -# Container, against an external RavenDB server +# Container, against an external RavenDB server. docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-source-report ``` -From source, build `src/ServiceControl` and run the same command from its output folder, as in [How to run/debug locally](../../README.md#how-to-rundebug-locally). +Where you run the report depends on the source: + +- External server: run the report while ServiceControl runs. The report only reads. +- Embedded database: stop the ServiceControl service first. Run the report, then start the service again. The report starts its own RavenDB process on the data directory, and it cannot do that while the instance holds the directory. + +The report prints: + +- The source persistence that it read. +- The RavenDB server version. +- Whether the source is embedded or external, and where it is. +- The name of each of the two databases, and the setting that each name came from. +- A row count for every collection in both databases. + +The report exits with 0 when it prints a report, and with 1 when it cannot. If the report fails, read [Errors from the RavenDB source](#errors-from-the-ravendb-source). + +## Step 2: Create an empty SQL database + +1. Create a new database on SQL Server or PostgreSQL. +2. In the configuration of the instance, set `ServiceControl/PersistenceType` to `SQLServer` or `PostgreSQL`. +3. Set `ServiceControl/Database/ConnectionString` to the new database. +4. Set `ServiceControl/MessageBody/StorageType` to `FileSystem`, `AzureBlob` or `S3`, and set the settings for that storage type. +5. Run `ServiceControl.exe --setup`. This creates the SQL schema, which includes the table where the migration saves its progress. + +## Step 3: Turn the migration on + +1. Set `ServiceControl/Migration/Enabled` to `true`. +2. If you want less optional data, set a shorter window for the optional categories. A window is how far back an optional category copies. Set it to `0` to turn the category off. + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/EventLogWindow` | The value of `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | The value of `ServiceControl/ErrorRetentionPeriod` | How far back the copy of archived and resolved messages goes. | + +By default, the migration copies all the optional data that RavenDB still holds (current retension period setting). A window longer than the retention period copies nothing extra. After a category starts copying, you cannot change its window. To stop it early, abandon it. + +## Step 4: Run the dry run + +The dry run reads RavenDB and tells you what the migration will do, including a range for how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. + +Book the outage from the range in the report. The range is the shortest time the outage can take, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. + +## Step 5: Back up RavenDB + +> [!IMPORTANT] +> Back up both RavenDB databases, primary and throughput, before you start the copy. Or turn off RavenDB expiration on them. + +RavenDB keeps deleting old failed messages and event log items during the migration and after it. Without a backup, the RavenDB database that you keep to go back to loses data every day. + +## Step 6: Start ServiceControl + +Start the ServiceControl service, or the container. ServiceControl then does these things in order: -The report prints the RavenDB server version, whether the source is embedded or external and where it is, both database names with the setting each came from, and a row count for every collection in both databases. +1. It runs every startup check. If a check fails, ServiceControl does not start and names the check. It copies nothing. +2. It copies every required category while it is closed. If a required category fails, ServiceControl stays closed. It says which category failed, why, and which command to run. +3. It opens when every required category is Done or Abandoned. You do not need to restart it. +4. It copies the optional categories in the background. -- **External server:** run it while ServiceControl is running. It sends only reads, and the note above about expiration applies to a server you are keeping as a fallback. -- **Embedded database:** stop the ServiceControl service, run the report, then start the service again. The report starts its own RavenDB process against the data directory, which cannot happen while the instance holds it. -- **Container with an embedded database:** not supported, because the container image does not ship the RavenDB server. Point the instance at an external RavenDB server instead. +> [!IMPORTANT] +> Do not start any `--error-ingestion-only` worker until ServiceControl opens. A worker refuses to start while a required category is unfinished, but a worker that starts at the same time as ServiceControl can start before the copy saves its first progress, and nothing stops it. -## If the report fails +If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). -The error names the setting to fix: +## Step 7: Watch the background copy -- **"has no database named ..."**: the database name setting it quotes is wrong. -- **"refused its client certificate access ..."**: grant that certificate Read access to the database, or supply a certificate that has it. +ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: -## Not available yet +- The ServicePulse activity feed. It shows when a category fails and when RavenDB cannot be reached. +- The ServiceControl log. +- The output of `--migration-status`. This command does not open RavenDB, so you can run it at any time. + +If the copy slows down your production work, raise `ServiceControl/Migration/ThrottlePauseMilliseconds` and restart. This is the pause between two background batches. The default is 100 milliseconds. + +> [!IMPORTANT] +> Turning the migration off does not stop the copy, because ServiceControl refuses to start until every category is Done or Abandoned. See [how to abandon a category](#a-category-is-failed). + +Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. + +## Step 8: Finish the migration + +1. Stop ServiceControl. +2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. +3. Set `ServiceControl/Migration/Enabled` to `false`. +4. Start ServiceControl. + +Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time. + +If verify shows rows that the copy did not read, keep RavenDB. Those rows are only in RavenDB. RavenDB can show fewer rows than the copy read when nothing is wrong, because RavenDB deletes old rows. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) explains why. + +If you turn the migration off before every category is Done or Abandoned, ServiceControl refuses to start. The refusal names each unfinished category and the ways to continue. + +## If something goes wrong + +### A startup check fails + +ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then start ServiceControl again. The dry run runs the same checks, so you can use it to test your correction. + +### A category is Failed + +A category becomes Failed when it cannot copy rows, or when an error stops it. The rows that it already copied stay in SQL. A restart does not change a Failed category. You must run one of two commands. + +#### To retry a category + +1. Stop ServiceControl. +2. Read why the category failed. The reason is in the startup refusal and in the output of `--migration-status`. +3. Correct the cause. It is usually outside the migration, for example the body storage cannot be reached, a certificate expired, a database is down, or a disk is full. +4. Run `--migration-retry ` to reset the migration progress for that category. +5. Start ServiceControl. It copies the category again from the start. + +#### To abandon a category + +1. Stop ServiceControl. +2. Run `--migration-abandon `. When you abandon a category, the categories that depend on it are abandoned too. +3. Start ServiceControl. + +> [!IMPORTANT] +> **Abandon is final**. The rows that the category already copied stay in SQL, and the rest are not copied. Abandon a category when a retry cannot fix it, or when the data is not worth the time. You cannot retry the event log, so you can only abandon it. + +You can abandon a required category only after it starts copying. You can abandon an optional category at any time, for example to stop a large copy early. + +### You want to go back to RavenDB + +You can go back to RavenDB without losing data only while ServiceControl is still closed: + +1. Set `ServiceControl/Migration/Enabled` to `false`. +2. Set `ServiceControl/PersistenceType` back to RavenDB. +3. Start ServiceControl. + +You lose only the copy. To try the migration again later, start from step 2 with a new, empty SQL database. + +> [!IMPORTANT] +> Do not use the old copy again. The migration skips the categories that it finished, and misses everything that RavenDB received after that. + +After ServiceControl opens on SQL, you cannot go back. You can only finish the migration, or abandon what is left. + +### Errors from the RavenDB source + +The source report, the dry run and the migration show these errors when they cannot read RavenDB: + +- `has no database named ...`: the database name in the setting that the message names is wrong. Correct that setting. +- `refused its client certificate access ...`: give that certificate Read access to the database, or use a certificate that already has this access. +- `is secured but no client certificate is configured`: the connection string starts with `https://`. Set `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`. +- `is valid from ... to ..., which does not include now`: the client certificate expired, or it is not valid yet. Use a current certificate. +- `ServiceControl expects RavenDB Server version ... or higher`: the external RavenDB server is older than this version of ServiceControl supports. Upgrade the RavenDB server. +- `could not start a server for the embedded database ...`: the rest of the line gives the reason. Usually a ServiceControl instance still holds the data directory. Stop that instance and try again. The other reasons are a port that is already in use, and a missing RavenDB server. +- `did not finish loading within 5 minutes`: another process holds the embedded data directory, or the directory is damaged. You cannot change the 5 minutes. + +## Commands and settings + +On an embedded source, stop ServiceControl before you run a command that opens RavenDB. **Only one RavenDB process can use the data directory**. The overview lists [every command and when it can run](ravendb-to-sql-migration-overview.md#the-dry-run), [the retry and abandon commands](ravendb-to-sql-migration-overview.md#retry-and-abandon), and [every migration setting](ravendb-to-sql-migration-overview.md#monitoring-and-settings). + +### Run a command in a container + +In a container, run each command as a one-off container of the same image, against the same databases: + +```shell +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-status +``` -Copying the data (`MigrationMode`), the dry run, and the status and verify commands are planned but not built. The planned steps are in [Migration workflow](ravendb-to-sql-migration-overview.md#migration-workflow). +The RavenDB source must be an external server. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 9c167c3c0b..613dca1680 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -1,384 +1,496 @@ -# Moving data from RavenDB to SQL +# Migration Engine: Design Overview -## Problem +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. -A customer can already point ServiceControl at SQL Server or PostgreSQL. They cannot bring their existing data with them. +This page describes the migration as designed, not what is built today. [The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. -This covers the error instance only. The audit instance has no SQL persister, so a customer who finishes this migration still runs RavenDB for audit. +## Overview -## Strategy +### What this is -- Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention defaults to 14 days, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. -- The required set is what the instance needs the moment it opens. Most of it is small, but two parts grow: unresolved failures, and the last 7 days of event log, which on a busy instance holds a row for every failure, retry and submission. The strategy assumes you keep unresolved failures low by resolving and archiving. **A neglected instance breaks that assumption**: unresolved failures can legitimately be months old, and a large backlog of them makes the closed window long rather than short. The dry run is what tells you which case you are in. -- Anything not selected simply ages out of RavenDB, and the customer deletes the old database when they are ready. +- The migration moves an **error instance's** data from RavenDB to SQL Server or PostgreSQL, so a customer can switch persisters and keep their data. +- It runs inside ServiceControl itself. There is no separate tool. +- **The audit instance is out of scope.** There is no migration path for the audit instance. +- **The monitoring instance is out of scope.** It keeps its data in memory, so there is nothing to move. -## Goals +### TL;DR -- **Minimal downtime**. Only the required data copies with ServiceControl closed. Optional data copies in the background while it serves traffic. -- **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. -- **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. -- **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and the migration has written nothing to RavenDB: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. -- **No duplicates and no gaps**. Rows and the resume cursor, a marker of the last row copied, commit in one transaction, so a crash needs no reconciliation. -- **Every identifier anything depends on is carried across**. The event log, historic retry operations and pending integration events are renumbered, because nothing references their keys. -- **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. -- **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. The one exception is rows RavenDB's own expiration deletes while the copy runs: they are never read, so they are not skips, and the recount is what tells them apart from a real loss. See [what does not come across](#what-does-not-come-across). -- **Bounded impact on a live instance**. The background copy waits a fixed pause between batches, and reads are streamed, so memory does not track the size of the database. -- **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. -- **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. +- **Retention does most of the work.** Error retention is 5 to 45 days and event retention defaults to 14 days, so most of RavenDB ages out on its own within weeks. +- **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. +- **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. +- **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They never age out, they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. -## Deliberately not built, and not currently planned - -- **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. -- **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB, so once the host has served traffic there is no rollback of any kind. -- **Steerable while running.** No pause or resume, and no abort command. Going back during the closed window means stopping and reconfiguring, and changing anything else means editing configuration and restarting. -- **A general-purpose migration tool.** The source is always RavenDB and the target is always a ServiceControl EF Core persister, both at versions this build can read. -- **Custom migration UI via ServicePulse.** Custom checks and the event log will be used for progress reporting, but migration configuration and migration engine control will not be available via the UI. - -## Supported migration scenarios +```mermaid +flowchart LR + A["Checks"] --> B["Copy required data
ServiceControl closed"] + B --> V["Required all Done
or Abandoned"] --> C["ServiceControl opens
optional data copies
in the background"] + C --> D["Everything Done or
Abandoned: turn
migration off"] +``` -The copier runs inside the ServiceControl host, so every row and every message body travels from the source, through ServiceControl, to the target. There is no database-to-database transfer, no backup and restore, and no replication. Whether a combination works therefore comes down to whether the ServiceControl host can reach both ends at once, and how long it takes comes down to how far the data has to travel. +### When something goes wrong, you decide what happens next -**Supported locations:** +The end goal is simple: **either all the data comes across, or you explicitly give up on a named part of it.** -- **The RavenDB source**: embedded on the ServiceControl host (Windows installations only, see below), self-hosted on the same network in a container, VM or bare metal, or RavenDB Cloud -- **The SQL target**: SQL Server or PostgreSQL on the ServiceControl host, elsewhere on the same network, in a container, or as a managed cloud service such as Azure SQL, Amazon RDS or Google Cloud SQL -- **Any combination of the two**, subject to the requirements below +- **Each *category* of data is in one of four states:** Copying, Done, Failed or Abandoned. +- **A category goes Failed when something went wrong**, for example a row it could not copy. It stays Failed, and the status says why. +- **You act with one of two commands**, with ServiceControl stopped: + - `--migration-retry `, after fixing the cause: the next start copies the category again from the beginning. + - `--migration-abandon `: give up on what it has not copied, and keep what it has. +- **A restart never decides anything for you.** A Failed category stays Failed until you run a command, so a restart policy cannot loop or lose data. +- **ServiceControl opens once every required category is Done or Abandoned**, and the migration is finished once every category is. -**By where the data has to travel:** +```mermaid +stateDiagram-v2 + [*] --> Copying + Copying --> Done + Copying --> Failed: something went wrong + Failed --> Copying: --migration-retry + Failed --> Abandoned: --migration-abandon + Copying --> Abandoned: --migration-abandon + Done --> [*] + Abandoned --> [*] +``` -- **On-prem to on-prem.** The common case and the fastest. Embedded or self-hosted RavenDB to SQL on the same host or the same network. -- **On-prem to cloud.** RavenDB on the network, managed SQL in a cloud. Works, but each batch is one round trip, so write latency multiplies by the number of batches rather than being amortised away. -- **Cloud to on-prem.** RavenDB Cloud down to local SQL. Works, and the customer pays egress on everything copied, most of which is archived messages and their bodies. -- **Cloud to cloud.** Works, and is only sensible when ServiceControl runs alongside one of them. A host sitting on-prem between two clouds pulls every byte down and pushes it straight back up. -- **The message body store is a third location.** Bodies come out of RavenDB attachments and go wherever the target is configured to put them: a filesystem, Azure Blob or S3, with small text bodies kept inline in the database. That decision is made at the same time as the database move. +### Goals -**Infrastructure requirements:** +- **Short outage.** Only the required data copies while ServiceControl is closed. +- **Support every RavenDB hosting shape.** Embedded, self-hosted or RavenDB Cloud, on one code path. +- **Never write to RavenDB.** But RavenDB's own expiration keeps deleting old failed messages and event log items, during the migration and afterwards. So the old database is a fallback that degrades from the moment you start, which is why [the instructions](ravendb-to-sql-migration-instructions.md#step-5-back-up-ravendb) have you back it up first. +- **No duplicates and no gaps.** The copied rows and the *checkpoint* commit together, so a crash needs no clean-up. +- **Every identifier anything points at is kept.** Only the event log, historic retry operations and pending integration events get new ids, because nothing points at theirs. +- **Refuses rather than half-migrates.** Every check runs before the first row moves. +- **No silent loss.** Every skipped row is counted with a reason, and a migration cannot finish with a category unfinished unless someone deliberately *abandons* it. +- **Gentle on a live instance.** The copy is streamed, so memory does not grow with the database, and there is a pause between background batches. +- **Known before it starts, visible while it runs.** The dry run predicts the outage, and progress shows up as it happens. -- The ServiceControl host needs network access to the RavenDB source, the SQL target and the body store simultaneously -- Both RavenDB databases, primary and throughput, on one server or cluster -- A SQL Server target must have Full-Text Search installed. `--setup` checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent, because message search is not optional. A stock SQL Server container image does not include it. PostgreSQL needs nothing extra, since its index is a GIN over `to_tsvector` -- A managed target's transient failures are already survivable: retry on failure is on by default and there is no setting to turn it off +### Non-goals -**Not supported:** +- **Zero downtime.** There is a real, if short, outage. +- **Rollback once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. +- **Pausing or steering a running copy.** There is no pause, resume or abort. Decisions are two commands run with ServiceControl stopped, and every other change is a configuration edit and a restart. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine was design to be expandable in the future. -- Any server-to-server copy: no backup and restore, no RavenDB ETL or replication into SQL, no external data pipeline -- A host that can reach only one of the two databases at a time, so no staged move by way of an offline copy -- **An embedded RavenDB source when ServiceControl runs in a container.** Reading an embedded database means starting a RavenDB server process, and the container image does not carry one: `ServiceControl.Persistence.RavenDB.csproj:36` excludes the `RavenDBServer` directory from the artifact, and the copy that would restore it at `:44` is conditional on `CI` not being set, which the Dockerfile sets. A containerised instance migrating away from embedded RavenDB has to point at an external RavenDB server rather than at a data directory. Windows installations are unaffected: the installer unzips the server unconditionally -- Primary and throughput RavenDB databases in different locations -- Anything but RavenDB as the source, or anything but a ServiceControl EF Core persister as the target - -## Migration workflow - -1. Upgrade ServiceControl as normal, still on RavenDB. -2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and whether you want the one [optional](#optional) category, archived and resolved messages, copied. -3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. -4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the dry run reports](#dry-run) before booking an outage around its estimate. -5. Start ServiceControl (`MigrationMode=true`). -6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. -7. The copying of [required data](#required) starts, with ServiceControl still closed. How long it takes depends on how many unresolved failures you have and how busy the last 7 days were, and the [dry run](#dry-run) gives you an estimate. If a required category halts, the host stays closed until you fix the cause and restart, or abandon that category. -8. ServiceControl opens, and whatever [optional data](#optional) they asked for is copied in the background while the instance runs normally. They can watch it from ServicePulse custom checks and events, but not steer it. -9. You run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. Counts can differ in both directions without anything being wrong. SQL can hold more rows, because RavenDB keeps expiring rows the copier already took. SQL can also hold fewer, because once ServiceControl opens it sends pending integration events, removes group comments whose group has no failed messages left, and removes failed error imports once they are imported again. -10. RavenDB data can be removed. - -- If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. -- A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. -- An explicit override exists for a customer who has changed their mind and accepts leaving data behind. It marks the outstanding categories as abandoned, which is a deliberate end state rather than a failure, so the progress check settles and the guard stays armed for any later migration. -- While a selected category is still unfinished, ServiceControl pauses its own clean-up: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. Your SQL database grows until the copy finishes, and these restart on their own once it does. -- **Steps 5 to 7 are the abort window**, which is not the override above: see [the one point you can go back](#the-one-point-you-can-go-back). -- A category that stops because too many rows failed is *halted*, and it stays that way until someone acts: fix the cause and restart to carry on from where it stopped, or abandon it deliberately if you accept the loss. See [a halt stops one category, and clearing it is a restart](#a-halt-stops-one-category-and-clearing-it-is-a-restart). - -## Architecture +### The moving parts ```mermaid -flowchart TB - cfg["Configuration + restart
the only way to change anything"] - checks["Custom checks + activity feed
progress, with no new client needed"] - - subgraph host["One ServiceControl host process, started with MigrationMode = true"] - direction LR - raven["RavenDB persister
own AssemblyLoadContext
new read-only lifecycle"] - engine["MigrationEngine
categories, throttle,
dry run, verification"] - target["EF Core persister
SQL Server or PostgreSQL
own AssemblyLoadContext"] - raven -->|"IMigrationSource"| engine - engine -->|"IMigrationTarget"| target +flowchart LR + subgraph host["One ServiceControl process"] + src["RavenDB
source"] --> eng["Migration
engine"] --> tgt["SQL
target"] end - - old[("Old RavenDB
read only, never written to")] - sql[("SQL Server or PostgreSQL
plus a new checkpoint table")] - bodies[("Message body store
filesystem, Azure Blob or S3")] - - cfg --> host - host --> checks - old --> raven - target --> sql - target --> bodies -``` - -- **The engine and the host know no store.** They deal in categories, cursors and counts. The source maps a category to what it reads and describes itself as labelled facts, the target maps a category to where it writes and how to count it, and each contributes its own startup checks. RavenDB to SQL is the only pair built, and another pair, such as SQL to RavenDB, would add a source or a target without changing the engine. -- **The source reads the instance's own RavenDB settings**, so an existing customer sets nothing new. Leave them in place when switching `PersistenceType`. -- **Both persisters load into the same process**, each into its own `AssemblyLoadContext`. -- **The engine references neither assembly.** It knows only `IMigrationSource` and `IMigrationTarget`, and treats the resume cursor as an opaque value it passes from one to the other, so it can be tested against fakes on either side. - -## Startup sequence - -```mermaid -flowchart TB - A["Restart with MigrationMode = true"] --> B["Open the SQL target, exactly as today"] - B --> C["Open the old RavenDB, read only"] - C --> D{"All checks pass?"} - D -->|"No"| E["Host does not start.
Says which check failed.
Nothing has been copied."] - D -->|"Yes"| F["Copy what cannot be recreated.
How long depends on unresolved failures
and the last 7 days of events.
ServiceControl still closed."] - F -->|"A required category halts"| M["Host stays closed.
Fix the cause and restart,
or abandon the category."] - F -->|"Required data copied"| G["ServiceControl opens.
New failed messages go straight to SQL."] - G --> H["Copy the selected history in the background,
with a fixed pause between batches"] - H --> I["Verify row counts on both sides,
category by category"] - I --> J{"MigrationMode = false,
everything complete?"} - J -->|"No"| K["Host does not start.
Names what is outstanding.
An override exists."] - J -->|"Yes"| L["RavenDB is never opened again"] + old[("RavenDB
read only")] --> src + tgt --> sql[("SQL Server or PostgreSQL
plus the checkpoint table")] + tgt --> bod[("Body store
filesystem, Blob or S3")] ``` -**Checked before a single row moves:** +- **Both persisters load into one ServiceControl process**, each in its own `AssemblyLoadContext`, and both are live during the copy. +- **The engine knows neither database.** It knows only `IMigrationSource` and `IMigrationTarget`, and deals in categories, *cursors* and counts. This allows it to be tested, and expanded in the future. + - The source maps each category to what it reads, and describes itself as labelled facts. + - The target maps each category to where it writes and how to count it. + - Each side adds its own startup checks. + - The engine passes the cursor from one side to the other without looking inside it. + - RavenDB to SQL is the only supported pair, but the engine does not depend on it. +- **The source reuses the instance's existing RavenDB settings**, so an existing customer configures nothing new for it. Leave them in place when switching `ServiceControl/PersistenceType`. -- The SQL schema is current -- Message body storage is writable -- Both RavenDB databases are reachable -- The client certificate is valid, where the source is an external server -- The source is at a version this build can read -- The selected categories are valid -- `RetryHistoryDepth` is greater than zero. At zero or less, the first completed retry after the migration deletes the entire copied retry history, and no row count would ever show it +### A migration from start to finish -## Data to be migrated (Categories) +[The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. -### Required +- **The [free abort](#going-back-to-ravendb) window runs from the first start with migration on until ServiceControl opens.** Until then, you can go back to RavenDB and lose only the copy. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The required copy runs before any of them starts, so they start only once it finishes. Optional categories never hold them back. -- Unresolved **and retry-issued** failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table. Retry-issued messages are required for the same reason unresolved ones are: issuing a retry deletes the expiry, so they never age out. Leaving one behind means the retry confirmation arrives with no row to mark resolved, and the message stays missing from the customer's list while the retry actually succeeded -- Message redirects -- Endpoint settings -- Known endpoints, including the monitored flag. One category, because the flag is a property of the endpoint row and cannot be copied without it -- Notification settings -- The licence trial end date -- Throughput history -- Retry operations, unacknowledged and historic. One category, because RavenDB holds both lists in a single document -- Licensing report masks -- The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again -- Subscriptions -- The last 7 days of the event log, so the ServicePulse activity feed shows what led up to the failures you are about to act on the moment ServiceControl opens. The 7 days count back from whichever is earlier: the newest event in RavenDB, or the last time RavenDB stored one. Counting from the data rather than from when the copy runs means a restart copies the same window. The second limit is there because event times come from the endpoints, and an endpoint whose clock runs ahead would otherwise push the window forward. Older events are not copied and age out of RavenDB on their own: at the default 14-day event retention that is at most another 7 days of history -- Integration events still waiting to be sent when you switch over. There are only any if the old instance was falling behind or could not reach the broker, and each one is an event a subscriber has not yet received. They are sent once ServiceControl opens, later than they would have been. RavenDB already sends them in no particular order, so no ordering is lost -- Custom checks, with the status each last reported. They are required because not every check reports again: an endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one behind could hide a known failure until that endpoint restarts -- Failed error imports, with their bodies. Each is a failed message ServiceControl took off the error queue but could not ingest, so it exists nowhere else, and it never expires. After the move, the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB before you start is better still, and the [dry run](#dry-run) tells you how many there are -- Group comments, copied after the unresolved failed messages, so a note such as "do not retry this group" is there the moment ServiceControl opens. They follow those messages because a comment belongs to a failure group, and the groups ServicePulse shows first are built from unresolved failed messages. Every comment except a blank one is copied. A comment on a group whose messages are all archived or resolved waits while those messages copy, and once the migration settles ServiceControl's own clean-up removes any comment whose group has no failed messages left, exactly as it always does on SQL +## Design -### Optional +### Supported locations -- Archived and resolved failed messages: the biggest category by far, and most of the copying time +The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no database-to-database transfer. RavenDB holds documents and SQL holds tables, so the rows have to be reshaped, and ServiceControl's own persisters already know how to read one and write the other. So what matters is whether the host can reach both ends, and how far the data travels. -### Not migrated +| | Supported locations | +| --- | --- | +| RavenDB source | Embedded on the host (Windows installations only), self-hosted on the network (container, VM or bare metal), or RavenDB Cloud | +| SQL target | SQL Server or PostgreSQL on the host, on the network, in a container, or managed (Azure SQL, Amazon RDS, Google Cloud SQL) | +| Body store | Wherever the target is configured to put bodies: filesystem, Azure Blob or S3. Small text bodies stay in the database. This choice is made at the same time as the database move | -- The RavenDB index definitions -- The transient in-flight collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations` -- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works -- The `ConnectedApplications` document, which only versions 6.0 and 6.1 wrote. Since 6.2 the MassTransit connector status that ServicePulse uses to turn features on and off comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document -- Broker and audit service version details, which refill on the throughput collector's next run -- Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if the message fails again afterwards: on RavenDB it can then never be edited again, and after the move it can be edited once more -- Heartbeat state, which neither persister stores: ServiceControl rebuilds it in memory from live heartbeats after every restart. The list of known endpoints and which ones are monitored is copied, so after the move heartbeat monitoring behaves exactly as it does after any restart. An endpoint instance that is down sends no heartbeat, so it is counted as failing on the dashboard with no last heartbeat time, and no heartbeat alert is raised for it +| Route | What to expect | +| --- | --- | +| On-prem to on-prem | The common case, and the fastest: embedded or self-hosted RavenDB to SQL on the same host or network | +| On-prem to cloud | Works. Each batch is a round trip, so write latency adds up per batch | +| Cloud to on-prem | Works. The customer pays egress on everything copied, mostly archived messages and their bodies | +| Cloud to cloud | Works, but only sensible when ServiceControl runs next to one of them. A host on-prem between two clouds pulls every byte down and pushes it straight back up | -## What does not come across +**Requirements:** -**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. Neither is an event log item raised before the start of the 7-day window, which falls outside the [required](#required) event log window rather than being skipped. +- The host reaches the RavenDB source, the SQL target and the body store at the same time. +- The SQL database is new and empty: only the schema `--setup` created. +- Both RavenDB databases, primary and throughput, are on one server or cluster. The migration opens both through one `IDocumentStore`, and so does the licensing component. +- A SQL Server target has Full-Text Search installed, because message search needs it. + - The `AddFullTextSearch` EF Core migration checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent. + - An EF Core migration runs once, so this is checked the first time `--setup` brings a database up to that migration, not on every `--setup`. + - A stock SQL Server container image does not include Full-Text Search. + - PostgreSQL needs nothing extra, because its search index is a GIN index over `to_tsvector`. +- Transient failures on a managed target are already survivable. Retry on failure is on by default, and no setting turns it off. -**Rows skipped one at a time, and counted.** Each of these shows up in the skipped count for its category, broken out by reason, so you can see how much went and why: +**Not supported:** -- A failed message whose `UniqueMessageId` is not a GUID. The target column is a `uniqueidentifier` and the value is never regenerated, because it is simultaneously the primary key, the ServicePulse URL, the retry correlation key and the body lookup key. -- A failed message with no processing attempts recorded against it. The SQL model keeps the newest attempt and derives the failure time, the failing endpoint and the exception from it, all of which are required columns, so a message with nothing to derive them from cannot be written at all rather than being written blank. -- A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. -- A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. -- An archived or resolved failed message, or an event log item, already past its retention period. SQL's retention clean-up would delete it on its first pass, so it is counted rather than copied only to be deleted. -- Endpoint settings for an endpoint ServiceControl does not know. ServiceControl removes those settings shortly after it starts. -- A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way: RavenDB can store one, but SQL never does. +- Any server-to-server copy: backup and restore, RavenDB ETL or replication, or an external pipeline. +- A host that can reach only one database at a time, so no staged move through an offline copy. +- An embedded RavenDB source when ServiceControl runs in a container. Reading an embedded database means starting a RavenDB server process, and the container image does not include one. +- Primary and throughput RavenDB databases in different places. +- Any source other than RavenDB, or any target other than a ServiceControl SQL persister. + +### What data moves + +There are eighteen *categories*: sixteen required and two optional. + +| Category | Kind | Why | +| --- | --- | --- | +| Unresolved and retry-issued failed messages, with bodies | Required | The point of the instance. Issuing a retry removes the expiry, so retry-issued messages never age out. Leave one behind and the retry confirmation finds no row to resolve, so the message stays missing even though the retry succeeded | +| Message redirects | Required | | +| Known endpoints, with the monitored flag | Required | The flag is part of the endpoint row | +| Endpoint settings | Required | Copied after known endpoints | +| Notification settings | Required | | +| Licence trial end date | Required | | +| Licensing endpoint records | Required | The per-endpoint rows in the throughput database | +| Throughput history | Required | Copied after licensing endpoint records, because each day's row hangs off one | +| Retry operations, unacknowledged and historic | Required | One category, because RavenDB holds both lists in one document | +| Licensing report masks | Required | | +| Uploaded licensed endpoint details file | Required | Nothing recomputes it. Losing it means downloading it again from the licence portal and uploading it again | +| Subscriptions | Required | | +| Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been. RavenDB sends them in no particular order anyway, so no ordering is lost | +| Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts | +| Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | +| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. A comment on a group whose messages are all archived or resolved is removed by ServiceControl's usual clean-up soon after it opens, because that group has no failed messages in SQL yet | +| Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | +| Archived and resolved failed messages, with bodies | Optional | The biggest category by far, and most of the copying time | + +**Never copied, because nothing needs them:** + +- RavenDB index definitions. +- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations`. +- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. +- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. +- Broker and audit service version details, which refill on the next throughput collection. +- Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if it fails again: on RavenDB it can then never be edited again, and after the move it can be edited once more. +- Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. + +### Optional data windows + +- **Each optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. +- **The window defaults to the instance's own retention period**: `ServiceControl/EventRetentionPeriod` for the event log, `ServiceControl/ErrorRetentionPeriod` for archived and resolved messages. So by default everything RavenDB still holds is copied. +- **`0` turns a category off before it starts.** A longer window than the retention period copies nothing extra, because older rows would only be deleted by SQL's clean-up. +- **The window counts back from when the category first started copying**, which the checkpoint records, so a restart copies the same window rather than one that has slid forward. +- **The window is fixed once the category starts.** A changed value, including a changed retention period behind a default window, is refused at startup, naming the category. To stop a started category short, abandon it. + +### Data that changes shape + +- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so a message that failed five times arrives showing one attempt. This applies to every failed message, whatever its status. +- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. +- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. + - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. + - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. + - The copy logs a warning naming both endpoints and the one it kept. + - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. + +The dry run counts both kinds of merge before anything moves. + +### Skipped rows + +Every skipped row is counted under a reason, and its id is written to the log. + +> [!IMPORTANT] +> The log is the only record of each skipped row. Look them up in RavenDB while you still have it. + +- *Fault* skips mean something went wrong. +- *Harmless* skips are rows SQL would have removed anyway. + +| Reason | Kind | Can a retry fix it? | +| --- | --- | --- | +| A message body that cannot be read after three tries, or that RavenDB does not hold at all. The whole message is skipped, because a message with no body is worse than none | Fault | Yes, once the cause is fixed | +| An error from the SQL target | Fault | Yes, once the cause is fixed | +| A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | Fault | No | +| A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | Fault | No | +| A subscription whose message type or address is over 200 characters, the SQL key limit | Fault | No | +| A row missing a value SQL requires, such as a known endpoint with no name | Fault | No | +| A blank group comment, which SQL never stores | Harmless | Not needed | +| An archived or resolved message, or event log item, already past its retention period | Harmless | Not needed | + +- **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. +- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them shortly after it opens, by the same rule it always applies, so the migration does not repeat that rule. +- **Rows outside a window are not skips.** They are simply not part of the copy. +- **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. + - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). + - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. + - The exception: a message that failed again after being archived or resolved on version 6.18 or earlier kept its old expiry, and upgrading does not remove it. So a few unresolved messages can still expire during the copy. + - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. + +### Startup checks + +In this order, cheapest first: + +1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. A RavenDB instance with migration on is refused here, because RavenDB is only ever the source. +2. The optional category windows are valid time spans of zero or more. +3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history, and no row count would ever show it. +4. The SQL schema is current. This also refuses a database upgraded without `--setup`. +5. Until a required category has started copying: the SQL database holds no ServiceControl data. `--setup` writes none, but even one plain start of ServiceControl writes rows. Copying into a database that already served on SQL would mix the two. +6. Message body storage is writable. The probe body it writes is deleted again. +7. The SQL target opens, and no optional category's window has changed since that category started. +8. The RavenDB source opens: + - an embedded server starts; + - on an external server, the client certificate is in date and an `https://` address has one, checked before the first request; + - an external server is at least the RavenDB client's version; + - both databases load, within 5 minutes each on an embedded source. + + If every required category is already Done or Abandoned, a source that will not open does not stop the start. ServiceControl opens, and the optional categories wait for the source. + +Checks stop at the first one that refuses, and the refusal names it. Once open, the source can add checks of its own. The RavenDB source adds none. -**Things that change shape, and are not counted as skips at all.** The dry run counts the two merges before anything moves. Attempt history and renumbering apply to every row of their kind, so there is nothing to count. They are also the ones to read twice: +```mermaid +flowchart LR + A["Checks pass?"] -->|No| X["Host stays closed
and says why"] + A -->|Yes| B["Copy every required
category"] + B --> C["All Done
or Abandoned?"] + C -->|No| X + C -->|Yes| D["ServiceControl opens"] + D --> E["Copy optional
categories"] +``` -- **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, whether it is unresolved, archived or resolved. A message that failed five times arrives showing one attempt, and the other four are gone. -- **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. -- **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. -- **Event log items, historic retry operations and pending integration events are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. +- **The host also stays closed** if a second instance is writing the same checkpoints. +- **When ServiceControl opens**, the target is marked as opened, which ends the [free abort](#going-back-to-ravendb). +- **Once it has opened, a RavenDB outage does not stop it.** Only the background copy needs the source then, so ServiceControl opens anyway, and the optional categories [wait for the source](#failed-categories) to come back. -**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, when it is unarchived, or when a retry is issued. The exception is a message that failed again after being archived or resolved while the instance ran version 6.18 or earlier: it kept its old expiry, and upgrading does not remove it, so a few unresolved messages can still expire during the copy. Apart from those, only the archived and resolved messages category and the event log can shrink underneath the copier. Archived and resolved messages copy in the background, where the window is longest. The event log's 7 days copy while ServiceControl is closed, and at the default 14-day event retention even the oldest of them is a week from expiring on a source that stopped recently, so the sweep reaches the window only on a source left stopped for days before the move. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere. The copier counts each category before it starts, and if the copy comes up short of that count it counts the source again: rows that no longer exist were removed by RavenDB and are an absence, while rows that still exist but were never read halt the category. The dry run's count is a snapshot rather than a promise, and the recount is what shows that RavenDB removed rows while the copy ran. +### Reading from RavenDB -**A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. +- **The source opens RavenDB read-only**: connect, check the version, stop. It never runs RavenDB's database setup, and it refuses any write request. +- **Both source databases are opened through one `IDocumentStore`**, which is why they must be on the same server or cluster. +- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. A RavenDB start rewrites no stored document, so the copy reads exactly what this build's RavenDB persister would read. The only version check compares the RavenDB server to the RavenDB client, and only for an external source. +- **Distance to the source sets the pace.** The copier already holds each document from the stream, so each body costs one round trip rather than two. But it is one per message, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. -## The one point you can go back +### Writing to SQL -While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data. To start again later, start from an empty SQL database: drop it, create it, and run `--setup` again. Do not reuse the old copy, because a second attempt skips every category the first one finished, so anything that reached RavenDB since is left behind, and integration events RavenDB has since sent would be sent again. +- **A failed message is written whole**, with its status unchanged. +- **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. +- **`StatusChangedAt` is rebuilt.** + - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. + - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. +- **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. +- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier applies the inline threshold itself, because that threshold lives on the ingestion path. It defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. +- **Throughput rows are written directly, not through the collector, and each day's count is set, not added.** + - Throughput is required, so it copies while ServiceControl is closed, before any collector has written to SQL. + - Setting makes the category safe to resume after a crash, where adding would double-count. The collector's own path does add (`LicensingDataStore.cs:198`). + - Copying the rows stops the audit and broker collectors gathering the same days again, because `LastCollectedDate` is worked out from the newest throughput row (`LicensingDataStore.cs:45`). + - The checkpoint stops a second pass overwriting days the collectors have written since. -That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. +### Batch size and throttling -## Reading from RavenDB +- **Batch size comes from the database.** SQL Server divides its parameter budget by the column count, PostgreSQL uses a flat 50 rows. +- **Background batches have a pause between them**, 100 ms by default, set by `ServiceControl/Migration/ThrottlePauseMilliseconds`. The first batch of each category is not paused. +- **Raise the pause if the copy competes with production.** Turning migration off is not a remedy: the host refuses to start until every category is Done or Abandoned. +- **The required copy is never paused**, because ServiceControl is closed and nothing competes with it. -- A third RavenDB lifecycle opens the source: connect, check the version, stop. It never calls `DatabaseSetup.Execute`. -- Both source databases must be on the same server or cluster (`LicensingDataStore.cs:35`). -- The source has to be at a ServiceControl version this build can read, and nothing in RavenDB records one today. The only version check that exists compares the RavenDB server version to the RavenDB client version, and runs only for an external source. So a marker is stamped into the database on upgrade, and a source without one, or one from a newer major version, is refused by name rather than misread. -- Duration scales with distance to the source. The copier already holds the document from the stream, so each body costs **one** round trip rather than two, but it is one per message and they are not batched. Egress out of RavenDB Cloud is billed to the customer. See [batching and throttling](#batching-and-throttling). +### Progress and checkpoints -## Writing to SQL +The checkpoint is one row per category, kept in the SQL database and created by `--setup`. It holds the category's *state*, its cursor, its copied, skipped and already-present counts, how many of the already-present rows were merges, the count per skip reason, how many rows SQL held for the category when it settled, timings, the last error, the window it copies with, and a version number to catch a second writer. -- A whole `FailedMessage` is written with its stored status intact. No existing caller does that, though the dialect upsert already accepts a status, so the gap is smaller than it looks. -- `UniqueMessageId` keeps its value, but converts type: the source holds a string and the target column is a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. -- `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages normally have no `@expires` (see [the exception](#what-does-not-come-across) for messages from version 6.18 or earlier), so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. -- Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. -- Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. Throughput is a required category, so it copies while ServiceControl is closed, before any collector has written to SQL. Setting is what makes the category safe to resume after a crash, where adding would double-count. Copying the rows is also what stops the audit and broker collectors re-gathering the same days when the host opens, because `LastCollectedDate` is derived from the newest throughput row rather than stored (`LicensingDataStore.cs:45`). The checkpoint is what stops a second pass overwriting days the collectors have written since. -- Identifiers narrow on the way across, and the dry run counts every kind. What narrows, merges or cannot be stored at all is in [what does not come across](#what-does-not-come-across). +```mermaid +sequenceDiagram + participant E as Engine + participant S as RavenDB + participant T as SQL + loop each batch + E->>S: Read rows after the cursor, and their bodies + E->>T: Write rows + new counts + new cursor + Note over T: One transaction + E->>E: Stop as Failed if too much of this run was skipped + end + E->>T: Settle as Done or Failed +``` -## Batching and throttling +- **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. +- **A crash costs only the batch in flight.** The next start carries on from the committed cursor. +- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. Unreadable bodies stay off the checkpoint until the write commits. +- **Every skip has a reason, or the save is refused.** The per-reason counts must add up exactly to the skipped count, and the target's counts must match what it committed. +- **The stored counts always describe the rows in SQL.** The target adds its own outcome and saves the result beside the rows, so nothing is saved later or separately. +- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts, or a command and a host, pointed at one database cannot interleave. +- **A copy stopped between two categories still shows the ones it never reached.** Before copying anything, a start saves a *not started* row for each category it is about to copy, so every gate sees them as unfinished. +- **Only the copier and the two commands write the checkpoint.** The status and verify commands only read it. -- Batch size comes from the provider: SQL Server divides its own parameter budget by the column count, PostgreSQL uses a flat 50 rows. -- The throttle is a configurable pause between batches, defaulting to 100 ms. Raising it slows the background copy and eases the load on production. Turning `MigrationMode` off is not a remedy: while a selected category is unfinished the host refuses to start, unless you abandon that category. +### Failed categories -## Checkpointing and resume +A category is *Failed* when something went wrong. Everything already copied stays committed. Nothing is retried in the background and nothing waits for a timer: the category stays Failed, with the reason written on it, until you run a [command](#retry-and-abandon). -A copy that runs for hours will be interrupted at some point: a restart, a dropped connection, a machine reboot. The checkpoint is what makes an interruption cost only the batch that was in flight. It is one row per category, kept on the target and created by `--setup` along with the rest of the schema, and it is written in the same database transaction as the rows it describes. Only the copier writes to it; the status and verify commands read it. +**What makes a category Failed:** -**What one row holds:** the category it tracks, its state, the resume cursor, how many rows were copied, skipped and already present, a count per skip reason, how many rows the source held when the category started, when it started, when it last made progress, when it settled, the last error, and a version number used to spot a second writer. +- **Any fault skip by the end of the category**, even one. +- **Too many fault skips in this run, which stops it early.** More than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Needing both means a tiny category does not stop on one bad row, and a huge one does not stop on its 101st. Under the threshold a category runs to the end, so you get the whole list of bad rows in one pass. +- **The category's own counts not balancing:** every row read must be copied, skipped or already present. +- **In a required category only, an exception or a stall of 30 minutes with nothing committed**, such as the body store unreachable, a certificate expired, the source or target down, or the disk full. The error and the cursor are recorded. -**The states, and which ones a restart re-enters.** `Complete`, `CompleteWithErrors` and `Abandoned` are terminal, so a restart passes straight over the category. `Halted` and `Blocked` are not: a halt is resumed from its cursor once the cause is fixed, and a block clears itself once the category it waits on settles, which is how group comments end up behind unresolved failed messages. `NotStarted` and `InProgress` both mean there is work to do. +**What does not make it Failed:** -### One batch, and why nothing provisional is ever saved +- **Harmless skips.** +- **A shutdown or a crash.** The category stays Copying and the next start carries on from the cursor. +- **An exception or a stall in an optional category.** The error is written on the category, shown in status, and the next start carries on from the cursor. A RavenDB outage after opening is handled the same way. So a network blip never forces anything to be abandoned. If the error keeps coming back, retry or abandon the category. -```mermaid -sequenceDiagram - participant E as Migration engine - participant S as RavenDB source - participant T as SQL target - participant C as Checkpoint row - - E->>C: Read this category's row - C-->>E: State, cursor, totals so far - loop One batch at a time - E->>S: Read the next batch after the cursor - S-->>E: Rows, and the cursor they end at - opt The category carries message bodies - E->>S: Read each body, up to three attempts - S-->>E: The bodies, and which ones could not be read - end - E->>T: Write the rows, with the totals so far,
the unreadable bodies and the new cursor - Note over T,C: One transaction. The rows, the target's own skips,
the new totals and the cursor all commit, or none of them do - T-->>E: The checkpoint exactly as it committed - E->>E: Halt if too much of this run was skipped - end - E->>C: Settle as complete, complete with errors, or halted -``` +**What a Failed category costs:** -The thing to read twice is that the counts never travel back through the engine to be saved on some later write. The engine hands the target the totals so far, the target adds its own outcome to them and saves the result beside the rows, and the engine then keeps whatever committed. So there is no window in which the stored row claims rows that are not there, and a crash at any instant leaves counts and cursor that both describe exactly the rows in SQL. +- **The other categories still run.** Every required category gets its go in one start, so one fix-and-retry covers them all. A category that must follow a Failed one waits. +- **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. +- **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. -**What that buys, and why each part is needed:** +**What the checks cannot see, and accept:** -- **Progress never gets ahead of the data.** The rows and the cursor commit together, so a restart cannot skip past rows that were never written. -- **A crash costs the batch in flight and nothing else.** The next run reads from the committed cursor. -- **Re-reading a batch cannot double-count it.** Unreadable bodies stay off the checkpoint until the write commits, so a batch that is read twice is counted once, and rows the earlier attempt did write come back as *already present* rather than as fresh copies. -- **Every skipped row has a reason, or the save is refused.** The checkpoint rejects a batch reporting more skips than it explains, because verification has to account for each one rather than report a healthy migration as broken. -- **Each category resumes independently**, so a half-copied category picks up where it stopped while its neighbours are untouched. -- **The halt counters are per run and deliberately not stored.** If the skips that tripped a halt stayed on the row, a restart with the cause fixed would re-trip it on its first batch. -- **A second writer is caught rather than merged.** Each save carries the version it read, and a save against a row that has moved on is refused, so two hosts pointed at one target cannot quietly interleave their progress. -- **If a message is already in SQL the SQL row wins and the copier skips it**, which is what makes every category safe to run twice. +- **Nothing counts RavenDB during the copy.** So a read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read (see [the read-only commands](#the-dry-run)). +- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Tests on each reader cover those. +- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. +- **The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. -## Error handling +### Retry and abandon -- Which rows are skipped, and why, is in [what does not come across](#what-does-not-come-across). What follows is the mechanics around those rules. -- A body is read up to three times before the message is skipped whole, and the exhausted attempts count toward the halt threshold. -- Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. -- A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. -- The halt threshold is proportional, 5 percent by default, with an absolute floor, and skips halt a category only when both are exceeded. The one exception is a category smaller than the floor, which halts if it loses more than half its rows. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. -- Rows left behind because SQL would remove them anyway (past retention, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. -- The percentage is measured against what the run has processed so far rather than against the category's total, so a run that starts badly looks worse than it is. The floor is what keeps that harmless, since fewer than 101 skipped rows never consults the percentage at all. More than that, bunched at the start, does halt a category whose overall rate would have been fine, and the cost is one restart: the skipped rows commit with the cursor, so the next run resumes past them with its counters back at zero. -- A source therefore must not read a category in an order that puts the rows most likely to be skipped at the front of it. -- Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. +Stop ServiceControl, run one of these, then start it again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. -### A halt stops one category, and clearing it is a restart +| Command | Works on | What it does | +| --- | --- | --- | +| `--migration-retry ` | A Failed category, or an optional one still copying with an error | The next start copies it again from the beginning | +| `--migration-abandon ` | A Failed or started required category, or any optional one | Gives up on what it has not copied, and on any category whose rows hang off it. What they copied stays in SQL. Final | -A halt is the copy refusing to keep going on one category because something is wrong beyond the odd bad row. It is not a crash and not data loss: everything already copied is committed, the cursor points at the row after the last one that committed, and the reason is written on the category. Nothing is retried in the background and nothing waits for a timer. The category sits halted until a person does something about it. +**To retry:** -```mermaid -stateDiagram-v2 - [*] --> NotStarted: nothing has run yet - NotStarted --> InProgress: the host starts with MigrationMode = true - NotStarted --> Blocked: the category it must follow has not settled - Blocked --> InProgress: that category settles, then the next restart - InProgress --> InProgress: the host was stopped mid-copy,
so the next start resumes from the cursor - InProgress --> Complete: every row reached, none skipped - InProgress --> CompleteWithErrors: every row reached, some skipped - InProgress --> Halted: too many rows skipped, most of a small category lost,
an error it did not expect, or a shortfall the recount confirms - Halted --> InProgress: fix the cause, restart,
carry on from the cursor - Halted --> Abandoned: accept the loss, deliberately - InProgress --> Abandoned: accept the loss, deliberately - Complete --> [*] - CompleteWithErrors --> [*] - Abandoned --> [*] -``` +1. Read the reason in the refusal, the log or `--migration-status`. It names the skipped rows by reason, or the error. +2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or target down, or the disk full. +3. Run `--migration-retry `, then start ServiceControl. -**Four things halt a category.** +**What a retry does:** -- **Too many skips.** The skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental. -- **Most of a small category lost.** A category with fewer rows than the floor loses more than half of them. -- **An error the copy did not expect.** The error type and the cursor it stopped at are recorded. -- **A shortfall that survives the recount.** The copy came up short of the starting count, and the missing rows still exist in RavenDB but were never read. +- **It re-reads the whole category from the start**, every time, which recovers from anything. Rows already in SQL are left alone, so only the rows that failed can change. Every message body is read again, so a retry takes about as long as the first copy. +- **Historic retry operations and pending integration events** get new keys from SQL on every insert, so a re-read could not recognise rows already copied. A retry of either first deletes what that category copied, in the same save as the reset. That is safe because both are required, so ServiceControl has not opened and the target started empty. +- **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. -A host being shut down is none of these: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. +**To abandon:** -**A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the unresolved failed messages their groups are built from. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. +- **A required category can be abandoned once it is Failed or has started copying.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone before it starts, point back or start over: ServiceControl has not opened, so nothing being served is lost. +- **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. +- **Abandoning cascades to categories whose rows hang off it**, and the command names each: throughput history (it points at licensing endpoint records) and endpoint settings (it belongs to known endpoints). Group comments only waits for unresolved failed messages, so it is not abandoned with them. +- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a missing required value. The refusal and the dry run say which, so abandon after the first failure rather than retrying. -**What it costs depends on which category halted.** A halted optional category means the instance keeps serving traffic and that one slice of history is missing until it is resumed. A halted required category means the host stays closed, so the outage carries on until the halt is cleared or the category is abandoned. That is deliberate: opening the host is the point of no return, and it should not happen with required data left behind by accident. +> [!IMPORTANT] +> Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. -**Clearing it:** +**Nothing happens on a restart that you did not ask for.** A container or service restart policy restarts a refused host without anyone looking, and gets the same refusal each time, without reading anything. That is loud and loses nothing. -1. Read the reason on the category, in the custom check or the status command. It names the count that tripped the threshold, or the error, and the cursor either way. -2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or the target down, or the disk full. -3. Restart the host with `MigrationMode=true`. The category picks up at its cursor, its run counters start again at zero, and the skips already recorded stay on the row so the totals still add up at the end. -4. Repeat only if it halts again. A restart that halts at the same point is telling you the cause is still there, and a restart that gets further has made real progress, because the rows it skipped are committed and will not be read again. +### Ingestion workers -**Or abandon it, on purpose.** Abandoning marks the category as deliberately given up rather than failed, which lets the host open and lets the migration end. It is the right answer when the data is not worth the outage, and the wrong one if it was picked by accident, because nothing goes back for an abandoned category afterwards. What it leaves behind is stated in the counts rather than guessed at. +- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, as before, because only SQL can scale out. +- **On SQL it starts only when every required category is Done or Abandoned**, whether or not its own migration setting is on. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. +- **That check runs even on a worker someone forgot to flag**, because ingesting into a part-copied database would make going back a loss. +- **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. +- **A worker or import that gets past the check marks the target as opened**, whether or not its own migration setting is on. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. -## Dry run +> [!IMPORTANT] +> **Start workers only after the main host has opened.** One started at the same moment as the first migration start can slip in before the copy has written its first checkpoint, and nothing stops it. -Runnable before anything starts, and again later against whatever is still outstanding. It never writes to RavenDB. +### Going back to RavenDB -What it resolves and reports: +**While ServiceControl is closed**, only the copier has written to SQL and the migration has written nothing to RavenDB, which is still the source of truth. This is the *free abort*. RavenDB's own expiration still runs, though, unless you disabled it. -- Whether the source is embedded or external, and which server -- Which two RavenDB databases, and the setting each name came from -- What it found in each of them -- Rows per category, and message-body volume per category -- A duration for the window while ServiceControl is closed, as a range +- **To go back:** set `ServiceControl/Migration/Enabled=false`, point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. +- **To try again later:** start from a new, empty SQL database (`--setup`). -It runs the same startup checks that gate startup, so a missing setting surfaces before a customer books an outage. +> [!IMPORTANT] +> Never reuse the old copy. It would skip every category already finished and miss anything RavenDB received since, and nothing can detect the reuse, because it looks exactly like a normal resume. +> +> **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. -If the source holds failed error imports, it says how many and advises running `--import-failed-errors` against RavenDB before the move. They are copied either way, but ones imported first arrive as ordinary failed messages rather than as imports still waiting. +**Turning migration off before every category is Done or Abandoned is refused**, not a quiet exit. That is what stops a migration ending by accident. -It counts, before anything moves, the rows the target says it would skip or merge, by reason. These include: +- **Every SQL start checks the checkpoint table, with migration on or off.** It costs nothing when nothing is outstanding, and a RavenDB instance never reaches it. +- **With nothing outstanding, nothing changes.** A SQL instance starts exactly as before, without opening RavenDB. +- **An optional category that never started and whose window is `0` is not outstanding.** It was turned off before it began. +- **With anything outstanding, the host refuses.** It names each category with its state, counts and last error, and the ways out: + - turn migration back on to finish, after `--migration-retry` for any Failed category; + - `--migration-abandon` what you are giving up on; + - point back at RavenDB, which is free only if ServiceControl never opened on SQL (the refusal says which); + - start over against a new, empty SQL database. -- Documents whose `UniqueMessageId` will not parse as a GUID -- Subscriptions that differ only in message-type version, and so merge onto one row -- Subscriptions whose message type or transport address exceeds the 200-character key limit -- Rows already past their retention period, and rows missing a value SQL requires +**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl has opened, nothing can start without it. After, ServiceControl still opens, and the optional categories wait for RavenDB to come back, or can be abandoned. -It reports no duration for the optional categories, and nothing about load on the source. +### The dry run -### When you can run the read-only commands +The dry run never writes to RavenDB, and can run again later against whatever is outstanding. It reports: -`--migration-source-report`, `--migration-verify` and `--migration-dry-run` all open the RavenDB source. **On an embedded source that means stopping the ServiceControl service first**, because a second RavenDB process cannot attach to a data directory the first one holds. Plan the dry run as part of the outage rather than as something you run the day before while the instance keeps serving traffic. On an external source, a container or RavenDB Cloud, all three run against a live instance with no interruption. +- Whether the source is embedded or external, and which server. +- Both RavenDB database names, and the setting each came from. +- Rows and body volume per category, counting only what is inside each window. +- **A range for how long ServiceControl will be closed. The range is a floor, not a promise:** it times counting the rows and sizing the bodies, not reading every row and body or writing to SQL, so the required copy can take longer. +- **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. +- The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. +- How many failed error imports there are, with advice to run `--import-failed-errors` on RavenDB first. They are copied either way, but ones imported first arrive as ordinary failed messages. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure. -A containerised instance runs all three as a one-off `docker run` of the same image with the command's flag, against an external RavenDB server, as the [instructions](ravendb-to-sql-migration-instructions.md#report-on-the-source) show for the source report. It cannot use an embedded source, because the image does not ship the RavenDB server. +It says nothing about load on the source. -`--migration-status` is the exception and is deliberately so: it reads only the checkpoint table in SQL and never opens the source, so it works on every source shape at any time, including during the background copy. It is the command to use for watching progress. +**When each read-only command can run:** -## Configuration and control +| Command | What it does | Opens RavenDB? | +| --- | --- | --- | +| `--migration-source-report` | Source facts and a document count per collection | Yes | +| `--migration-dry-run` | The report above | Yes | +| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read, with skips and merges broken out. Exits 0 only when every category is Done or Abandoned, so a script can ask whether the migration is finished. Rows the copy never read are shown, not failed | Yes | +| `--migration-status` | Each category's state, progress, last error and the commands it can take | No, so it runs any time | -- You set `MigrationMode`, and next to it whether to copy the one optional category, archived and resolved messages. A status command and a custom check report back. -- Categories are read fresh at every startup. Adding one copies it on the next restart, removing one deletes nothing. -- There is no HTTP API, no pause, no resume, no abort command, and no way to add a category to a running instance. All of those mean editing configuration and restarting. Going back during the closed window means stopping and reconfiguring, as [the one point you can go back](#the-one-point-you-can-go-back) describes. -- The checkpoint table is a record of what happened, not a control channel. -- Stopping a copy takes a restart, so it cannot be stopped in ten seconds. +- **On an embedded source, stop the ServiceControl service before any command that opens RavenDB.** A second RavenDB process cannot use a data directory the first one holds, so the dry run is part of the outage. +- **Verify reads every selected category in full**, though not the message bodies. On a big instance that takes a while, and on an embedded source that time is part of the outage too. +- **On an external source** they all run against a live instance. +- **In a container**, run each as a one-off `docker run` of the same image against an external RavenDB server. -## Out of scope +**What verify compares:** -- The audit instance, which has no EF Core persister at all, so a customer who finishes this migration is still running RavenDB for audit. This is stated up front under [Problem](#problem), because it changes whether the migration is worth doing at all -- The monitoring instance, which keeps its data in memory, so there is nothing to move +- **RavenDB against the rows the copy read, never against SQL as it is now.** Once ServiceControl opens, normal use changes most categories' SQL rows within minutes: ingestion, heartbeats, retries, archiving, the licensing collectors, ServiceControl's own custom checks and the retention sweep. A SQL count then measures that use, not the copy. +- **RavenDB holding more rows than the copy read means rows were never read.** Nothing writes to RavenDB once the copy starts, so it can only shrink. Verify flags these rows. They are not in SQL, so keep RavenDB. The same flag shows if something other than this migration wrote to RavenDB since, such as an old instance still running on it. +- **RavenDB holding fewer is shown, not judged.** RavenDB's expiry keeps deleting archived and resolved messages, event log items and a few unresolved messages from 6.18 or earlier, after the copy has read them. +- **The same expiry can hide rows that were never read**, because a deleted row and a missed row cancel out. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. +- **A windowed category is counted inside the window it started with**, on RavenDB only. +- **Verify also prints how many rows SQL held for each category when it settled.** Before ServiceControl opens nothing else writes, so for a required category that is exactly what the copy left in SQL. For the two optional categories it includes what ServiceControl wrote while they copied. + +### Monitoring and settings + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/Enabled` | `false` | Turns the migration on | +| `ServiceControl/Migration/EventLogWindow` | `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. `0` turns it off | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | `ServiceControl/ErrorRetentionPeriod` | How far back the archived and resolved copy goes. `0` turns it off | +| `ServiceControl/Migration/ThrottlePauseMilliseconds` | `100` | Pause between background batches | +| `ServiceControl/Migration/HaltThresholdPercent` | `5` | Percentage part of the early-stop threshold | +| `ServiceControl/Migration/HaltThresholdMinimum` | `100` | Floor part of the early-stop threshold | + +- **The source needs no new settings.** It reads the instance's existing RavenDB settings. +- **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. A start with migration still on after every category is Done or Abandoned logs a warning to turn it off. There is no migration custom check, because custom checks are being removed from ServiceControl. +- **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. +- **Stopping a copy takes a restart.** + +## Glossary + +- **Category:** one kind of data copied as a unit, such as known endpoints or the event log. +- **Required category:** one ServiceControl needs the moment it opens, so it copies while ServiceControl is closed. +- **Optional category:** one copied in the background after ServiceControl opens. It can be shortened or turned off before it starts. +- **Window:** how far back an optional category copies. +- **Closed period:** the time ServiceControl is not serving traffic because the required copy is running. +- **Checkpoint:** the row in SQL recording one category's progress, saved in the same transaction as the rows it describes. +- **Cursor:** the position in RavenDB the copy has reached in a category. +- **State:** one of Copying, Done, Failed or Abandoned. +- **Copying:** not finished yet, including waiting to start, or stopped on an error that the next start will try again. +- **Done:** every row is in SQL, or was left out for a harmless reason, and the counts balance. +- **Failed:** something went wrong. Stays Failed until you run `--migration-retry` or `--migration-abandon`. +- **Abandoned:** you gave up on what a category had not copied, or on one it depends on. Final. +- **Already present:** a row the copier found in SQL already, so it left it alone. +- **Fault skip:** a row lost because something went wrong. +- **Harmless skip:** a row left behind because SQL would have removed it anyway. +- **Free abort:** going back to RavenDB at no cost, possible only while ServiceControl is still closed. +- **Dry run:** a read-only rehearsal that reports what would move, what would be skipped and how long the outage would be. + +## Further reading + +- [The instructions](ravendb-to-sql-migration-instructions.md): how an operator runs the migration. +- [How the migration is put together](ravendb-to-sql-migration-system-design.md): which class does what, and what is built so far. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md new file mode 100644 index 0000000000..2e8d087297 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -0,0 +1,289 @@ +# Migration Engine: System Architecture Design + +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. + +[The overview](ravendb-to-sql-migration-overview.md) covers behaviour and guarantees. [The instructions](ravendb-to-sql-migration-instructions.md) cover the operator procedure. This page maps both onto types, call order and assembly boundaries. + +> [!IMPORTANT] +> Startup check order and category order are load-bearing. Read the required copy and category sections before reordering checks or adding a category. + +## Overview + +```mermaid +flowchart LR + subgraph host["ServiceControl host"] + req["RequiredCopyBeforeTheHostOpens
then MigrationStartup"] + opt["OptionalCategoryCopier"] + guard["EndOfMigrationGuard"] + cmd["Migration commands"] + end + subgraph neutral["ServiceControl.Persistence"] + eng["MigrationEngine"] + end + subgraph raven["ServiceControl.Persistence.RavenDB"] + src["RavenMigrationSource
IMigrationCategoryReader per category"] + end + subgraph ef["ServiceControl.Persistence.EFCore"] + tgt["EFCoreMigrationTarget
IMigrationCategoryWriter per category"] + cp["EFMigrationCheckpointStore"] + end + req --> eng + opt --> eng + guard --> cp + eng --> src + eng --> tgt + eng --> cp + cmd --> cp + cmd --> src + cmd --> tgt + src --> rdb[("RavenDB
read only")] + tgt --> sql[("SQL Server or PostgreSQL")] + cp --> sql + tgt --> bod[("Body store")] +``` + +`MigrationEngine` is store-agnostic. It drives `IMigrationSource` and `IMigrationTarget` per category and treats the cursor as an opaque string that it hands from source to target. Progress is persisted per category as a checkpoint row in the target database, committed in the same transaction as the rows it describes. Every command except `--migration-source-report` reads the checkpoint table. + +## Assemblies + +Dependencies point inward: each persister knows only its own store, the neutral assembly knows neither, and the host is the only composition root that names both. + +| Assembly | Types | Boundary rationale | +| --- | --- | --- | +| `ServiceControl.Persistence` | `MigrationEngine`, `MigrationBodyLoader`. Contracts: `IMigrationSource`, `IMigrationTarget`, `IMigrationCheckpointStore`, `IMigrationStartupCheck`, `IMigrationTargetReadiness`, `IMigrationSourceFactory`. Data: `MigrationCategory`, `MigrationCategoryKind`, `MigrationCategoryIds`, `MigrationCategoryRegistry`, `MigrationBatch`, `MigrationRow`, `MigrationBody`, `MigrationCheckpoint`, `MigrationWriteResult`, `MigrationSkipReason`, `MigrationSourceDescription`, `MigrationSourceFact`. Rules: `MigrationCategoryStateExtensions`, `MigrationSkipReasonExtensions`, `MigrationCheckpointRules`, `HaltThreshold`. Options: `MigrationEngineOptions`, `MigrationSettings`. `MigrationCheckpointConflictException`. Read-only views: `MigrationStatusView`, `MigrationVerification`, `MigrationDryRunArithmetic` | Referenced by both persisters and the host, references neither persister, so the engine is unit-tested against in-memory fakes in `ServiceControl.UnitTests/Migration` | +| `ServiceControl.Persistence.RavenDB` | `RavenMigrationSource`, `RavenReadOnlySourceLifecycle`, `RavenDocumentStream`, `IMigrationCategoryReader`, `IWindowedMigrationCategoryReader`, one reader per category | Owns document id prefixes, sessions and the embedded server | +| `ServiceControl.Persistence.EFCore` | `EFCoreMigrationTarget`, `EFCoreMigrationTargetReadiness`, `IMigrationCategoryWriter`, `IStoreKeyedCategoryWriter`, one writer per category, `PreparedBatch`, `IMigrationSqlDialect`, `MigrationInsert`, `StoreKeyedInsert`, `EFMigrationCheckpointStore`, `MigrationCheckpointExtensions`, `EFCoreMigrationRowAssessor`, and the three target readiness checks | Owns tables, column widths and provider parameter limits | +| `ServiceControl` (host) | `RequiredCopyBeforeTheHostOpens`, `RecordHostOpenedOnTarget`, `MigrationStartup` with nested `ClosedWindowProgress`, `MigrationStartupCheckRunner`, the four host checks, `OptionalCategoryCopier`, `RequiredStartSourceOutage`, `MigrationComponent`, `EndOfMigrationGuard`, `FinishedCopyBeforeAnIngestionNodeOpens`, `MigrationProgressReporter`, the six migration commands with `MigrationOperatorCommands` and `MigrationSourceDescriptionPrinter`, `PersistenceFactory.CreateMigrationSource` and `OpenMigrationSource` | Pair support is the one question neither persister can answer, so it lives here | + +Supporting assemblies: + +- `SqlServerMigrationSqlDialect` and `PostgreSqlMigrationSqlDialect` implement `IMigrationSqlDialect` and are the only provider-specific migration code. SQL Server reads key-column collations in `Open` and inserts with `MERGE ... WITH (HOLDLOCK)`, de-duplicating the batch with `ROW_NUMBER() OVER (PARTITION BY COLLATE )`. PostgreSQL uses `INSERT ... ON CONFLICT DO NOTHING RETURNING`. Both implement `SetLicensingEndpointThroughput`. Each provider assembly carries its own `AddMigrationCheckpoints` EF migration and model snapshot, so a checkpoint schema change needs a migration in both. +- `ServiceControl.DomainEvents` holds `MigrationStarted`, `MigrationCategoryHalted`, `MigrationFinished` and `MigrationSourceUnreachable`. + +The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadiness` or `IMigrationCheckpointStore`, so the guards below resolve `null` on RavenDB and return. `MigrationPairIsSupportedCheck` is what rejects RavenDB as a target. + +## What runs on every SQL instance + +These run whether or not `Migration/Enabled` is set. Each one closes a path to data loss or to a migration ending unnoticed. + +- **Registrations.** `BasePersistence.RegisterDataStores`, reached through each provider's `AddPersistence`, registers `IMigrationCheckpointStore` as `EFMigrationCheckpointStore`, `IMigrationTargetReadiness` as `EFCoreMigrationTargetReadiness` and `IMigrationTarget` as `EFCoreMigrationTarget`, all singletons. +- **`EndOfMigrationGuard`.** With `Migration/Enabled` off, `RunCommand` registers a hosted service in place of `RequiredCopyBeforeTheHostOpens` that runs `MigrationStartup.RunEndOfMigrationGuard` in `StartingAsync`, before any other hosted service and before Kestrel binds. It reads every checkpoint row and refuses if any category `IsOutstanding`. `IMigrationTargetReadiness.HasHostOpened` only selects the refusal text: whether going back to RavenDB is still free. With no checkpoint store registered (RavenDB) it returns immediately. +- **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import, flag on or off. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race it on the checkpoint version. +- **`RecordHostOpenedOnTarget` on workers.** Every host that passes the ingestion gate also registers `RecordHostOpenedOnTarget`, flag on or off, and stamps `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so the end-of-migration guard must never offer a free abort after a worker has written. + +The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row. + +> [!IMPORTANT] +> Workers must start only after the main host has opened. A worker started together with the first migration start can pass the gate before the first checkpoint row exists, and nothing enforces the order. + +## The required copy + +```mermaid +sequenceDiagram + participant Run as RunCommand + participant Copy as RequiredCopyBeforeTheHostOpens + participant Start as MigrationStartup + participant Checks as MigrationStartupCheckRunner + participant Source as RavenMigrationSource + participant Target as EFCoreMigrationTarget + participant Engine as MigrationEngine + + Run->>Run: registers Copy as a hosted service, then hostBuilder.Build() + Run->>Copy: StartingAsync, before any other hosted service starts + Copy->>Start: RunRequiredCopy(app.Services, settings) + Start->>Checks: MigrationPairIsSupportedCheck + Start->>Start: PersistenceFactory.CreateMigrationSource, not connected + Start->>Checks: RunChecksAndOpenTarget + Checks->>Target: Open + Start->>Start: IMigrationCheckpointStore.ReadAll + Start->>Source: Open + Start->>Checks: source.ContributedChecks(), empty for RavenDB + Start->>Engine: RunRequiredCategories under ClosedWindowProgress + Engine-->>Start: one checkpoint per category + Start->>Start: ReportWhatTheCopyLeftBehind + Start->>Start: RefuseIfAnyCategoryDidNotComplete + Start-->>Copy: returns, or throws and the host never opens + Note over Run: RecordHostOpenedOnTarget stamps the target in StartedAsync +``` + +Each position in the sequence is deliberate: + +1. **Inside host start.** `RunCommand` registers `RequiredCopyBeforeTheHostOpens` only when `Migration/Enabled` is true, before `hostBuilder.Build()`. `app.RunAsync` drives its `StartingAsync` ahead of every other hosted service, so the retention sweeper, heartbeat settings sync, throughput collectors and the API cannot run while a required category is unfinished. Running inside `RunAsync` also keeps the Windows Service Control Manager's 30-second start timeout satisfied. +2. **`MigrationPairIsSupportedCheck` first.** It needs no source, target or options. It resolves `PersistenceType` through `PersistenceManifestLibrary` and checks it against `PersistenceFactory.SqlPersistenceNames`; the source type is fixed at `PersistenceFactory.MigrationSourcePersistenceType`. RavenDB with the flag on is rejected here. +3. **`PersistenceFactory.CreateMigrationSource`** resolves the source persistence and hard-casts its configuration to `IMigrationSourceFactory`, implemented only by `RavenPersistenceConfiguration`. A non-implementing persister fails with `InvalidCastException`. No I/O happens yet. +4. **`MigrationStartup.RunChecksAndOpenTarget`**, the target half, in order: + - `OptionalCategoryWindowsAreValidCheck` runs `MigrationEngineOptions.FromSettings`, refusing a window that is not a non-negative `TimeSpan` and naming the key. A zero window removes the category from `SelectedOptionalCategoryIds`. + - `RetryHistoryDepthIsSafeCheck` is host-owned because it needs only `Settings.RetryHistoryDepth`, which keeps that value off `PersistenceSettings` and out of `/api/configuration`. + - `EFCoreMigrationTargetReadiness.ContributedChecks`: `SchemaIsCurrentCheck`, `TargetHoldsNoServiceControlDataCheck`, `BodyStorageIsWritableCheck`. The body probe is deleted after writing, so it does not skew body counts. + - `IMigrationTarget.Open`, where the SQL Server dialect loads key-column collations. + - `OptionalCategoryWindowsAreUnchangedCheck`, last because it reads checkpoint rows. It refuses a started, unfinished optional category whose configured window differs from `StartedWindowSeconds`. + + `MigrationStartupCheckRunner` short-circuits on the first refusal. +5. **`TargetHoldsNoServiceControlDataCheck`** passes immediately if a checkpoint row exists for any required category. Otherwise it refuses if any mapped table other than `MigrationCheckpoints` has a row, `Settings` included, naming tables by `GetTableName()` (`endpoint_settings`, `settings` on PostgreSQL). +6. **Source open, last.** It is the only step that can spawn a process. `RunRequiredCopy` reads the checkpoint rows and builds the engine first. `RavenReadOnlySourceLifecycle.Open` then: + - starts the embedded server, if configured; + - connects: on an external server it rejects an expired or not-yet-valid client certificate and an `https://` URL without one, then attaches `RefuseWrite` to the request pipeline before `Initialize`, so the version request is covered; + - checks an external server's version against the client; + - waits for both databases, with a 5-minute budget per database on an embedded server. The budget is only checked when RavenDB raises its own timeout, so the wait can exceed it. + + > [!IMPORTANT] + > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexCreation`. +7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the copy is skipped and the host opens. `OptionalCategoryCopier` then stays idle for that start. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. +8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. +9. **`MigrationStartup.RunRequiredCategories`** upserts `NotStarted` rows, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. +10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. +11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. +12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row as harmless. +13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. +14. **`RecordHostOpenedOnTarget`** is registered beside `RequiredCopyBeforeTheHostOpens` with the flag on. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. + +## The background copy + +`OptionalCategoryCopier` is a `BackgroundService` registered by `MigrationComponent` only with the flag on and only on the main instance; `--error-ingestion-only` passes its own component list. + +- Every 20 seconds it runs the optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. +- It opens its own source through `PersistenceFactory.OpenMigrationSource`, only when a category is still copying. On failure it calls `RecordSourceOutage` and idles for the rest of the start. It also idles when the required start already recorded an outage, which it learns from the in-process `RequiredStartSourceOutage` singleton (`Recorded`, `Error`) that `RunRequiredCopy` sets where it calls `RecordSourceOutage`. The checkpoint rows cannot carry this signal, because an optional row's `LastError` can survive from an earlier start. +- An exception leaves the category `InProgress` with `LastError`, untouched until the next start, so a transient failure never forces an abandon. + +Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. + +> [!IMPORTANT] +> Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. + +`Migration/ThrottlePauseMilliseconds` inserts a delay between optional-category batches, skipping the first batch of each category. Required categories are never throttled. + +## Copying a batch + +`MigrationEngine.RunCategoryAsync` is the per-category loop. Every store-specific step is delegated. + +| Step | Owner | Rationale | +| --- | --- | --- | +| Load the checkpoint | `EFMigrationCheckpointStore.Read` | The table lives in the target database | +| Short-circuit terminal or Failed rows | `MigrationCategoryStateExtensions.IsFinished`, `IsFailed` | Single definition shared by every caller; the source is not read | +| Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | +| Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | +| Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; single-document readers load by id; licensing settings readers use `LicensingSettingsSource` | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen | +| Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyUnreadable` after one attempt, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | +| Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold, so a crash leaves orphan bodies, never dangling references | +| Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `SetLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips and merges are logged after commit, and `Extend` adds the batch's `PreparedBatch.Merges` to `MergedCount`. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | +| Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | +| Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | +| Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | + +Body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageRowMapper.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. + +Invariants enforced across class boundaries: + +- `MigrationCheckpoint.Extend` throws unless the per-`MigrationSkipReason` counts sum exactly to the skipped count. `EFCoreMigrationTarget.AccountForEveryRow` throws unless prepared rows plus skips equal the batch size, catching rows dropped during preparation. `AlreadyPresentIn` throws when a writer over-reports. Every row is copied, skipped or already present. +- `MigrationSkipReasonExtensions.IsBenign` is fixed: `PastRetention`, `BlankGroupComment`. Everything else is a fault, `Unknown` included. +- `PreparedBatch.Merges` reports rows whose key collides with another under the target's collation. `EndpointSettingsWriter` detects them with `IMigrationSqlDialect.KeyComparer` against the batch and the existing table, and `EFCoreMigrationTarget` logs one warning per merge naming both keys and the one kept. The dropped row counts as already present, and also in `MergedCount`, the part of already present that merges explain. Every writer whose key can fold two source rows into one reports them here, or `MergedCount` understates. +- `TargetCountAtSettle` is read with `IMigrationTarget.Count` just before the save that settles a category at the end of the source, or that abandons it, and is never updated afterwards. For a required category it is exactly what the copy left in SQL: the host is closed, `TargetHoldsNoServiceControlDataCheck` started the target empty, and a second copier would move the row's version so the settle save is refused. That is why the count needs no shared transaction with the save. For an optional category it also includes what ServiceControl wrote while the copy ran. + +## Category states + +`MigrationCategoryState` has seven stored values, surfaced as four operator states: + +| Operator state | Stored | Meaning | +| --- | --- | --- | +| Copying | `NotStarted`, `InProgress`, `Blocked` | Not terminal, including a follower waiting on its predecessor and an optional category stopped by an error the next start retries | +| Done | `Complete` | Every row copied or benignly skipped, counts reconciled | +| Failed | `Halted`, `CompleteWithErrors` | Stays Failed until `--migration-retry` or `--migration-abandon` | +| Abandoned | `Abandoned` | Terminal, set by the operator directly or by cascade | + +The required-copy refusal prints both forms for a Failed row (`EndpointSettings is Failed (Halted)`) and the stored state alone for a Copying row. Retry and abandon refusals print the operator state only. + +Early stops during a run: + +- Threshold exceeded or post-commit mismatch: `Halted`, either kind. +- Required category exception or 30-minute stall: `Halted`. +- Optional category exception: stays `InProgress` with `LastError`. + +Settlement at end of source, in order: + +1. Rows read this run differ from rows processed: `Halted`. +2. Any fault skip in the checkpoint: `CompleteWithErrors`. +3. Otherwise: `Complete`. + +`MigrationCategoryStateExtensions.IsFinished` (`Complete` or `Abandoned`) and `IsFailed` (`Halted` or `CompleteWithErrors`) are the single definitions, used by the engine, the required-copy refusal, the background copier, the end-of-migration guard, the ingestion gate, status, verify and both operator commands. The host opens when every required category `IsFinished`; the migration is finished when every category is. + +`MigrationCheckpointRules` holds the rules shared by the guard, the commands and status: + +- `IsOutstanding`: not finished, except an optional `NotStarted` row whose window is `0`. +- `CanBeRetried`: an `IsFailed` row, or an optional `NotStarted` or `InProgress` row with `LastError` (which covers `RecordSourceOutage` rows). Never `Blocked` or a missing row. `MigrationOperatorCommands.Retry` rejects `EventLog` before consulting it. +- `CanBeAbandoned`: a required row that `IsFailed` or is `InProgress` with copied plus skipped plus already-present above 0, treating an id unknown to the registry as required; an optional category in any state except `Complete` or `Abandoned`, including no row. + +Checkpoint schema: one row per category in `MigrationCheckpoints` (`migration_checkpoints` on PostgreSQL), primary key `CategoryId`. Columns: `State`, `Cursor`, `CopiedCount`, `SkippedCount`, `AlreadyPresentCount`, `MergedCount`, `SkipReasons` (JSON keyed by enum name; unknown names deserialize to `Unknown`, a fault), `StartedAt`, `LastProgressAt`, `SettledAt`, `LastError`, `StartedWindowSeconds`, `TargetCountAtSettle` (null until the category first settles) and `Version`, the optimistic concurrency token. No attempt counter and no decision record. `AddMigrationCheckpoints` creates the table in both providers and has not been released, so `TargetCountAtSettle` replaces the never-written `SourceTotal` column inside it, and `MergedCount` is added there too, rather than in a second migration. Run-level facts are `Settings` rows under `Migration/`, such as `Migration/HostOpenedOnTarget`. + +## Commands + +Every migration command except the source report builds a host with `AddPersistence` and never calls `StartAsync`, so no hosted service runs, including the retention sweeper and the integration event dispatcher. The source report builds no host and calls `PersistenceFactory.OpenMigrationSource` directly. + +| Command | Class | Opens RavenDB | Reads and writes | +| --- | --- | --- | --- | +| `--migration-source-report` | `MigrationSourceReportCommand` | Yes | `IMigrationSource.Describe` and `Inventory` through `MigrationSourceDescriptionPrinter`; the copy never calls either. Each `MigrationSourceFact` carries its originating setting key. `Inventory` covers every collection, including uncopied ones | +| `--migration-dry-run` | `MigrationDryRunCommand`, `MigrationDryRunArithmetic` | Yes | `MigrationPairIsSupportedCheck`, then `MigrationStartup.RunChecksAndOpen` (both halves), so it refuses wherever a real start would. Reads `Count`, `Read`, `ReadBody`, `ReadBodyVolume`, `Inventory`, `Describe`, `IMigrationTarget.Assess` (via `EFCoreMigrationRowAssessor`) and `BatchSizeFor`. Writes no row; the body probe is the only write. Exit code 0 even when the report shows problems; a refused check throws | +| `--migration-status` | `MigrationStatusCommand`, `MigrationStatusView` | No | `IMigrationCheckpointStore.ReadAll` and `MigrationEngineOptions.FromSettings`. Read-only, safe while ServiceControl runs | +| `--migration-verify` | `MigrationVerifyCommand`, `MigrationVerification` | Yes | `ReadAll`, `FromSettings`, then `IMigrationSource.Count` per selected category, judged against the rows the copy read (below). A windowed category is counted from `CopiesFrom` set to `StartedAt` minus `StartedWindowSeconds`, the window the copy used, on the source side only. Never calls `IMigrationTarget.Count`. Streams every selected category, though not the bodies, so on an embedded source ServiceControl is stopped first. Exit code 1 only when a category `IsOutstanding`; rows never read on a Done category are reported, not failed | +| `--migration-retry ` | `MigrationRetryCommand`, `MigrationOperatorCommands` | No | `Retry` checks `CanBeRetried` and builds the reset row; `IMigrationTarget.ResetForRetry` persists it | +| `--migration-abandon ` | `MigrationAbandonCommand`, `MigrationOperatorCommands` | No | Checks `CanBeAbandoned`, then writes `Abandoned` rows through the checkpoint store only | + +`MigrationOperatorCommands.Retry` builds the reset row without I/O: state `NotStarted`; `Cursor`, the four counts, `SkipReasons`, `SettledAt`, `TargetCountAtSettle` and `LastError` cleared; `StartedAt` and `StartedWindowSeconds` kept. Change retry semantics there. `ResetForRetry` only persists the row it receives, in one transaction under the execution strategy. Store-keyed categories cannot be re-read idempotently, so their writers implement `IStoreKeyedCategoryWriter.DeleteCopiedRows` and the reset deletes their rows in the same transaction: + +- `RetryOperationsWriter`: `HistoricRetryOperations`, then `UnacknowledgedRetryOperations`. +- `PendingIntegrationEventsWriter`: `ExternalIntegrationDispatchRequests`. + +Both are required categories, so the host has not opened and the target started empty. `EventLog` is also store-keyed but copies after the host opens, when its rows are indistinguishable from live ones, so it is abandon-only. + +`--migration-abandon` first re-saves the named row unchanged, when it exists, to bump its version and fence off a concurrent copy. It then abandons dependants along `MigrationCategory.MustFollowIsDataLink` (`EndpointSettings` after `KnownEndpoints`, `LicensingThroughput` after `LicensingEndpoints`), deepest first, skipping `Complete` and `Abandoned` dependants and inserting `Abandoned` rows for dependants without one, and saves the named row last. `GroupComments` follows the unresolved failed messages for ordering only and is not cascaded. An optional category with no row gets a new `Abandoned` row. Every row it saves `Abandoned` carries `TargetCountAtSettle`, read just before the save, as a settle does. No domain event is raised, because ServiceControl is stopped. + +Both operator commands rely on `MigrationCheckpointConflictException` to detect a race with a running copy. + +`MigrationVerification` judges one thing per category: RavenDB now against the rows the copy read, `CopiedCount` plus `SkippedCount` plus `AlreadyPresentCount`. That sum is exactly the rows the last complete pass read, body skips included: `RunCategoryAsync` settles `Halted` when the rows a run read differ from the rows the target accounted for, a crash resume adds to the same totals, and a retry clears them. Once the source opens RavenDB only shrinks, because `RefuseWrite` refuses every client write and only RavenDB's expiry deletes. + +- **RavenDB higher than the sum** is rows the copy never read. Verify flags them, by category. +- **RavenDB lower than the sum** is rows RavenDB removed after the copy read them, normal for the event log, the archive and a few unresolved messages with a stale expiry. Printed, not judged. +- **Expiry can hide the same number of unread rows** in the categories it touches, so a flagged number is a floor. +- **The count streams the same reader the copy used** (`RavenMigrationSource.Count`), so verify finds rows one run missed, such as a resume that skipped a range or a stream that ended early. It cannot find a reader that always misses the same rows, or a source pointed at the wrong database. +- **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `MergedCount` and `TargetCountAtSettle` are printed beside the sum instead. +- **An abandoned category's sum covers its last pass only.** Rows an earlier pass copied before a retry are in `TargetCountAtSettle` but not in the sum, so RavenDB minus the sum overstates what was never copied. + +**Rows never read on a Done category are accepted, not fixed.** Verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before the host opens: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. + +## Readers and writers + +Readers live at `ServiceControl.Persistence.RavenDB/DataMigration/Readers/Reader.cs` and are keyed in `RavenMigrationSource`'s reader dictionary. Writers live at `ServiceControl.Persistence.EFCore/DataMigration/Writers/Writer.cs` and are keyed in the `FrozenDictionary` initialised in `EFCoreMigrationTarget`. `MigrationCategoryCoverageTests` asserts that every registry category has both. + +Every reader derives from `MigrationCategoryReader` and every writer from `MigrationCategoryWriter`, and the two for one category take the same `TDocument`, the type of each row's `Document`. The reader base streams whole documents from the primary database through `WholeDocuments`, and exposes `Lifecycle` to a reader that projects or reads the throughput database itself. The writer base casts every row once and hands `PrepareDocuments` the typed documents, so a row of another type fails the batch with an `InvalidCastException` naming the category, the row and both types. Four tests hold this: `MigrationCategoryCoverageTests.Every_reader_and_its_writer_agree_on_the_document_type`, `MigrationCategoryReadersTests.Every_reader_derives_from_the_typed_base`, `MigrationCategoryWritersTests.Every_writer_derives_from_the_typed_base` and `MigrationCategoryWritersTests.Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type`. + +| Order | Category | Kind | Reader, writer | Notes | +| --- | --- | --- | --- | --- | +| 1 | `KnownEndpoints` | Required | `KnownEndpointsReader`, `KnownEndpointsWriter` | | +| 2 | `EndpointSettings` | Required | `EndpointSettingsReader`, `EndpointSettingsWriter` | Data link to `KnownEndpoints`. Copies every setting, including one for an unknown endpoint, which `HeartbeatEndpointSettingsSyncHostedService` removes once the host opens. Reports case merges on a case-insensitive name column | +| 3 | `MessageRedirects` | Required | `MessageRedirectsReader`, `MessageRedirectsWriter` | | +| 4 | `Subscriptions` | Required | `SubscriptionsReader`, `SubscriptionsWriter` | Rows differing only in message-type version merge | +| 5 | `NotificationSettings` | Required | `NotificationSettingsReader`, `NotificationSettingsWriter` | One `Settings` row via `SettingRowMapper` | +| 6 | `TrialEndDate` | Required | `TrialEndDateReader`, `TrialEndDateWriter` | One `Settings` row via `SettingRowMapper` | +| 7 | `RetryOperations` | Required | `RetryOperationsReader`, `RetryOperationsWriter` | Two tables: historic rows store-keyed via `StoreKeyedInsert`, unacknowledged rows via `InsertMissing`, which sizes the batch. Deletes its rows on retry | +| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database | +| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. `SetLicensingEndpointThroughput` sets each day's count rather than incrementing, so re-reads are idempotent | +| 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | +| 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageRowMapper` | +| 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | +| 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | | +| 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | +| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment` | +| 1 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | +| 2 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | + +## Progress reporting + +There is no migration custom check: custom checks are being removed from ServiceControl, so progress uses the activity feed, the log and `--migration-status` only. + +- `MigrationProgressReporter` raises `MigrationStarted`, `MigrationCategoryHalted` (for both Failed states), `MigrationFinished` (carrying `CategoriesAbandoned`) and `MigrationSourceUnreachable`, mapped into the ServicePulse activity feed by their `EventLogMappingDefinition` classes. It logs and swallows publish failures but propagates cancellation. All four are raised from `OptionalCategoryCopier`, so they cover the background copy only, and a Failed required category produces no activity-feed entry; its record is the startup refusal and the log. +- `MigrationSourceUnreachable` is raised once per start when `OptionalCategoryCopier` idles on a source outage, its own failed open or one the required start recorded, carrying the error and the optional categories left waiting. +- On its first pass, `OptionalCategoryCopier` logs a warning to set `Migration/Enabled` to `false` and restart when `MigrationCheckpointRules.FinishedButStillEnabled` holds: no registry category `IsOutstanding`. `--migration-status` prints the same line. `IsOutstanding` rather than `IsFinished`, so an optional category turned off with a `0` window does not hold the warning back. +- `ClosedWindowProgress` logs running required categories every 30 seconds while the host is closed. +- `--migration-status` reports state, progress, last error and available commands per category at any time, and whether migration can be turned off. diff --git a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj index 718b22bc78..a55fcba55b 100644 --- a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj +++ b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj @@ -12,6 +12,9 @@ + + + @@ -20,6 +23,7 @@ + @@ -31,6 +35,9 @@ + + + diff --git a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj index cbc66d5b4a..0d67353174 100644 --- a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj +++ b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj @@ -12,6 +12,9 @@ + + + @@ -20,6 +23,7 @@ + @@ -31,6 +35,9 @@ + + + diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs index 58cef8074f..917700bea5 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs @@ -29,10 +29,9 @@ namespace ServiceControl.AcceptanceTests.Recoverability using ServiceControl.Infrastructure; using ServiceControl.MessageFailures; using ServiceControl.Operations; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; - using ServiceControl.Persistence.EFCore.Infrastructure; - using ServiceControl.Recoverability; using ServiceControl.Transports; class When_hosting_error_ingestion_only : AcceptanceTest @@ -98,9 +97,36 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured() "InternalCustomChecksHostedService", // reports this node's ingestion health to the database "MetricsReporterHostedService", "HealthCheckPublisherHostedService", // inert, no IHealthCheckPublisher is registered - "ExternalIntegrationRequestsDataStore" // its drain is inert here, nothing calls Subscribe + "ExternalIntegrationRequestsDataStore", // its drain is inert here, nothing calls Subscribe + "FinishedCopyBeforeAnIngestionNodeOpens" // keeps this node out of a database a copy has not finished filling ]; + [Test] + public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished() + { + var settings = await CreateSettings(); + + await new SetupCommand().Execute(new HostArguments([]), settings); + + var host = ErrorIngestionOnlyCommand.BuildHost(settings); + + try + { + await host.Services.GetRequiredService().Upsert( + new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, "The source stopped responding.")); + + var exception = Assert.ThrowsAsync(() => host.StartAsync()); + + Assert.That(exception.Message, Does.Contain("has not finished") + .And.Contain(MigrationCategoryIds.KnownEndpoints) + .And.Contain("--migration-abandon KnownEndpoints")); + } + finally + { + await host.DisposeAsync(); + } + } + [Test] public void Should_refuse_to_start_against_unsupported_storage() { @@ -205,6 +231,7 @@ public async Task Should_serve_health_over_https_with_the_configured_certificate await new SetupCommand().Execute(new HostArguments([]), settings); host = ErrorIngestionOnlyCommand.BuildHost(settings); + host.Urls.Clear(); host.Urls.Add("https://127.0.0.1:0"); await host.StartAsync(); diff --git a/src/ServiceControl.Migration.AcceptanceTests/.editorconfig b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig new file mode 100644 index 0000000000..da44eb13fb --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig @@ -0,0 +1,6 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0013.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs new file mode 100644 index 0000000000..adf724538b --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -0,0 +1,472 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using NServiceBus.Extensibility; +using NServiceBus.Transport; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.AcceptanceTesting.InfrastructureConfig; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure.WebApi; +using ServiceControl.Migration.Checks; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.Tests; +using ServiceControl.Persistence.UnitOfWork; +using TestHelper; +// Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. +using IDocumentStore = Raven.Client.Documents.IDocumentStore; + +// No base class: the persistence test bases sit in projects that cannot see RavenDB. +abstract class MigrationAcceptanceTest +{ + protected const string EndpointSettingsUrl = "api/endpointssettings"; + + readonly AcceptanceTestStorageConfiguration StorageConfiguration = new(); + readonly List setVariables = []; + CancellationTokenSource hostCancellation; + Task runningHost; + + ServiceProvider targetServices; + IMigrationCheckpointStore targetCheckpointStore; + + protected Settings Settings { get; private set; } + protected HttpClient HttpClient { get; private set; } + protected (string ServerUrl, string PrimaryDatabase, string ThroughputDatabase) Source { get; private set; } + protected IDocumentStore SourceStore { get; private set; } + + protected IMigrationTarget Target { get; private set; } + + protected InMemoryBodyStoragePersistence RecordedBodies { get; } = new(); + + [SetUp] + public async Task SetUp() + { + Source = await MigrationSourceServer.CreateDatabases(); + SourceStore = await (await MigrationSourceServer.GetInstance()).Connect(); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", Source.ServerUrl); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", Source.PrimaryDatabase); + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", Source.ThroughputDatabase); + SetSourceVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + + // Settings.Port has no public setter, so the port has to come in as the environment variable a customer would set. + SetSourceVariable("SERVICECONTROL_PORT", PortUtility.GetAssignedOrAvailablePort(33500).ToString(CultureInfo.InvariantCulture)); + + var transport = new ConfigureEndpointLearningTransport(); + + Settings = new Settings( + transportType: transport.TypeName, + persisterType: StorageConfiguration.PersistenceType, + forwardErrorMessages: false, + errorRetentionPeriod: TimeSpan.FromDays(10)) + { + TransportConnectionString = transport.ConnectionString + }; + + await StorageConfiguration.CustomizeSettings(Settings); + await new SetupCommand().Execute(new HostArguments([]), Settings); + + HttpClient = new HttpClient { BaseAddress = new Uri(Settings.RootUrl) }; + + var targetServiceCollection = new ServiceCollection(); + targetServiceCollection.AddLogging(); + targetServiceCollection.AddPersistence(Settings); + targetServiceCollection.AddSingleton(RecordedBodies); + targetServices = targetServiceCollection.BuildServiceProvider(); + + Target = targetServices.GetRequiredService(); + targetCheckpointStore = targetServices.GetRequiredService(); + await Target.Open(); + } + + // The variables go first because they are process wide: a failure in the cleanup below must not leave them set for the next test. + [TearDown] + public async Task TearDown() + { + foreach (var name in setVariables) + { + Environment.SetEnvironmentVariable(name, null); + } + + try + { + await StopHost(); + } + finally + { + HttpClient?.Dispose(); + SourceStore?.Dispose(); + + if (targetServices is not null) + { + await targetServices.DisposeAsync(); + } + + await StorageConfiguration.Cleanup(); + } + } + + protected void SetSourceVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + setVariables.Add(name); + } + + protected async Task SeedSource(string database, params (string Id, object Document)[] documents) + { + using var session = SourceStore.OpenAsyncSession(database); + + foreach (var (id, document) in documents) + { + await session.StoreAsync(document, id); + } + + await session.SaveChangesAsync(); + } + + protected Task SeedSource(params (string Id, object Document)[] documents) => + SeedSource(Source.PrimaryDatabase, documents); + + protected Task SeedSourceEndpointSettings(params (string Name, bool TrackInstances)[] settings) => + SeedSource([.. settings.Select(setting => ($"EndpointSettings/{DeterministicGuid.MakeId(setting.Name)}", (object)new EndpointSettings { Name = setting.Name, TrackInstances = setting.TrackInstances }))]); + + // The target copies an endpoint setting only when that endpoint is known, so seed the endpoint too. + protected Task SeedSourceKnownEndpoints(params string[] names) => + SeedSource([.. names + .Select(name => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" } }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint))]); + + protected Task OpenSource(CancellationToken cancellationToken = default) => + PersistenceFactory.OpenMigrationSource(Settings, cancellationToken); + + static IReadOnlyCollection sourceSupportedCategoryIds; + + // SupportedCategoryIds answers before Open and cannot change across it, so one source built once for the + // whole run answers it for every test instead of loading the persister assembly again per test. + protected async Task> SourceSupportedCategoryIds() + { + if (sourceSupportedCategoryIds is null) + { + await using var source = PersistenceFactory.CreateMigrationSource(Settings); + sourceSupportedCategoryIds = source.SupportedCategoryIds; + } + + return sourceSupportedCategoryIds; + } + + + // No build carries the whole migration until its last phase ships, so without this marker every copy is refused. + protected static Action AllowingAnUnreleasedMigration(Action customize = null) => + builder => + { + builder.Services.AddSingleton(); + customize?.Invoke(builder); + }; + + protected async Task RunHostUntilTheApiAnswers(Action customize = null) + { + await StopHost(); + + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + hostCancellation = new CancellationTokenSource(); + runningHost = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, customize)), hostCancellation.Token); + + await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + + // The API answers before StartedAsync runs, and a stop before StartedAsync finishes cancels the start, which RunCommand rightly treats as a failure. + await await Task.WhenAny(started.Task, runningHost).WaitAsync(TimeSpan.FromMinutes(1)); + } + + // ApplicationStarted fires only after every hosted service's StartedAsync, which is where the marker is written. + protected static Action SignallingOnceStarted(TaskCompletionSource started, Action customize) => + builder => + { + builder.Services.AddHostedService(provider => new SignalWhenStarted(provider.GetRequiredService(), started)); + customize?.Invoke(builder); + }; + + sealed class SignalWhenStarted(IHostApplicationLifetime lifetime, TaskCompletionSource started) : IHostedService + { + public Task StartAsync(CancellationToken cancellationToken = default) + { + lifetime.ApplicationStarted.Register(() => started.TrySetResult()); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + protected Settings SettingsWithMigrationEnabled(bool copyEventLog = false, Action customize = null) + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", copyEventLog ? null : "0"); + SetSourceVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "0"); + customize?.Invoke(Settings); + return Settings; + } + + protected async Task RunRequiredCopy( + Action customize = null, + bool leaveOptionalIncomplete = false, + CancellationToken cancellationToken = default) + { + SettingsWithMigrationEnabled(copyEventLog: leaveOptionalIncomplete); + + await RunHostUntilTheApiAnswers(builder => customize?.Invoke(builder.Services)); + + var copyable = MigrationStartup.CopyableCategoryIds(await SourceSupportedCategoryIds(), Target.SupportedCategoryIds); + + var copied = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Required && copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + await WaitUntil( + async () => (await targetCheckpointStore.ReadAll(cancellationToken)) + .Count(checkpoint => copied.Contains(checkpoint.CategoryId) && checkpoint.State.IsFinished()) == copied.Length, + "the required copy finished"); + + if (leaveOptionalIncomplete) + { + // The guarded services read IMigrationState.AnyCategoryIncomplete, which any selected category that has not finished holds true. Halted is one of those states. + await targetCheckpointStore.Upsert( + new MigrationCheckpoint("EventLog", MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, + "Halted by the test fixture so the guarded services see an unfinished category."), + cancellationToken); + } + } + + protected async Task> WaitForEndpointSettingsResponse(TimeSpan timeout) + { + IReadOnlyList settings = null; + + await WaitUntil(async () => + { + // A host that refused to start surfaces its own exception here instead of waiting out the timeout. + if (runningHost is { IsFaulted: true }) + { + await runningHost; + } + + try + { + settings = await GetEndpointSettings(); + return true; + } + catch (HttpRequestException) + { + return false; + } + }, $"GET {EndpointSettingsUrl} answered", timeout); + + return settings; + } + + protected async Task> GetEndpointSettings() => + await HttpClient.GetFromJsonAsync>(EndpointSettingsUrl, SerializerOptions.Default); + + async Task StopHost() + { + if (runningHost is null) + { + return; + } + + await hostCancellation.CancelAsync(); + await runningHost; + hostCancellation.Dispose(); + runningHost = null; + } + + // Copied from PersistenceTestBase.WaitUntil, which this project cannot compile. + protected static async Task WaitUntil(Func> conditionChecker, string condition, TimeSpan timeout = default) + { + timeout = timeout == default ? TimeSpan.FromSeconds(10) : timeout; + + var start = DateTime.UtcNow; + + while (DateTime.UtcNow - start < timeout) + { + if (await conditionChecker()) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(500)); + } + + throw new Exception($"{condition} has not been meet in defined timespan: {timeout})"); + } + + // Copied from RavenMigrationSourceTestBase.CollectBatches, which this project cannot compile. + protected static async Task> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } + + protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService(); + protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService(); + + protected async Task CopyCategory(string categoryId, CancellationToken cancellationToken = default) + { + var category = MigrationCategoryRegistry.Find(categoryId) ?? throw new ArgumentException($"'{categoryId}' is not a migration category.", nameof(categoryId)); + var before = await targetCheckpointStore.Read(categoryId, cancellationToken); + var target = new RecordingMigrationTarget(Target); + var options = new MigrationEngineOptions(TimeSpan.Zero, MigrationSettings.DefaultHaltThresholdPercent, MigrationSettings.DefaultHaltThresholdMinimum, []) { BodyRetryBackoff = TimeSpan.Zero }; + + await using var source = await OpenSource(cancellationToken); + var engine = new MigrationEngine(source, target, targetCheckpointStore, TimeProvider.System, options, NullLogger.Instance); + + var after = await engine.RunCategoryAsync(category, cancellationToken); + + // Both reached the end of the source, so a test can assert a Failed category's skips as well as a Done one's. + if (after.State is not (MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors)) + { + throw new InvalidOperationException($"Copying '{categoryId}' ended {after.State}: {after.LastError}"); + } + + return new MigrationWriteResult( + after, + (int)(after.CopiedCount - (before?.CopiedCount ?? 0)), + (int)(after.SkippedCount - (before?.SkippedCount ?? 0)), + target.SkippedIds, + (int)(after.AlreadyPresentCount - (before?.AlreadyPresentCount ?? 0))); + } + + + // Makes the target look like a database a migration has touched, which is the only state the host-opened marker is stamped in. + protected async Task SeedCheckpoint(string categoryId, MigrationCategoryState state = MigrationCategoryState.Complete) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.UpsertCheckpoint(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + } + + protected async Task DeleteCheckpoint(string categoryId) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.MigrationCheckpoints.Where(checkpoint => checkpoint.CategoryId == categoryId).ExecuteDeleteAsync(); + } + + protected async Task QueryTarget(Func> query) + { + using var scope = targetServices.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + return await query(dbContext); + } + + protected async Task GetFailedMessage(Guid uniqueMessageId) + { + var row = await QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + Assert.That(row, Is.Not.Null, $"No failed message row for {uniqueMessageId}"); + + return row; + } + + protected Task FindFailedMessage(Guid uniqueMessageId) => + QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + protected EFPersisterSettings EFSettings => targetServices.GetRequiredService(); + + protected Task Ingest(params IngestedFailure[] failures) => + InBatch(async unitOfWork => + { + foreach (var failure in failures) + { + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups); + } + }); + + // Ingestion takes the body and its content type from the message context, so only the context changes. + protected Task IngestWithBody(IngestedFailure failure, byte[] body, string contentType) => + InBatch(unitOfWork => + { + var headers = new Dictionary(failure.Headers) { [NServiceBus.Headers.ContentType] = contentType }; + var context = new MessageContext(failure.MessageId, headers, body, new TransportTransaction(), "receiveAddress", new ContextBag()); + + return unitOfWork.Recoverability.RecordFailedProcessingAttempt(context, failure.ProcessingAttempt, failure.Groups); + }); + + async Task InBatch(Func record) + { + await using var unitOfWork = await targetServices.GetRequiredService().StartNew(); + + await record(unitOfWork); + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + } + + protected Task ReadCheckpoint(string categoryId) => + QueryTarget(async dbContext => + { + var row = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(checkpoint => checkpoint.CategoryId == categoryId); + + return row is null ? null : new MigrationCheckpoint( + row.CategoryId, row.State, row.Cursor, + row.CopiedCount, row.SkippedCount, row.SourceTotal, row.SkipReasons, + row.StartedAt, row.LastProgressAt, row.SettledAt, row.LastError, + row.AlreadyPresentCount, row.Version); + }); + + protected Task ReadSetting(string key) => + QueryTarget(dbContext => dbContext.Settings.AsNoTracking() + .Where(setting => setting.Key == key) + .Select(setting => setting.Value) + .SingleOrDefaultAsync()); + + sealed class RecordingMigrationTarget(IMigrationTarget inner) : IMigrationTarget + { + public List SkippedIds { get; } = []; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var result = await inner.Write(category, batch, checkpointToExtend, cancellationToken); + SkippedIds.AddRange(result.SkippedIds); + return result; + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs new file mode 100644 index 0000000000..88a33b0003 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.RavenDB; +using TestHelper; + +// Its own copy rather than SharedEmbeddedServer: that one uses the RavenDB persister's own settings type, which this project cannot reference because it loads the persister from its manifest instead. +static class MigrationSourceServer +{ + public static async Task GetInstance(CancellationToken cancellationToken = default) + { + await startLock.WaitAsync(cancellationToken); + try + { + if (server != null) + { + return server; + } + + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "MigrationSource"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "MigrationSource"); + var port = PortUtility.GetAssignedOrAvailablePort(33350); + + var configuration = new EmbeddedDatabaseConfiguration($"http://localhost:{port}", "primary", dbPath, logPath, "Operations") { RunInMemory = true }; + + server = EmbeddedDatabase.Start(configuration, lifetime); + return server; + } + finally + { + startLock.Release(); + } + } + + public static async Task Stop() + { + await startLock.WaitAsync(); + try + { + if (server is null) + { + return; + } + + await server.Stop(CancellationToken.None); + server.Dispose(); + server = null; + lifetime.StopApplication(); + } + finally + { + startLock.Release(); + } + } + + public static async Task<(string ServerUrl, string PrimaryDatabase, string ThroughputDatabase)> CreateDatabases(CancellationToken cancellationToken = default) + { + var instance = await GetInstance(cancellationToken); + var primary = $"sc_src_{Guid.NewGuid():n}"; + var throughput = $"{primary}-throughput"; + + using var store = await instance.Connect(cancellationToken); + foreach (var name in new[] { primary, throughput }) + { + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(name)), cancellationToken); + } + + return (instance.ServerUrl, primary, throughput); + } + + static EmbeddedDatabase server; + static readonly ApplicationLifetime lifetime = new(new NullLogger()); + static readonly SemaphoreSlim startLock = new(1, 1); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs new file mode 100644 index 0000000000..b42d2671f2 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[SetUpFixture] +public class StopMigrationSourceServer +{ + [OneTimeTearDown] + public Task Teardown() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs new file mode 100644 index 0000000000..fd07ff535b --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs @@ -0,0 +1,121 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +static class TestMigrationTargets +{ + public static void ParkFirstMigrationWrite(this WebApplicationBuilder builder, TaskCompletionSource parked, Task release) => + builder.DecorateMigrationTarget(inner => new ParkingMigrationTarget(inner, parked, release)); + + public static void FailTheSecondMigrationWrite(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 2)); + + public static void HaltTheEndpointSettingsCategory(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 1)); + + public static void StopAfterTheFirstCategorySettles(this WebApplicationBuilder builder, CancellationTokenSource stopping) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationCheckpointStore)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationCheckpointStore is registered by a factory rather than by type, so the test decorator cannot rebuild the inner store. Register it as AddSingleton() or give the decorator a different seam."); + } + + builder.Services.AddSingleton(provider => + new StoppingAfterTheFirstSettleCheckpointStore((IMigrationCheckpointStore)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType), stopping)); + } + + public static void DecorateMigrationTarget(this WebApplicationBuilder builder, Func decorate) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationTarget)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationTarget is registered by a factory rather than by type, so the test decorator cannot rebuild the inner target. Register it as AddSingleton() or give the decorator a different seam."); + } + + builder.Services.AddSingleton(provider => + decorate((IMigrationTarget)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType))); + } +} + +// Throws on one EndpointSettings write, which is the only way to see what a copy does after a write has failed. +sealed class FaultingMigrationTarget(IMigrationTarget inner, int failingWrite) : IMigrationTarget +{ + int endpointSettingsWrites; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + // Two rows a batch, so five settings take three writes and a failure on the second leaves exactly one batch committed. + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings ? Task.FromResult(2) : inner.BatchSizeFor(category, cancellationToken); + + public Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings && ++endpointSettingsWrites == failingWrite + ? throw new Exception("injected mid-category failure") + : inner.Write(category, batch, checkpointToExtend, cancellationToken); + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; +} + +// Stops the copy the moment the first category settles Done or Abandoned, which is the only way to stop it between two categories. +sealed class StoppingAfterTheFirstSettleCheckpointStore(IMigrationCheckpointStore inner, CancellationTokenSource stopping) : IMigrationCheckpointStore +{ + public Task> ReadAll(CancellationToken cancellationToken = default) => inner.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var stored = await inner.Upsert(checkpoint, cancellationToken); + + if (stored.State.IsFinished()) + { + await stopping.CancelAsync(); + throw new OperationCanceledException(stopping.Token); + } + + return stored; + } +} + +// Holds the copy open at a moment a test can observe, which is the only way to ask what the API answers mid-copy. +sealed class ParkingMigrationTarget(IMigrationTarget inner, TaskCompletionSource parked, Task release) : IMigrationTarget +{ + int writes; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref writes, 1) == 0) + { + parked.SetResult(); + await release.WaitAsync(cancellationToken); + } + + return await inner.Write(category, batch, checkpointToExtend, cancellationToken); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs new file mode 100644 index 0000000000..a55deaba69 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs @@ -0,0 +1,182 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; + +[TestFixture] +[NonParallelizable] +class When_a_copy_is_killed_mid_category : MigrationAcceptanceTest +{ + [Test] + public void A_migrating_host_stopped_while_starting_exits_cleanly() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.DoesNotThrowAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); + } + + [Test] + public void A_host_not_migrating_stopped_while_starting_still_fails() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); + } + + // Only a stop is a clean exit; a cancellation nobody asked for is still a failed start, migrating or not. + [Test] + public void A_migrating_host_whose_start_is_cancelled_without_a_stop_still_fails() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(CancelledWithoutAStop), cancellation.Token)); + } + + [Test] + public async Task It_resumes_rather_than_starting_over() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + // Bounded rather than None: a copy that did not fail would otherwise hold this call open forever. + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var killed = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + Assert.That(killed, Is.Not.Null); + Assert.That(await TargetEndpointSettingsCount(), Is.EqualTo(2), "the first batch and its cursor should have committed together"); + + var refusedAgain = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + Assert.That(refusedAgain.Message, Is.EqualTo(killed.Message), "a start re-ran a Failed category the operator had not put back"); + + // The row a kill part way through the category leaves: still copying, with the first batch's cursor and counts. + var halted = await ReadCheckpoint("EndpointSettings"); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null })); + + await RunHostUntilTheApiAnswers(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await GetEndpointSettingsNames(), Is.EquivalentTo(new[] { "A", "B", "C", "D", "E" }), "no gaps"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(5), "no row copied twice"); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero, "a resumed run must not re-read rows the first run committed"); + } + } + + // Proves the already-present count in the test above can fail: both runs end with five correct rows, so that count is the only thing telling a resume from a fresh copy. + [Test] + public async Task Without_its_checkpoint_the_second_run_re_reads_what_the_first_already_wrote() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + await DeleteCheckpoint("EndpointSettings"); + await RunHostUntilTheApiAnswers(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + } + + // A copy stopped between two categories must still list the second, or every gate reading the table finds nothing outstanding. + [Test] + public async Task A_worker_started_after_a_copy_stopped_between_categories_names_the_one_it_never_reached() + { + await SeedSourceKnownEndpoints("A", "B"); + await SeedSourceEndpointSettings(("A", true), ("B", true)); + Settings.MaximumConcurrencyLevel = 1; + + using var stopping = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.StopAfterTheFirstCategorySettles(stopping)), stopping.Token)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var worker = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + try + { + using (Assert.EnterMultipleScope()) + { + Assert.That((await ReadCheckpoint(MigrationCategoryIds.KnownEndpoints))?.State, Is.EqualTo(MigrationCategoryState.Complete), "the stop has to land between the two categories for this test to mean anything"); + Assert.That(async () => await worker.StartAsync(cancellation.Token), + Throws.Exception.With.Message.Contain("EndpointSettings is Copying (NotStarted)").And.Message.Not.Contain("KnownEndpoints is"), + "a worker let into a database whose copy stopped after its first category"); + Assert.That((await ReadCheckpoint(MigrationCategoryIds.EndpointSettings))?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + } + } + finally + { + await worker.DisposeAsync(); + } + } + + Task TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + Task GetEndpointSettingsNames() => + QueryTarget(dbContext => dbContext.EndpointSettings.Select(settings => settings.Name).ToArrayAsync()); + + static void StoppedWhileStarting(WebApplicationBuilder builder) => + builder.Services.AddHostedService(provider => new StopsTheHostWhileStarting(provider.GetRequiredService())); + + static void CancelledWithoutAStop(WebApplicationBuilder builder) => + builder.Services.AddHostedService(_ => new CancelsWhileStarting()); + + sealed class CancelsWhileStarting : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) => throw new OperationCanceledException(); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + // What a service stop or the critical-error handler does to a host that is still starting. + sealed class StopsTheHostWhileStarting(IHostApplicationLifetime lifetime) : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) + { + lifetime.StopApplication(); + throw new OperationCanceledException(lifetime.ApplicationStopping); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs new file mode 100644 index 0000000000..86cd583c29 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs @@ -0,0 +1,266 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[NonParallelizable] +class When_a_startup_check_fails : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + + [Test] + public async Task An_optional_category_window_that_is_not_a_time_span_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the optional category windows are valid").And.Contain("a week")); + } + + [Test] + public async Task A_retry_history_depth_that_empties_the_migrated_table_is_refused() + { + // Assigned rather than set as an environment variable: Settings reads RetryHistoryDepth once, in the constructor the fixture has already run. + Settings.RetryHistoryDepth = 0; + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the retry history depth will not empty a migrated table") + .And.Contain("RetryHistoryDepth") + .And.Contain("HistoricRetryOperations")); + } + + [Test] + public async Task A_target_whose_schema_migrations_were_never_applied_is_refused() + { + await QueryTarget(async dbContext => + { + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService(); + var applied = await history.GetAppliedMigrationsAsync(); + + foreach (var row in applied) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + return applied.Count; + }); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the target schema is current").And.Contain("--setup")); + } + + [Test] + public async Task A_target_that_already_holds_data_is_refused() + { + await QueryTarget(dbContext => + { + dbContext.Settings.Add(new SettingEntity { Key = "NotificationEmails", Value = "{}" }); + return dbContext.SaveChangesAsync(); + }); + var settingsTable = await QueryTarget(dbContext => Task.FromResult(dbContext.Model.FindEntityType(typeof(SettingEntity))!.GetTableName()!)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration target holds no ServiceControl data").And.Contain(settingsTable).And.Contain("--setup")); + } + + [Test] + public async Task Message_body_storage_that_cannot_be_written_to_is_refused() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + ((FileSystemBodyStorageSettings)EFSettings.BodyStorage).StoragePath = Path.Combine(parentThatIsAFile, "bodies"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("message body storage is writable")); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + [Test] + public async Task A_client_certificate_that_has_expired_is_refused() + { + var notBefore = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero); + var notAfter = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEBASE64", ExpiredClientCertificate(notBefore, notAfter)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain($"{notBefore.UtcDateTime:u} to {notAfter.UtcDateTime:u}")); + } + + [Test] + public async Task A_throughput_database_that_does_not_exist_is_refused() + { + var missing = $"{Source.ThroughputDatabase}-does-not-exist"; + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", missing); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain(missing) + .And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + } + + // Without this check a build that copies the required set commits a customer to SQL before the rest of the migration exists. + [Test] + public async Task A_build_without_the_whole_migration_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(allowUnreleasedMigration: false); + + Assert.That(refusal.Message, Does.Contain("this build carries the whole migration") + .And.Not.Contain("the optional category windows are valid"), + "this refusal comes first, so no other check is consulted on an unreleased build"); + } + + [Test] + public async Task A_halted_required_category_stops_the_host_rather_than_opening_it() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var exception = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.HaltTheEndpointSettingsCategory()), cancellation.Token)); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain("EndpointSettings").And.Contain("Halted")); + Assert.That(exception.Message, Does.Contain("nothing has been lost")); + Assert.That(exception.Message, Does.Contain("--migration-retry EndpointSettings").And.Contain("--migration-abandon EndpointSettings"), "a Failed category waits for the operator, so the refusal has to name what moves it on"); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf(), "Kestrel bound its socket behind a halted required category"); + }); + } + + [Test] + public async Task A_source_that_will_not_open_once_the_required_copy_settled_still_lets_the_host_open() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await RunHostUntilTheApiAnswers(); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + await RunHostUntilTheApiAnswers(); + + var endpointSettings = await GetEndpointSettings(); + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + var archived = await ReadCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpointSettings.Select(setting => setting.Name), Does.Contain("Sales")); + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(eventLog?.LastError, Does.Contain(missing).And.Not.Contain("will not start"), "status reads the outage from the row, on a start that did open"); + Assert.That(archived?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(archived?.LastError, Does.Contain(missing)); + } + } + + [Test] + public async Task A_source_that_will_not_open_before_the_required_copy_settled_still_refuses() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + await SeedCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + var refusal = await RefusedStartup(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("the migration source opens").And.Contain(missing)); + Assert.That(refusal.Message, Does.Contain("--migration-abandon"), "a required category that never started has no exit once the source is gone, and the refusal has to say so"); + Assert.That(await ReadCheckpoint(MigrationCategoryIds.EventLog), Is.Null, "a refusal writes nothing"); + } + } + + [Test] + public async Task A_failed_optional_row_does_not_stop_the_host_opening() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await RunHostUntilTheApiAnswers(); + + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.CompleteWithErrors); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", $"{Source.PrimaryDatabase}-does-not-exist"); + + await RunHostUntilTheApiAnswers(); + + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(eventLog?.LastError, Is.Null, "the outage leaves a Failed row alone"); + } + } + + // Every refusal is asserted against a source that has rows to copy: an empty target proves nothing otherwise. + // Only the release gate's own test withholds the marker; every other refusal has to get past it. + async Task RefusedStartup(bool allowUnreleasedMigration = true) + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var refusal = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, allowUnreleasedMigration ? AllowingAnUnreleasedMigration() : null, cancellation.Token)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("this start has copied nothing")); + Assert.That(await TargetEndpointSettingsCount(), Is.Zero, "a check that fires after rows have moved is worse than no check"); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing, so the abort is still free"); + } + + return refusal; + } + + Task TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + static string ExpiredClientCertificate(DateTimeOffset notBefore, DateTimeOffset notAfter) + { + using var key = RSA.Create(2048); + var request = new CertificateRequest("CN=sc-migration-source-expired", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = request.CreateSelfSigned(notBefore, notAfter); + + return Convert.ToBase64String(certificate.Export(X509ContentType.Pkcs12)); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs new file mode 100644 index 0000000000..07d17190d4 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_categories_change_between_restarts : MigrationAcceptanceTest +{ + [Test] + public async Task Changing_the_windows_between_restarts_deletes_nothing() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "a category no run has copied has no row, and no column anywhere says it was selected"); + var firstRun = await ReadCheckpoint("EndpointSettings"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "the required copy runs required categories only, so turning an optional one on starts nothing here"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await RunHostUntilTheApiAnswers(); + + var endpointSettings = await ReadCheckpoint("EndpointSettings"); + + Assert.Multiple(() => + { + Assert.That(endpointSettings.CopiedCount, Is.EqualTo(1), "a finished category must not be copied again when the selection changes around it"); + Assert.That(endpointSettings.Cursor, Is.Not.Null, "changing the selection must not delete or reset what an earlier run recorded"); + Assert.That(endpointSettings.Version, Is.EqualTo(firstRun.Version), "a row saved again, even unchanged, would have moved its version"); + Assert.That(endpointSettings.StartedAt, Is.EqualTo(firstRun.StartedAt), "a copy run again from scratch would have a new start"); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs new file mode 100644 index 0000000000..29d5b8b432 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_endpoint_settings_are_migrated : MigrationAcceptanceTest +{ + [Test] + public async Task The_source_server_starts_and_creates_both_databases() + { + var source = await MigrationSourceServer.CreateDatabases(); + + Assert.That(source.ServerUrl, Does.StartWith("http://localhost:")); + Assert.That(source.PrimaryDatabase, Is.Not.Empty); + Assert.That(source.ThroughputDatabase, Is.EqualTo($"{source.PrimaryDatabase}-throughput")); + } + + [Test] + public async Task They_are_readable_through_the_product_read_api() + { + await SeedSourceKnownEndpoints("Sales", "Billing"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", false), ("", true)); + + // A clock that never moves keeps HeartbeatEndpointSettingsSyncHostedService's twenty second + // delay from elapsing and rewriting these rows while the assertions read them. + await RunHostUntilTheApiAnswers(builder => builder.Services.AddSingleton(new FakeTimeProvider())); + + var settings = await GetEndpointSettings(); + + Assert.Multiple(() => + { + Assert.That(settings.Single(row => row.Name == "Sales").TrackInstances, Is.True); + Assert.That(settings.Single(row => row.Name == "Billing").TrackInstances, Is.False); + Assert.That(settings.Single(row => row.Name == string.Empty).TrackInstances, Is.True); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs new file mode 100644 index 0000000000..a57aa10426 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; + +[TestFixture] +// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set +// process-global environment variables. One fixture added without it makes the whole suite intermittent. +[NonParallelizable] +class When_the_api_is_called_during_the_required_copy : MigrationAcceptanceTest +{ + [Test] + public async Task The_api_does_not_answer_until_the_required_copy_has_finished() + { + await SeedSourceKnownEndpoints("Sales", "Billing", "Shipping"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", true), ("Shipping", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); + + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(releaseTheCopy.Task.IsCompleted, Is.False, "the copy must still be parked for this assertion to mean anything"); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf(), + "Kestrel had already bound its socket while the required copy was still running"); + + releaseTheCopy.SetResult(); + var settings = await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + + Assert.That(settings.Select(row => row.Name), Is.SupersetOf(new[] { "Sales", "Billing", "Shipping" })); + + await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); + await cancellation.CancelAsync(); + await host; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs new file mode 100644 index 0000000000..7620e21618 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs @@ -0,0 +1,258 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Hosting.Commands; + +[TestFixture] +// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set +// process-global environment variables. One fixture added without it makes the whole suite intermittent. +[NonParallelizable] +class When_the_host_opens_on_the_target : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + const string CategoryFromANewerBuild = "CategoryFromANewerBuild"; + + [Test] + public async Task The_row_does_not_exist_while_the_required_copy_is_still_running() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); + + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the abort is still free at this moment, and this row is what says it is not"); + + releaseTheCopy.SetResult(); + await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); + + await cancellation.CancelAsync(); + await host; + } + + [Test] + public async Task A_start_with_migration_off_does_not_stamp_the_target() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + + await RunHostUntilTheApiAnswers(); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "only a start with the migration on may write the marker"); + } + + [Test] + public async Task A_start_with_migration_off_does_not_read_the_checkpoint_table() + { + await DropTheCheckpointTable(); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + + Assert.That(async () => await RunHostUntilTheApiAnswers(), Throws.Nothing, "every hosted service, StartedAsync included, ran without touching the missing table"); + } + + [Test] + public async Task An_ingestion_only_host_is_refused_while_a_required_category_is_failed() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(except: MigrationCategoryIds.MessageRedirects); + await SeedCheckpoint(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("MessageRedirects is Failed (CompleteWithErrors)") + .And.Contain("--migration-retry MessageRedirects") + .And.Contain("--migration-abandon MessageRedirects")); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_starts_while_an_optional_category_is_still_copying_or_failed() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + Assert.That(async () => await app.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds a worker back"); + await app.StopAsync(cancellation.Token); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_is_refused_over_a_category_this_build_does_not_know() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(CategoryFromANewerBuild, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain($"{CategoryFromANewerBuild} is Copying (InProgress)"), "nobody here can say whether an id this build does not hold is optional, so it holds the worker back"); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task Importing_failed_errors_is_refused_while_a_copy_is_unfinished() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + var refusal = Assert.ThrowsAsync(async () => await host.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("--import-failed-errors will not start").And.Contain("KnownEndpoints is Copying (InProgress)")); + } + + [Test] + public async Task Importing_failed_errors_starts_while_an_optional_category_is_still_copying_or_failed() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + Assert.That(async () => await host.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds the import back"); + await host.StopAsync(cancellation.Token); + } + + // A customer whose migration never started must not be told they have passed the point of no return. + [Test] + public async Task The_row_does_not_exist_after_a_startup_check_refused() + { + // Seeded so the marker would be written if anything stamped it, which makes the refusal the only reason it is not. + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "NoSuchWindow"); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Throws.Exception.With.Message.Contains("NoSuchWindow")); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing"); + } + + // Ingestion-only hosts write failed messages into the same database a migration targets, so a rollback + // gate that had not seen them would discard everything they ingested. + [Test] + public async Task An_ingestion_only_host_records_the_row_too() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + // The ingestion-only host runs a real transport, which the migration fixture does not otherwise configure. + Settings.MaximumConcurrencyLevel = 1; + + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null, "only RunCommand used to record this, so a scaled-out ingestion host wrote to the target and left no trace of having opened on it"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + + // A customer running on SQL who has never migrated must not be told the way back is gone. + [Test] + public async Task A_database_no_migration_has_touched_is_not_stamped() + { + Settings.MaximumConcurrencyLevel = 1; + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "no checkpoint row exists, so no migration has ever run against this database"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + async Task SeedRequiredCategoriesComplete(string except = null) + { + foreach (var category in MigrationCategoryRegistry.All.Where(category => category.Kind == MigrationCategoryKind.Required && category.Id != except)) + { + await SeedCheckpoint(category.Id); + } + } + + Task DropTheCheckpointTable() => + QueryTarget(dbContext => + { + var table = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + var name = dbContext.GetService().DelimitIdentifier(table.GetTableName()!, table.GetSchema()); + + // The name comes from the EF model and the provider delimits it, so no outside input reaches this SQL. +#pragma warning disable EF1003 + return dbContext.Database.ExecuteSqlRawAsync("DROP TABLE " + name); +#pragma warning restore EF1003 + }); +} diff --git a/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs new file mode 100644 index 0000000000..1bdc4ed89f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs @@ -0,0 +1,100 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationCategoryCoverageTests +{ + Settings settings; + + [SetUp] + public void SetUp() + { + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", + Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n"))); + + // SQL Server stands in for PostgreSQL too, because both register the same EFCoreMigrationTarget and its writer list does not depend on the provider. + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + // A reader with no writer, or the reverse, is a category the intersection would otherwise drop in silence. + [Test] + public async Task Every_reader_has_a_writer_and_the_reverse() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + var sourceCategoryIds = source.SupportedCategoryIds; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService(); + + var readerOnly = sourceCategoryIds.Except(target.SupportedCategoryIds).ToArray(); + var writerOnly = target.SupportedCategoryIds.Except(sourceCategoryIds).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(readerOnly, Is.Empty, $"the reader claims a category the writer does not: {string.Join(", ", readerOnly)}"); + Assert.That(writerOnly, Is.Empty, $"the writer claims a category the reader does not: {string.Join(", ", writerOnly)}"); + } + } + + [Test] + public async Task Every_reader_and_its_writer_agree_on_the_document_type() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService(); + + var shared = source.DocumentTypes.Keys.Intersect(target.DocumentTypes.Keys).ToArray(); + Assert.That(shared, Is.Not.Empty, "the test proves nothing if no category has both a reader and a writer"); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in shared) + { + Assert.That(target.DocumentTypes[categoryId], Is.EqualTo(source.DocumentTypes[categoryId]), $"the {categoryId} reader yields a document type its writer does not take"); + } + } + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List variables = []; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..f9b6c71345 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,991 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20261007000244_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("request_id"); + + b.Property("ArchiveType") + .HasColumnType("integer") + .HasColumnName("archive_type"); + + b.Property("OperationType") + .HasColumnType("integer") + .HasColumnName("operation_type"); + + b.Property("CurrentBatch") + .HasColumnType("integer") + .HasColumnName("current_batch"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("group_name"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_name"); + + b.Property("NumberOfBatches") + .HasColumnType("integer") + .HasColumnName("number_of_batches"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Started") + .HasColumnType("timestamp with time zone") + .HasColumnName("started"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("integer") + .HasColumnName("total_number_of_messages"); + + b.HasKey("RequestId", "ArchiveType", "OperationType") + .HasName("pk_archive_operations"); + + b.HasIndex("Started") + .HasDatabaseName("ix_archive_operations_started"); + + b.ToTable("archive_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Category") + .IsRequired() + .HasColumnType("text") + .HasColumnName("category"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("custom_check_id"); + + b.Property("FailureReason") + .HasColumnType("text") + .HasColumnName("failure_reason"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_host"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("originating_endpoint_host_id"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_name"); + + b.Property("ReportedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("reported_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_custom_checks"); + + b.HasIndex("ReportedAt") + .HasDatabaseName("ix_custom_checks_reported_at"); + + b.HasIndex("Status", "ReportedAt") + .HasDatabaseName("ix_custom_checks_status_reported_at"); + + b.ToTable("custom_checks", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("event_log_items", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("dispatch_context_json"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("dispatch_context_type_name"); + + b.HasKey("Id") + .HasName("pk_external_integration_dispatch_requests"); + + b.ToTable("external_integration_dispatch_requests", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("edit_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_edits"); + + b.HasIndex("EditId") + .HasDatabaseName("ix_failed_message_edits_edit_id"); + + b.ToTable("failed_message_edits", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastTimeOfFailure") + .HasDatabaseName("ix_failed_messages_status_last_time_of_failure"); + + b.HasIndex("Status", "LastModified", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_last_modified_unique_message_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified", "UniqueMessageId"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "MessageType", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_message_type_unique_message_id"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.HasIndex("Type", "GroupId") + .HasDatabaseName("ix_failed_message_groups_type_group_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "FailedMessageUniqueId", "Title" }); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.Property("StageAttempts") + .HasColumnType("integer") + .HasColumnName("stage_attempts"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.HasIndex("RetryBatchId") + .HasDatabaseName("ix_failed_message_retries_retry_batch_id"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("text") + .HasColumnName("comment"); + + b.HasKey("GroupId") + .HasName("pk_group_comments"); + + b.ToTable("group_comments", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("Id") + .HasName("pk_historic_retry_operations"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending() + .HasDatabaseName("ix_historic_retry_operations_completion_time_id"); + + b.ToTable("historic_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.PrimitiveCollection>("EndpointIndicators") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("endpoint_indicators"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("name"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("normalized_sanitized_name"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sanitized_name"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("scope"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("user_indicator"); + + b.HasKey("NormalizedName", "ThroughputSource") + .HasName("pk_licensing_endpoints"); + + b.HasIndex("NormalizedSanitizedName") + .HasDatabaseName("ix_licensing_endpoints_normalized_sanitized_name"); + + b.ToTable("licensing_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.Property("DateUtc") + .HasColumnType("date") + .HasColumnName("date_utc"); + + b.Property("MessageCount") + .HasColumnType("bigint") + .HasColumnName("message_count"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc") + .HasName("pk_licensing_endpoint_throughput"); + + b.HasIndex("DateUtc") + .HasDatabaseName("ix_licensing_endpoint_throughput_date_utc"); + + b.ToTable("licensing_endpoint_throughput", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("from_physical_address"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("to_physical_address"); + + b.HasKey("FromPhysicalAddress") + .HasName("pk_message_redirects"); + + b.ToTable("message_redirects", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Classifier") + .HasColumnType("text") + .HasColumnName("classifier"); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("InitialBatchSize") + .HasColumnType("integer") + .HasColumnName("initial_batch_size"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasColumnType("text") + .HasColumnName("initiated_by_name"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("request_id"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_session_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("staging_id"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_retry_batches"); + + b.HasIndex("Status", "RetrySessionId") + .HasDatabaseName("ix_retry_batches_status_retry_session_id"); + + b.ToTable("retry_batches", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("integer") + .HasColumnName("id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.HasKey("Id") + .HasName("pk_retry_batch_now_forwarding"); + + b.ToTable("retry_batch_now_forwarding", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("key"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Key") + .HasName("pk_settings"); + + b.ToTable("settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("classifier"); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("RequestId", "RetryType") + .HasName("pk_unacknowledged_retry_operations"); + + b.ToTable("unacknowledged_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licensing_endpoint_throughput_licensing_endpoints_normalize"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..6eaf796b17 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class AddMigrationCheckpoints : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "migration_checkpoints", + columns: table => new + { + category_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: false), + state = table.Column(type: "integer", nullable: false), + cursor = table.Column(type: "text", nullable: true), + copied_count = table.Column(type: "bigint", nullable: false), + skipped_count = table.Column(type: "bigint", nullable: false), + source_total = table.Column(type: "bigint", nullable: true), + skip_reasons = table.Column(type: "text", nullable: true), + started_at = table.Column(type: "timestamp with time zone", nullable: true), + last_progress_at = table.Column(type: "timestamp with time zone", nullable: true), + settled_at = table.Column(type: "timestamp with time zone", nullable: true), + last_error = table.Column(type: "text", nullable: true), + already_present_count = table.Column(type: "bigint", nullable: false), + version = table.Column(type: "bigint", nullable: false), + started_window_seconds = table.Column(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_migration_checkpoints", x => x.category_id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "migration_checkpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index cb89bee289..145f62fe0e 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -717,6 +717,72 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("message_redirects", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs new file mode 100644 index 0000000000..fdac0674c0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +///

+/// The migration's PostgreSQL statements. An insert that does nothing on a conflicting key is all it takes to +/// add only what is absent, and the keys it inserted come back from the same statement. +/// +class PostgreSqlMigrationSqlDialect : PostgreSqlDialect, IMigrationSqlDialect +{ + // Deterministic collations compare keys byte for byte, so there is no schema fact to read and none to name in the statement. + public Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public IEqualityComparer KeyComparer(Type entityType, string propertyName) => StringComparer.Ordinal; + + // A fixed chunk, because PostgreSQL's parameter limit is far away and the row width does not bring it closer. + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement; + + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert.For(dbContext); + var keyColumns = string.Join(", ", insert.KeyColumns); + var inserted = new List(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + inserted.AddRange(await insert.Execute( + dbContext, + $""" + INSERT INTO {insert.Table} ({string.Join(", ", insert.Columns)}) + VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ON CONFLICT ({keyColumns}) DO NOTHING + RETURNING {keyColumns} + """, + chunk, + cancellationToken)); + } + + return inserted; + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index a32f5fa6b4..4b23202f13 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -23,6 +23,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..2f862bcb80 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,794 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20261007000240_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ArchiveType") + .HasColumnType("int"); + + b.Property("OperationType") + .HasColumnType("int"); + + b.Property("CurrentBatch") + .HasColumnType("int"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfBatches") + .HasColumnType("int"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Started") + .HasColumnType("datetime2"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("int"); + + b.HasKey("RequestId", "ArchiveType", "OperationType"); + + b.HasIndex("Started"); + + b.ToTable("ArchiveOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Category") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailureReason") + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReportedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ReportedAt"); + + b.HasIndex("Status", "ReportedAt"); + + b.ToTable("CustomChecks"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("ExternalIntegrationDispatchRequests"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("EditId"); + + b.ToTable("FailedMessageEdits"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("FailingEndpointAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "LastTimeOfFailure"); + + b.HasIndex("Status", "MessageType", "UniqueMessageId"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.HasIndex("Type", "GroupId"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "Title" }); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.Property("StageAttempts") + .HasColumnType("int"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("RetryBatchId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("GroupId"); + + b.ToTable("GroupComments"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending(); + + b.ToTable("HistoricRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.PrimitiveCollection("EndpointIndicators") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("NormalizedName", "ThroughputSource"); + + b.HasIndex("NormalizedSanitizedName"); + + b.ToTable("LicensingEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.Property("DateUtc") + .HasColumnType("date"); + + b.Property("MessageCount") + .HasColumnType("bigint"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc"); + + b.HasIndex("DateUtc"); + + b.ToTable("LicensingEndpointThroughput"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("FromPhysicalAddress"); + + b.ToTable("MessageRedirects"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property("CopiedCount") + .HasColumnType("bigint"); + + b.Property("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property("SettledAt") + .HasColumnType("datetime2"); + + b.Property("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property("SkippedCount") + .HasColumnType("bigint"); + + b.Property("SourceTotal") + .HasColumnType("bigint"); + + b.Property("StartedAt") + .HasColumnType("datetime2"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property("State") + .HasColumnType("int"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Classifier") + .HasColumnType("nvarchar(max)"); + + b.Property("Context") + .HasColumnType("nvarchar(max)"); + + b.Property("InitialBatchSize") + .HasColumnType("int"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasColumnType("nvarchar(max)"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Status", "RetrySessionId"); + + b.ToTable("RetryBatches"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("int"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.ToTable("RetryBatchNowForwarding"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Value") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("RequestId", "RetryType"); + + b.ToTable("UnacknowledgedRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..c3dab88c81 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class AddMigrationCheckpoints : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "MigrationCheckpoints", + columns: table => new + { + CategoryId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: false), + State = table.Column(type: "int", nullable: false), + Cursor = table.Column(type: "nvarchar(max)", nullable: true), + CopiedCount = table.Column(type: "bigint", nullable: false), + SkippedCount = table.Column(type: "bigint", nullable: false), + SourceTotal = table.Column(type: "bigint", nullable: true), + SkipReasons = table.Column(type: "nvarchar(max)", nullable: true), + StartedAt = table.Column(type: "datetime2", nullable: true), + LastProgressAt = table.Column(type: "datetime2", nullable: true), + SettledAt = table.Column(type: "datetime2", nullable: true), + LastError = table.Column(type: "nvarchar(max)", nullable: true), + AlreadyPresentCount = table.Column(type: "bigint", nullable: false), + Version = table.Column(type: "bigint", nullable: false), + StartedWindowSeconds = table.Column(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_MigrationCheckpoints", x => x.CategoryId); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "MigrationCheckpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index acca0378a6..f6e5ed81b7 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -573,6 +573,57 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("MessageRedirects"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property("CopiedCount") + .HasColumnType("bigint"); + + b.Property("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property("SettledAt") + .HasColumnType("datetime2"); + + b.Property("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property("SkippedCount") + .HasColumnType("bigint"); + + b.Property("SourceTotal") + .HasColumnType("bigint"); + + b.Property("StartedAt") + .HasColumnType("datetime2"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property("State") + .HasColumnType("int"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs index 0e26ef0ece..132096e217 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs @@ -111,6 +111,11 @@ protected static string ParameterRows(int rowCount, int columnCount) return sql.ToString(); } + // The same rows as ParameterRows, each ending with its position in the batch, so a statement can keep the first of two duplicate keys. + protected static string ParameterRowsWithOrdinal(int rowCount, int columnCount) => + string.Join(",\n", Enumerable.Range(0, rowCount).Select(row => + $"({string.Join(", ", Enumerable.Range(0, columnCount).Select(column => $"@p{(row * columnCount) + column}"))}, {row})")); + protected static int MaxRowsPerStatement(int columns) => MaxParametersPerStatement / columns; // SQL Server's ceiling is 2100 (https://learn.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server). diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs new file mode 100644 index 0000000000..45d8ea8454 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -0,0 +1,114 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// The migration's SQL Server statements. Two keys that differ only in case can be one row here, because a +/// column's collation decides what counts as equal, so the statements compare keys under the collation the +/// schema gives them and the batch is de-duplicated the same way before it is offered to the table. +/// +class SqlServerMigrationSqlDialect : SqlServerDialect, IMigrationSqlDialect +{ + const int IgnoreCaseStyle = 1; + + OpenedKeys? OpenedState { get; set; } + + public async Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) + { + var sql = dbContext.GetService(); + var collations = new Dictionary<(string Table, string Column), string>(); + var comparers = new Dictionary<(Type EntityType, string Property), IEqualityComparer>(); + + foreach (var entityType in dbContext.Model.GetEntityTypes()) + { + if (entityType.GetTableName() is not { } tableName || entityType.FindPrimaryKey() is not { } key) + { + continue; + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var table = sql.DelimitIdentifier(tableName, entityType.GetSchema()); + + foreach (var property in key.Properties.Where(property => property.ClrType == typeof(string))) + { + // A key property with no column of its own is refused by name when MigrationInsert builds the statement. + if (property.GetColumnName(storeObject) is not { } column) + { + continue; + } + + var collation = await dbContext.Database + .SqlQuery($""" + SELECT c.collation_name AS [Name], + CONVERT(int, COLLATIONPROPERTY(c.collation_name, 'ComparisonStyle')) AS [ComparisonStyle] + FROM sys.columns AS c + WHERE c.object_id = OBJECT_ID({table}) AND c.name = {column} AND c.collation_name IS NOT NULL + """) + .SingleOrDefaultAsync(cancellationToken) + ?? throw new InvalidOperationException($"The migration target found no collation for {tableName}.{column}, so the SQL Server schema is missing or not current. Run ServiceControl with --setup against this database first."); + + collations[(table, sql.DelimitIdentifier(column))] = collation.Name; + comparers[(entityType.ClrType, property.Name)] = (collation.ComparisonStyle & IgnoreCaseStyle) == 0 ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; + } + } + + OpenedState = new OpenedKeys(collations, comparers); + } + + public IEqualityComparer KeyComparer(Type entityType, string propertyName) => + Keys.Comparers.TryGetValue((entityType, propertyName), out var comparer) + ? comparer + : throw new ArgumentException($"{entityType.Name}.{propertyName} is not a string primary-key column, so it has no collation to compare under."); + + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement(parametersPerRow); + + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert.For(dbContext); + var columns = string.Join(", ", insert.Columns); + // The collation is an identifier sys.columns returned, and COLLATE takes no parameter. + var partitionBy = string.Join(", ", insert.KeyColumns.Select(column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column)); + var inserted = new List(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // HOLDLOCK, or two writers can both find a key missing and both insert it. + inserted.AddRange(await insert.Execute( + dbContext, + $""" + MERGE {insert.Table} WITH (HOLDLOCK) AS t + USING ( + SELECT {columns} + FROM ( + SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {partitionBy} ORDER BY [MigrationOrdinal]) AS [MigrationDuplicate] + FROM (VALUES + {ParameterRowsWithOrdinal(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}, [MigrationOrdinal]) + ) AS deduplicated + WHERE [MigrationDuplicate] = 1 + ) AS s ({columns}) + ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} + WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) + OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"inserted.{column}"))}; + """, + chunk, + cancellationToken)); + } + + return inserted; + } + + OpenedKeys Keys => OpenedState ?? throw new InvalidOperationException($"The SQL Server migration dialect is not open. Call {nameof(Open)} first."); + + sealed record OpenedKeys( + IReadOnlyDictionary<(string Table, string Column), string> Collations, + IReadOnlyDictionary<(Type EntityType, string Property), IEqualityComparer> Comparers); + + sealed record KeyColumnCollation(string Name, int ComparisonStyle); +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index cff42ab67d..90ee720b51 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -18,6 +18,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs index 1e0a590cdb..f0fc102b07 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs @@ -8,12 +8,14 @@ namespace ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.StartupChecks; using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.EFCore.DataMigration; using ServiceControl.Persistence.EFCore.Implementation; using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; using ServiceControl.Persistence.EFCore.Implementation.Recoverability; using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.Infrastructure.Metrics; +using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Recoverability; using ServiceControl.Persistence.UnitOfWork; @@ -41,6 +43,10 @@ protected static void RegisterDataStores(IServiceCollection services, EFPersiste services.AddSingleton(p => p.GetRequiredService()); services.AddHostedService(p => p.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + if (settings.RunRetentionSweep) { services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs new file mode 100644 index 0000000000..5a5c116bdf --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes, reads and deletes one probe body, so a body store that is misconfigured stops the copy before it +/// starts. Bodies can live on a file share, in Azure Blob Storage or in S3, and without this the first category +/// that carries bodies would find out part way through and skip every message it could not write. +/// +sealed class BodyStorageIsWritableCheck(IBodyStoragePersistence bodyStorage) : IMigrationStartupCheck +{ + const string ProbeBodyId = "migration-writable-probe"; + + public string Name => "message body storage is writable"; + + public async Task Run(CancellationToken cancellationToken = default) + { + await bodyStorage.WriteBody(ProbeBodyId, Encoding.UTF8.GetBytes(ProbeBodyId), "text/plain", cancellationToken); + + try + { + var probe = await bodyStorage.ReadBody(ProbeBodyId, cancellationToken) + ?? throw new Exception($"Message body storage accepted the probe body '{ProbeBodyId}' and then did not return it."); + + await probe.Stream.DisposeAsync(); + } + finally + { + // The probe is not migrated data, so it goes even when the read fails: a probe left behind is a body the source never had. + await bodyStorage.DeleteBodyIfExists(ProbeBodyId, cancellationToken); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs new file mode 100644 index 0000000000..09510292ff --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -0,0 +1,113 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Collections.Frozen; +using Implementation; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; +using Writers; + +/// +/// Writes a migration into SQL Server or PostgreSQL. It holds one writer per category and knows nothing about +/// any of them beyond that, so a category this build cannot write is simply absent from +/// and never reaches the engine. +/// +sealed class EFCoreMigrationTarget(IServiceScopeFactory scopeFactory, IMigrationSqlDialect migrationDialect, ILogger logger) : DataStoreBase(scopeFactory), IMigrationTarget +{ + readonly FrozenDictionary writers = new IMigrationCategoryWriter[] + { + new KnownEndpointsWriter(migrationDialect), + new EndpointSettingsWriter(migrationDialect) + }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); + + public Task Open(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => migrationDialect.Open(dbContext, token), cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, _) => Task.FromResult(WriterFor(category).BatchSize(dbContext)), cancellationToken); + + public Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, writeToken) => + { + var prepared = await WriterFor(category).Prepare(dbContext, batch, writeToken); + + AccountForEveryRow(category, batch, prepared); + + var skipReasons = prepared.Skips.Count == 0 ? null : prepared.Skips.GroupBy(skip => skip.Reason).ToDictionary(group => group.Key, group => group.LongCount()); + + var (copied, alreadyPresent, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + // This block is retried, and an attempt that failed leaves the checkpoint it added still tracked, so the next attempt would insert it a second time. + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + var inserted = await prepared.Insert(dbContext, token); + var present = AlreadyPresentIn(category, batch, inserted, prepared.Skips.Count); + var stored = await dbContext.UpsertCheckpoint( + checkpointToExtend.Extend(inserted, prepared.Skips.Count, present, skipReasons), + token); + await transaction.CommitAsync(token); + + return (inserted, present, stored); + }, writeToken); + + foreach (var (sourceId, reason, detail) in prepared.Skips) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId} as {SkipReason}: {Detail}", sourceId, category.Id, reason, detail); + } + + foreach (var (sourceId, detail) in prepared.Merges) + { + logger.LogWarning("Merged {SourceId} in category {CategoryId} into a row with the same key: {Detail}", sourceId, category.Id, detail); + } + + return new MigrationWriteResult( + saved, + copied, + prepared.Skips.Count, + [.. prepared.Skips.Select(skip => skip.SourceId)], + AlreadyPresent: alreadyPresent, + SkipReasons: skipReasons); + }, cancellationToken); + + // Checked before the insert, because the subtraction below can only catch a writer that over-reports. + internal static void AccountForEveryRow(MigrationCategory category, MigrationBatch batch, PreparedBatch prepared) + { + if (prepared.PreparedRowCount + prepared.Skips.Count != batch.Rows.Count) + { + throw new InvalidOperationException($"The {category.Id} writer prepared {prepared.PreparedRowCount} rows and skipped {prepared.Skips.Count} of the {batch.Rows.Count} rows in the batch. Every row must be one or the other, or the rows it dropped would be counted as rows the target already held."); + } + } + + // Without the throw, a miscount would quietly shrink the halt threshold's denominator instead of failing. + internal static int AlreadyPresentIn(MigrationCategory category, MigrationBatch batch, int copied, int skipped) + { + var alreadyPresent = batch.Rows.Count - copied - skipped; + + if (alreadyPresent < 0) + { + throw new InvalidOperationException($"The target copied {copied} and skipped {skipped} of the {batch.Rows.Count} rows in category {category.Id}, leaving {alreadyPresent} already present. Every row is copied, skipped or already present."); + } + + return alreadyPresent; + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => WriterFor(category).Count(dbContext, token), cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => writers.Keys; + + public IReadOnlyDictionary DocumentTypes => writers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + + IMigrationCategoryWriter WriterFor(MigrationCategory category) => + writers.TryGetValue(category.Id, out var writer) + ? writer + : throw new NotSupportedException($"The migration target cannot yet write the '{category.Id}' category."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs new file mode 100644 index 0000000000..9988718ea8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// What a SQL Server or PostgreSQL database has to be before a copy writes to it, and the stamp saying a host +/// has opened on it since. The stamp is a row in the settings table, so it survives a restart and belongs to +/// the database rather than to the instance. +/// +public class EFCoreMigrationTargetReadiness( + IServiceScopeFactory scopeFactory, + IBodyStoragePersistence bodyStorage, + TimeProvider timeProvider) : DataStoreBase(scopeFactory), IMigrationTargetReadiness +{ + // Cheapest first: the schema costs one query, and the body store costs a round trip to a file share or a + // cloud service. + public IReadOnlyList ContributedChecks() => + [ + new SchemaIsCurrentCheck(scopeFactory), + new TargetHoldsNoServiceControlDataCheck(scopeFactory), + new BodyStorageIsWritableCheck(bodyStorage) + ]; + + public Task RecordHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + if (await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget, token) is null) + { + await dbContext.StoreSetting(SettingKeys.MigrationHostOpenedOnTarget, timeProvider.GetUtcNow().UtcDateTime, token); + } + }, cancellationToken); + + public Task HasHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget, token) is not null, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs new file mode 100644 index 0000000000..3349a45a38 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Everything the target needs to know about one category: how big a batch is, how its documents +/// become rows, and how many rows the target holds for it. +/// +interface IMigrationCategoryWriter +{ + /// The category this writer handles, named as in . + string CategoryId { get; } + + /// + /// The type of takes. A row holding any other type + /// fails the batch with an . + /// + Type DocumentType { get; } + + /// + /// The most rows the target will take from the source in one batch. It comes from how many values each row + /// carries and how many parameters one statement can hold, so a wider table takes fewer rows. The context is read for its model only. + /// + int BatchSize(ServiceControlDbContext dbContext); + + /// + /// Turns one batch of source documents into the rows to insert and the rows to skip, writing nothing: the insert it returns runs later, inside the target's transaction. + /// Every batch row produces exactly one row or exactly one skip, never both and never neither, because the target works out how many were already there by subtracting both from the batch size. + /// + Task Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default); + + /// + /// How many rows the target holds for this category. It counts only this category, even where two of them + /// share a table. + /// + Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs new file mode 100644 index 0000000000..4550acc0bb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// A writer that takes rows carrying a . It casts every row once, before +/// sees it, so a row from a reader that yields another type fails the batch by name. +/// +abstract class MigrationCategoryWriter : IMigrationCategoryWriter where TDocument : class +{ + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract int BatchSize(ServiceControlDbContext dbContext); + + public abstract Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + public Task Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default) => + PrepareDocuments(dbContext, [.. batch.Rows.Select(row => (row, DocumentOf(row)))], cancellationToken); + + /// + /// Does what promises, for rows already cast. + /// + /// The context to read the target through. Nothing may be written through it here. + /// Every row in the batch, in source order, each with its document. + /// Cancels any read of the target. + /// One prepared row or one skip for every entry in . + protected abstract Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default); + + TDocument DocumentOf(MigrationRow row) => + row.Document as TDocument + ?? throw new InvalidCastException($"The {CategoryId} writer takes {typeof(TDocument).FullName} documents, but row {row.SourceId} holds {row.Document?.GetType().FullName ?? "no document"}. The reader and writer for one category must agree on the document type."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs new file mode 100644 index 0000000000..c4e3faebb6 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// One batch turned into rows, ready for the target to insert inside its own transaction. Nothing here has +/// touched the database yet. +/// +/// Runs the insert and returns how many rows it added. The target calls it inside the transaction that also saves the checkpoint. +/// How many rows the writer built, which is more than the insert adds when the table already holds some of their keys. +/// One entry per row the writer will not insert, with the reason and a detail line for the log. +/// One entry per prepared row whose key the target treats as the same key as another row's, either earlier in the batch or already in the table, so the insert keeps the other row and drops this one. Each carries a detail line naming both keys and the one kept, which the target logs as a warning after the commit. The dropped row is counted as already present. +sealed record PreparedBatch( + Func> Insert, + int PreparedRowCount, + IReadOnlyList<(string SourceId, MigrationSkipReason Reason, string Detail)> Skips, + IReadOnlyList<(string SourceId, string Detail)> Merges); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs new file mode 100644 index 0000000000..24f64d1de2 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy into a database whose schema is behind this build. The writers insert into tables by name, so +/// a missing schema migration turns into a failure part way through the first category instead of a refusal. +/// +sealed class SchemaIsCurrentCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the target schema is current"; + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + string[] pending = [.. await dbContext.Database.GetPendingMigrationsAsync(token)]; + + if (pending.Length > 0) + { + throw new Exception($"The database is missing {pending.Length} schema migration(s): {string.Join(", ", pending)}. Run ServiceControl with --setup before starting a migration."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs new file mode 100644 index 0000000000..b545a8bea6 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy into a database that already holds ServiceControl data. A copy into a database that ran on SQL +/// mixes the customer's rows with rows SQL already held, and a retry of a category could then delete rows the copy +/// never wrote. The check stands down once a row that is not known to be optional exists, because that means a copy +/// has started and what the tables hold is what it wrote. +/// +sealed class TargetHoldsNoServiceControlDataCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the migration target holds no ServiceControl data"; + + // One entry per mapped entity but the checkpoint: its CLR type, and a query asking whether its table holds a row. + internal static readonly IReadOnlyList<(Type Entity, Func> HasRows)> Tables = + [ + Entry(dbContext => dbContext.CustomChecks), + Entry(dbContext => dbContext.EndpointSettings), + Entry(dbContext => dbContext.KnownEndpoints), + Entry(dbContext => dbContext.FailedMessages), + Entry(dbContext => dbContext.FailedMessageGroups), + Entry(dbContext => dbContext.GroupComments), + Entry(dbContext => dbContext.MessageRedirects), + Entry(dbContext => dbContext.RetryBatches), + Entry(dbContext => dbContext.RetryBatchNowForwarding), + Entry(dbContext => dbContext.FailedMessageRetries), + Entry(dbContext => dbContext.FailedErrorImports), + Entry(dbContext => dbContext.Settings), + Entry(dbContext => dbContext.Subscriptions), + Entry(dbContext => dbContext.EventLogItems), + Entry(dbContext => dbContext.HistoricRetryOperations), + Entry(dbContext => dbContext.UnacknowledgedRetryOperations), + Entry(dbContext => dbContext.ArchiveOperations), + Entry(dbContext => dbContext.FailedMessageEdits), + Entry(dbContext => dbContext.LicensingEndpoints), + Entry(dbContext => dbContext.LicensingEndpointThroughput), + Entry(dbContext => dbContext.ExternalIntegrationDispatchRequests) + ]; + + // Both halves of an entry come from one DbSet, so the type a test compares and the table the query reads cannot differ. + static (Type Entity, Func> HasRows) Entry(Func> set) where T : class => + (typeof(T), (dbContext, cancellationToken) => set(dbContext).AnyAsync(cancellationToken)); + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var checkpointed = await dbContext.MigrationCheckpoints.Select(checkpoint => checkpoint.CategoryId).ToListAsync(token); + + if (checkpointed.Any(id => MigrationCategoryRegistry.Find(id)?.Kind != MigrationCategoryKind.Optional)) + { + return; + } + + var holdingRows = new List(); + + foreach (var (entity, hasRows) in Tables) + { + if (await hasRows(dbContext, token)) + { + holdingRows.Add(dbContext.Model.FindEntityType(entity)!.GetTableName()!); + } + } + + if (holdingRows.Count > 0) + { + throw new Exception($"The database already holds ServiceControl data in {string.Join(", ", holdingRows)}, so ServiceControl has already run on it, and a migration copies only into an empty database. Create a new database, run ServiceControl with --setup against it, and do not start ServiceControl on it before migrating; then start again with migration on."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs new file mode 100644 index 0000000000..485518ca67 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes every per-endpoint setting as stored. A setting for an endpoint the target does not know is copied too, +/// and the heartbeat settings sync removes it by its own rule once the instance opens. On a target whose name +/// column ignores case, two names that differ only in case are one row, and the writer reports each setting the +/// insert drops as a merge. +/// +sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.EndpointSettings.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => new EndpointSettingsEntity { Name = document.Document.Name, TrackInstances = document.Document.TrackInstances })]; + List rowSourceIds = [.. documents.Select(document => document.Row.SourceId)]; + + // The source yields document-id order, and the statement keeps the first of any keys the database treats as one. + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + [], + await MergesIn(dbContext, rows, rowSourceIds, cancellationToken)); + } + + // Mirrors the insert's de-duplication, so an operator can see which settings a case-insensitive name column dropped. + async Task> MergesIn(ServiceControlDbContext dbContext, List rows, List rowSourceIds, CancellationToken cancellationToken) + { + var sameName = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); + string[] rowNames = [.. rows.Select(row => row.Name)]; + + // The database compares under the column's collation, so this also returns a stored name that differs only in case. + var keptNames = await dbContext.EndpointSettings.AsNoTracking() + .Select(stored => stored.Name) + .Where(name => rowNames.Contains(name)) + .ToListAsync(cancellationToken); + + var merges = new List<(string SourceId, string Detail)>(); + + for (var index = 0; index < rows.Count; index++) + { + var name = rows[index].Name; + var keptName = keptNames.FirstOrDefault(kept => sameName.Equals(kept, name)); + + if (keptName is null) + { + keptNames.Add(name); + } + else if (!string.Equals(keptName, name, StringComparison.Ordinal)) + { + merges.Add((rowSourceIds[index], $"the name column treats '{name}' and '{keptName}' as one endpoint, so it kept '{keptName}' and dropped the settings of '{name}'")); + } + } + + return merges; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs new file mode 100644 index 0000000000..a0ec9ff796 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the endpoints ServiceControl has heard from. The row's key is worked out from the endpoint's own +/// details rather than carried over, so the same endpoint lands on the same row whichever side wrote it first. +/// +sealed class KnownEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.KnownEndpoints.LongCountAsync(cancellationToken); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, KnownEndpoint Document)> documents, CancellationToken cancellationToken = default) + { + var rows = new List(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, endpoint) in documents) + { + if (endpoint.EndpointDetails?.Name is null || endpoint.EndpointDetails.Host is null) + { + var column = endpoint.EndpointDetails?.Name is null ? nameof(KnownEndpointEntity.Name) : nameof(KnownEndpointEntity.Host); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"KnownEndpoints.{column} is NOT NULL and the document has no value for it")); + continue; + } + + rows.Add(new KnownEndpointEntity + { + Id = endpoint.EndpointDetails.GetDeterministicId(), + Name = endpoint.EndpointDetails.Name, + HostId = endpoint.EndpointDetails.HostId, + Host = endpoint.EndpointDetails.Host, + Monitored = endpoint.Monitored + }); + } + + return Task.FromResult(new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + skips, + Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 9ff490aa65..2b7acaa01e 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -38,6 +38,7 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont public DbSet LicensingEndpoints { get; set; } public DbSet LicensingEndpointThroughput { get; set; } public DbSet ExternalIntegrationDispatchRequests { get; set; } + public DbSet MigrationCheckpoints { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); @@ -80,6 +81,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new LicensingEndpointConfiguration()); modelBuilder.ApplyConfiguration(new LicensingEndpointThroughputConfiguration()); modelBuilder.ApplyConfiguration(new ExternalIntegrationDispatchRequestConfiguration()); + modelBuilder.ApplyConfiguration(new MigrationCheckpointConfiguration()); } public abstract bool IsDuplicateKeyException(DbUpdateException exception); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs new file mode 100644 index 0000000000..2f43e79e78 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +using ServiceControl.Persistence.DataMigration; + +public class MigrationCheckpointEntity +{ + public required string CategoryId { get; set; } + public MigrationCategoryState State { get; set; } + public string? Cursor { get; set; } + public long CopiedCount { get; set; } + public long SkippedCount { get; set; } + public long? SourceTotal { get; set; } + public IReadOnlyDictionary? SkipReasons { get; set; } + public DateTime? StartedAt { get; set; } + public DateTime? LastProgressAt { get; set; } + public DateTime? SettledAt { get; set; } + public string? LastError { get; set; } + public long AlreadyPresentCount { get; set; } + public long Version { get; set; } + public long? StartedWindowSeconds { get; set; } + + internal MigrationCheckpoint ToCheckpoint() => new( + CategoryId, State, Cursor, + CopiedCount, SkippedCount, SourceTotal, SkipReasons, + StartedAt, LastProgressAt, SettledAt, LastError, + AlreadyPresentCount, Version, StartedWindowSeconds); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs new file mode 100644 index 0000000000..9cc35d77f5 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using System.Text.Json; +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using ServiceControl.Persistence.DataMigration; + +class MigrationCheckpointConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => e.CategoryId); + builder.Property(e => e.CategoryId).HasMaxLength(ColumnLengths.ShortTextLength).ValueGeneratedNever(); + builder.Property(e => e.Version).IsConcurrencyToken(); + + // Null never reaches a converter: EF Core stores NULL for it, so a checkpoint with no skips has no JSON. + builder.Property(e => e.SkipReasons).HasConversion( + reasons => ToJson(reasons!), + json => FromJson(json), + new ValueComparer>( + (left, right) => left!.Count == right!.Count && !left.Except(right).Any(), + reasons => reasons.Aggregate(0, (hash, entry) => HashCode.Combine(hash, entry.Key, entry.Value)), + reasons => reasons.ToDictionary(entry => entry.Key, entry => entry.Value))); + } + + // Keyed by name, so a stored count keeps its meaning whatever order the enum's members are in. + static string ToJson(IReadOnlyDictionary reasons) => + JsonSerializer.Serialize(reasons.ToDictionary(entry => entry.Key.ToString(), entry => entry.Value)); + + // A name this build has no member for buckets into Unknown rather than throwing, because ReadAll is what + // decides whether the host may start; grouped because several unknown names collapse onto the one member. + static IReadOnlyDictionary FromJson(string json) => + JsonSerializer.Deserialize>(json)! + .GroupBy(entry => Enum.TryParse(entry.Key, out var reason) && Enum.IsDefined(reason) ? reason : MigrationSkipReason.Unknown) + .ToDictionary(group => group.Key, group => group.Sum(entry => entry.Value)); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs new file mode 100644 index 0000000000..3f343f44e3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +// Every method opens its own scope and context through DataStoreBase, so none of them can join a +// caller's transaction. Inside one, call ServiceControlDbContext.UpsertCheckpoint instead. +public class EFMigrationCheckpointStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationCheckpointStore +{ + public Task> ReadAll(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entities = await dbContext.MigrationCheckpoints.AsNoTracking().ToListAsync(token); + return (IReadOnlyList)[.. entities.Select(entity => entity.ToCheckpoint())]; + }, cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entity = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(e => e.CategoryId == categoryId, token); + return entity?.ToCheckpoint(); + }, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => dbContext.UpsertCheckpoint(checkpoint, token), cancellationToken); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs new file mode 100644 index 0000000000..864562c443 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; + +/// +/// The migration's provider-specific SQL. Statements run inside the caller's transaction and insert only what is absent. +/// +public interface IMigrationSqlDialect +{ + /// + /// Reads, once, whatever the statements need from the schema. On SQL Server that is the collation of every + /// string column of every primary key, composite keys included. + /// + Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + /// + /// How the database compares one string key column, so a caller can work out which of its own rows the insert will treat as one. The insert compares in the database itself; this only predicts what it will find. + /// Call first, and name a property that is a string column of the entity's primary key. + /// + IEqualityComparer KeyComparer(Type entityType, string propertyName); + + /// + /// How many rows the provider will take in one statement, for rows carrying this many values each. + /// + int RowsPerStatement(int parametersPerRow); + + /// + /// Inserts the rows whose key the table does not hold, merges any whose keys the database treats as one, and returns the rows it inserted. + /// + Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class; +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs new file mode 100644 index 0000000000..37ac476786 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs @@ -0,0 +1,141 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; + +/// +/// The table, columns and primary key maps to, and the run of one provider's insert statement over a chunk of those rows. +/// +public sealed class MigrationInsert where TEntity : class +{ + static readonly ExactKey KeyEquality = new(); + + readonly IReadOnlyList properties; + readonly IReadOnlyList keyProperties; + + MigrationInsert(string table, IReadOnlyList properties, IReadOnlyList keyProperties, Func columnName) + { + Table = table; + this.properties = properties; + this.keyProperties = keyProperties; + Columns = [.. properties.Select(columnName)]; + KeyColumns = [.. keyProperties.Select(columnName)]; + } + + /// The table name, quoted for this provider and carrying its schema. + public string Table { get; } + + /// Every column the insert writes, quoted, in the order the parameters of one row follow. + public IReadOnlyList Columns { get; } + + /// The primary key columns, quoted, in the order the statement has to return them. + public IReadOnlyList KeyColumns { get; } + + /// + /// Reads the table, columns and key out of the EF Core model for this entity. + /// + /// The entity is not mapped, has no primary key, or has a column this insert cannot write, such as one the database generates. Rows like that go through EF in the caller's context instead. + public static MigrationInsert For(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not an entity in the EF Core model, so the migration cannot tell which table it lands in."); + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not mapped to a table."); + var key = entityType.FindPrimaryKey() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no primary key, so there is nothing for a row to be absent by."); + IProperty[] properties = [.. entityType.GetProperties()]; + + if (properties.Any(property => property.ValueGenerated != ValueGenerated.Never) + || entityType.GetComplexProperties().Any() + || entityType.GetNavigations().Any(navigation => navigation.TargetEntityType.IsOwned())) + { + throw new InvalidOperationException($"{typeof(TEntity).Name} has a store-generated, complex or owned property, whose columns InsertMissing does not write. Add these rows through EF in the caller's context instead."); + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var sql = dbContext.GetService(); + + return new MigrationInsert( + sql.DelimitIdentifier(tableName, entityType.GetSchema()), + properties, + key.Properties, + property => sql.DelimitIdentifier(property.GetColumnName(storeObject) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name}.{property.Name} has no column in {tableName}."))); + } + + /// + /// Runs one insert statement over a chunk of rows and returns the rows of that chunk it inserted. + /// The statement reads parameters named @p0 upward, row after row in order, and must return the of each row it inserted, in that order. + /// It runs on the caller's open transaction, and throws when there is none or when a returned key matches no row of the chunk. + /// + public async Task> Execute(ServiceControlDbContext dbContext, string sql, TEntity[] chunk, CancellationToken cancellationToken = default) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("InsertMissing must run inside the caller's transaction, so its rows commit with the checkpoint.")).GetDbTransaction(); + command.CommandText = sql; + // A raw command starts at the provider's own 30 seconds, where SaveChangesAsync took the configured value. + command.CommandTimeout = dbContext.Database.GetCommandTimeout() ?? command.CommandTimeout; + + var index = 0; + foreach (var row in chunk) + { + foreach (var property in properties) + { + // The mapping applies the value converter and the provider type, as EF's own inserts do. + command.Parameters.Add(property.GetRelationalTypeMapping().CreateParameter(command, $"@p{index++}", property.GetGetter().GetClrValue(row), property.IsNullable)); + } + } + + var rowsByKey = new Dictionary(KeyEquality); + foreach (var row in chunk) + { + // A key repeated exactly is one row to every database, so the first row given carries it. + rowsByKey.TryAdd(KeyOf(row), row); + } + + var inserted = new List(chunk.Length); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + object?[] key = [.. keyProperties.Select((property, column) => FromProvider(property, reader.GetValue(column)))]; + + // A key can come back as a different CLR type than it went in as, a date column read back as + // DateTime say, and a plain lookup would report that as a bare KeyNotFoundException. + inserted.Add(rowsByKey.TryGetValue(key, out var row) + ? row + : throw new InvalidOperationException($"The {Table} insert returned the key [{string.Join(", ", key.Select(Describe))}], which matches no row in the batch that produced it. Compare it against the key types the batch carried: [{string.Join(", ", keyProperties.Select(property => property.ClrType.Name))}].")); + } + + return inserted; + } + + object?[] KeyOf(TEntity row) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(row))]; + + static string Describe(object? value) => value is null ? "null" : $"{value} ({value.GetType().Name})"; + + static object? FromProvider(IProperty property, object value) => + property.GetRelationalTypeMapping().Converter is { } converter ? converter.ConvertFromProvider(value) : value; + + // Exact, because the database decided which keys were one before the statement ran, and OUTPUT and RETURNING echo the value inserted. + sealed class ExactKey : IEqualityComparer + { + public bool Equals(object?[]? x, object?[]? y) => x!.SequenceEqual(y!); + + public int GetHashCode(object?[] key) + { + var hash = new HashCode(); + + foreach (var value in key) + { + hash.Add(value); + } + + return hash.ToHashCode(); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index e1b11c0057..4e958292ad 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -11,4 +11,7 @@ static class SettingKeys public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string LicensedEndpointDetailsAmendment = "LicensedEndpointDetailsAmendment"; public const string NotificationEmails = "NotificationEmails"; + // Written the first time a host starts on a database a migration has already written to. Once it is set, + // going back to RavenDB loses everything ServiceControl has written here since. + public const string MigrationHostOpenedOnTarget = "Migration/HostOpenedOnTarget"; } diff --git a/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs new file mode 100644 index 0000000000..db12c9b895 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.EFCore; + +using DbContexts; +using Entities; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +public static class MigrationCheckpointExtensions +{ + /// + /// Saves a migration checkpoint on the caller's own context, so it commits with that context's transaction + /// and flushes whatever else that context is tracking, and returns it carrying the version the save landed on. + /// Throws when the stored row no longer holds + /// . + /// + public static async Task UpsertCheckpoint(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + try + { + return await dbContext.UpsertCheckpointCore(checkpoint, cancellationToken); + } + catch (DbUpdateConcurrencyException e) + { + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, which the stored row no longer holds. Another writer advanced it, or a retried save is meeting its own committed write after a lost acknowledgement.", e); + } + } + + // Deliberately not UpsertAsync: its duplicate-key fallback reloads and saves again, which inside a + // caller-opened PostgreSQL transaction runs after the failed insert aborted it and fails with 25P02. + static async Task UpsertCheckpointCore(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken) + { + var entity = await dbContext.FindAsync([checkpoint.CategoryId], cancellationToken: cancellationToken); + + if (entity is null) + { + if (checkpoint.Version != 0) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but no row exists for it."); + } + + var inserted = Insert(checkpoint); + dbContext.Add(inserted); + + try + { + await dbContext.SaveChangesAsync(cancellationToken); + } + catch (DbUpdateException e) when (dbContext.IsDuplicateKeyException(e)) + { + // Another writer created this category's first checkpoint between the read and the insert. Not + // retried: PostgreSQL has already aborted the caller's transaction, so a retry cannot run. + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved as a first write, but another writer had already created it.", e); + } + + return inserted.ToCheckpoint(); + } + + // The expected version goes in the UPDATE's WHERE clause, so a save from an out-of-date copy matches no row. + dbContext.Entry(entity).Property(e => e.Version).OriginalValue = checkpoint.Version; + entity.Version = checkpoint.Version + 1; + Apply(checkpoint, entity); + + await dbContext.SaveChangesAsync(cancellationToken); + + return entity.ToCheckpoint(); + } + + static MigrationCheckpointEntity Insert(MigrationCheckpoint checkpoint) + { + var entity = new MigrationCheckpointEntity { CategoryId = checkpoint.CategoryId, Version = 1 }; + + Apply(checkpoint, entity); + + return entity; + } + + // One place both branches copy the row through, so a column added to the checkpoint cannot be written on + // an update and forgotten on an insert. + static void Apply(MigrationCheckpoint checkpoint, MigrationCheckpointEntity entity) + { + entity.State = checkpoint.State; + entity.Cursor = checkpoint.Cursor; + entity.CopiedCount = checkpoint.CopiedCount; + entity.SkippedCount = checkpoint.SkippedCount; + entity.SourceTotal = checkpoint.SourceTotal; + entity.SkipReasons = checkpoint.SkipReasons; + entity.StartedAt = checkpoint.StartedAt; + entity.LastProgressAt = checkpoint.LastProgressAt; + entity.SettledAt = checkpoint.SettledAt; + entity.LastError = checkpoint.LastError; + entity.AlreadyPresentCount = checkpoint.AlreadyPresentCount; + entity.StartedWindowSeconds = checkpoint.StartedWindowSeconds; + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs new file mode 100644 index 0000000000..8c05a27f97 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs @@ -0,0 +1,25 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Which documents one category is made of, and how they become rows. Read keeps the contract +/// sets out, for this one category. +/// +interface IMigrationCategoryReader +{ + /// The category this reader handles, named as in . + string CategoryId { get; } + + /// + /// The type of in every row yields. + /// + Type DocumentType { get; } + + IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs new file mode 100644 index 0000000000..590e534e18 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs @@ -0,0 +1,40 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// A reader whose every row carries a , which is the type its category's writer +/// must take. Reading through ties the documents RavenDB streams to that type. +/// +abstract class MigrationCategoryReader(RavenReadOnlySourceLifecycle lifecycle) : IMigrationCategoryReader where TDocument : class +{ + /// + /// The source connection, for a reader that streams or opens sessions itself instead of using . + /// + protected RavenReadOnlySourceLifecycle Lifecycle { get; } = lifecycle; + + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); + + /// + /// Streams the primary database's documents whose id starts with , each one whole as a row. + /// + protected IAsyncEnumerable WholeDocuments(string prefix, string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix( + Lifecycle, + CategoryId, + Lifecycle.Settings.DatabaseName, + prefix, + resumeAfter, + batchSize, + RavenDocumentStream.WholeDocument, + cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs new file mode 100644 index 0000000000..892641c1f5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs @@ -0,0 +1,82 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Threading; +using Raven.Client.Documents.Commands; +using ServiceControl.Persistence.DataMigration; + +/// +/// The one way every category reads RavenDB: stream the documents whose id starts with a prefix, in id order, +/// and hand them back in batches. Id order is what makes the cursor work, because a resume asks RavenDB to +/// start after an id rather than to skip a count. +/// +static class RavenDocumentStream +{ + /// + /// Streams one collection and yields it in batches of at most . + /// + /// The document id prefix that selects the collection, such as "KnownEndpoints/". + /// The cursor a previous run saved, or null to start at the beginning. + /// Turns one document into a row, or into null for a document this category does not copy. The cursor still moves past a document that becomes null. + /// The source holds no document with the id in , which means the cursor and the database no longer belong together. + public static async IAsyncEnumerable ByPrefix( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string databaseName, + string prefix, + string? resumeAfter, + int batchSize, + Func, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken = default) where TDocument : class + { + using var session = lifecycle.OpenSession(databaseName); + + // RavenDB starts a stream after whatever id it is given, even one that does not exist, so a stale cursor would skip rows nothing has copied. + if (resumeAfter is not null && !await session.Advanced.ExistsAsync(resumeAfter, cancellationToken)) + { + throw new InvalidOperationException( + $"The migration cannot resume the '{categoryId}' category after '{resumeAfter}', because the RavenDB database '{databaseName}' holds no document with that id. The cursor is saved in the target database and names a source document, so restoring RavenDB from a backup or changing which database it reads separates the two. Point the migration back at the RavenDB database this copy started from and restart."); + } + + await using var enumerator = await session.Advanced.StreamAsync( + prefix, startAfter: resumeAfter, token: cancellationToken); + + var rows = new List(batchSize); + var cursor = resumeAfter; + var lastYielded = resumeAfter; + + while (await enumerator.MoveNextAsync()) + { + // The cursor moves on every document the stream hands back, even one that becomes no row, so a resume never walks it again. + cursor = enumerator.Current.Id; + + if (project(enumerator.Current) is { } row) + { + rows.Add(row); + } + + if (rows.Count == batchSize) + { + yield return new MigrationBatch(rows, cursor); + rows = new List(batchSize); + lastYielded = cursor; + } + } + + // A tail whose documents all became no row still goes out, empty, so that the cursor past them is saved. + if (rows.Count > 0 || cursor != lastYielded) + { + yield return new MigrationBatch(rows, cursor!); + } + } + + public static MigrationRow WholeDocument(StreamResult result) where TDocument : class => + new(result.Id, result.Document, NoMetadata); + + static readonly IReadOnlyDictionary NoMetadata = ReadOnlyDictionary.Empty; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index cc3c59a998..8cb44a7cf7 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -3,6 +3,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System; +using System.Collections.Frozen; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -11,11 +12,25 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration.Readers; +/// +/// Reads a migration out of RavenDB. It holds one reader per category and knows nothing about any of them +/// beyond that, so a category this build cannot read is simply absent from +/// and never reaches the engine. +/// sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMigrationSource { + readonly FrozenDictionary readers = new IMigrationCategoryReader[] + { + new KnownEndpointsReader(lifecycle), + new EndpointSettingsReader(lifecycle) + }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); + public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); + public IReadOnlyList ContributedChecks() => []; + public async Task Describe(CancellationToken cancellationToken = default) { var settings = lifecycle.Settings; @@ -47,18 +62,37 @@ public async Task> Inventory(Cancel return entries; } - public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => - throw new NotSupportedException($"The RavenDB migration source cannot count category {category.Id} yet"); + // Counted through the reader the copy uses, not from RavenDB's collection statistics, because a collection does not map one to one onto a category. + public async Task Count(MigrationCategory category, CancellationToken cancellationToken = default) + { + var total = 0L; + + await foreach (var batch in Read(category, resumeAfter: null, batchSize: CountBatchSize, cancellationToken)) + { + total += batch.Rows.Count; + } + + return total; + } public IAsyncEnumerable Read( MigrationCategory category, string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - throw new NotSupportedException($"The RavenDB migration source cannot read category {category.Id} yet"); + readers.TryGetValue(category.Id, out var reader) + ? reader.Read(resumeAfter, batchSize, cancellationToken) + : throw new NotSupportedException($"The migration source cannot yet read the '{category.Id}' category."); public Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) => throw new NotSupportedException($"The RavenDB migration source cannot read bodies for category {category.Id} yet"); + public IReadOnlyCollection SupportedCategoryIds => readers.Keys; + + public IReadOnlyDictionary DocumentTypes => readers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); + + // Counting only adds up row counts, so this size changes nothing but how often the stream stops to hand one back. + const int CountBatchSize = 1024; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs index 354587cb5e..7adf8dc8c8 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -3,7 +3,9 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System; +using System.Diagnostics; using System.Net.Http; +using System.Security.Cryptography.X509Certificates; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; @@ -17,6 +19,11 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using ServiceControl.Configuration; using ServiceControl.RavenDB; +/// +/// The connection to the RavenDB database a migration reads from, and the embedded server it starts when that +/// database is a data directory rather than a URL. Nothing here writes, so the old database stays exactly as it +/// was and the copy can still be thrown away. +/// sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings, SettingsRootNamespace settingsRoot) : IAsyncDisposable { public RavenPersisterSettings Settings => settings; @@ -34,7 +41,7 @@ public async Task Open(CancellationToken cancellationToken = default) try { - var serverUrl = settings.UseEmbeddedServer ? StartEmbedded() : settings.ConnectionString; + var serverUrl = settings.UseEmbeddedServer ? await StartEmbedded(cancellationToken) : settings.ConnectionString; documentStore = Connect(serverUrl); if (!settings.UseEmbeddedServer) @@ -63,6 +70,8 @@ public IAsyncDocumentSession OpenSession(string databaseName) => async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) { + var elapsed = Stopwatch.StartNew(); + while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -72,10 +81,12 @@ async Task EnsureReadable(string databaseName, string settingKey, CancellationTo await DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); return; } + catch (DatabaseLoadTimeoutException e) when (settings.UseEmbeddedServer && elapsed.Elapsed >= EmbeddedLoadBudget) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but it did not finish loading within {EmbeddedLoadBudget.TotalMinutes:N0} minutes. A data directory held by another process, or one that is corrupt, is the usual cause.", e); + } catch (DatabaseLoadTimeoutException) when (settings.UseEmbeddedServer) { - // A large embedded database routinely exceeds the load timeout on first open, which - // RavenEmbeddedPersistenceLifecycle already allows for the same way. await Task.Delay(EmbeddedLoadRetryDelay, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -92,7 +103,7 @@ async Task EnsureReadable(string databaseName, string settingKey, CancellationTo } catch (Exception e) when (e is not DatabaseLoadTimeoutException) { - throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it.", e); + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it: {e.Message}", e); } } } @@ -103,13 +114,26 @@ internal static string Located(RavenPersisterSettings sourceSettings, SettingsRo ? $"{sourceSettings.ServerUrl} (embedded, data directory '{sourceSettings.DatabasePath}', from '{root}/{RavenBootstrapper.DatabasePathKey}')" : sourceSettings.ConnectionString; - string StartEmbedded() + async Task StartEmbedded(CancellationToken cancellationToken) { - var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode); + // A dynamic query is a POST to /queries, which the read-only guard allows, and it builds an auto-index on the + // customer's database. Nothing on the client can tell those queries apart, so the server is told to refuse them. + var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode) { DisableAutoIndexes = true }; embedded = EmbeddedDatabase.Start(configuration, lifetime); - return embedded.ServerUrl; + try + { + return await embedded.WaitUntilReady(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + throw new InvalidOperationException($"The RavenDB migration source could not start a server for the embedded database at {Located()}: {e.Message} A ServiceControl instance still holding that data directory is one cause, and stopping it lets the report run; a port already in use or a missing RavenDB server are others, which the message above tells apart.", e); + } } IDocumentStore Connect(string serverUrl) @@ -124,6 +148,7 @@ IDocumentStore Connect(string serverUrl) if (!settings.UseEmbeddedServer) { store.Certificate = RavenClientCertificate.FindClientCertificate(settings); + RefuseUnusableCertificate(store.Certificate); } store.OnBeforeRequest += RefuseWrite; @@ -131,6 +156,30 @@ IDocumentStore Connect(string serverUrl) return store.Initialize(); } + // An unusable certificate otherwise surfaces on the first request as a refused connection, which says nothing about the cause. + void RefuseUnusableCertificate(X509Certificate2? certificate) + { + if (certificate is null) + { + if (settings.ConnectionString.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"The RavenDB migration source at '{settings.ConnectionString}' is secured but no client certificate is configured. Set '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}'."); + } + + return; + } + + // X509Certificate2 reports both bounds in local time, so they move to UTC before the comparison. + var notBefore = certificate.NotBefore.ToUniversalTime(); + var notAfter = certificate.NotAfter.ToUniversalTime(); + var now = DateTime.UtcNow; + + if (now < notBefore || now > notAfter) + { + throw new InvalidOperationException($"The RavenDB client certificate '{certificate.Subject}' is valid from {notBefore:u} to {notAfter:u}, which does not include now."); + } + } + static void RefuseWrite(object? sender, BeforeRequestEventArgs e) { if (IsRead(e.Request.Method, new Uri(e.Url).AbsolutePath)) @@ -171,6 +220,8 @@ public async ValueTask DisposeAsync() static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); + // Generous because one DatabaseLoadTimeoutException already means RavenDB waited its own load timeout. + static readonly TimeSpan EmbeddedLoadBudget = TimeSpan.FromMinutes(5); IDocumentStore? documentStore; EmbeddedDatabase? embedded; diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs new file mode 100644 index 0000000000..087194c67c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the per-endpoint settings, whole, out of the primary database. The collection holds one document per +/// endpoint plus one with an empty name, which carries the default for every endpoint. +/// +sealed class EndpointSettingsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(EndpointSettingsStore.CollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs new file mode 100644 index 0000000000..3372368c5f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs @@ -0,0 +1,18 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the endpoints ServiceControl has heard from, whole, out of the primary database. +/// +sealed class KnownEndpointsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(RavenMonitoringDataStore.KnownEndpointsCollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs index 0792bcd8ff..5c88f09c5a 100644 --- a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs @@ -47,5 +47,5 @@ public async Task UpdateEndpointSettings(EndpointSettings settings, Cancellation await session.SaveChangesAsync(cancellationToken); } - const string CollectionName = "EndpointSettings"; + internal const string CollectionName = "EndpointSettings"; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs index a23b266f1b..dc2bcb5fc6 100644 --- a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs @@ -24,6 +24,8 @@ public Task Setup(IHostApplicationBuilder hostBuilder) return Task.CompletedTask; } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public Task PostSetup(IHost host) => Task.CompletedTask; public Task TearDown() => Task.CompletedTask; diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs index 4ce91496d5..b99a642473 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs @@ -51,7 +51,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -59,6 +59,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs new file mode 100644 index 0000000000..66b7214ec9 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +class KnownEndpointSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_known_endpoints_by_the_plural_prefix_in_batches_with_their_monitored_flag() + { + using (var session = await SessionProvider.OpenSession()) + { + foreach (var (name, monitored) in new[] { ("Sales.Orders", true), ("Billing", false), ("Shipping", false) }) + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }, + HostDisplayName = "HOST01", + Monitored = monitored + }; + + await session.StoreAsync(endpoint, $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"); + } + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var batches = await CollectBatches(source, category, batchSize: 2); + + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 1 })); + Assert.That(batches.SelectMany(batch => batch.Rows).Count(row => ((KnownEndpoint)row.Document).Monitored), Is.EqualTo(1)); + } + + // KnownEndpointsWriter skips these as RequiredValueMissing, which it can only do if the reader hands them over despite the required members. + [Test] + public async Task Reads_a_known_endpoint_missing_a_required_value_with_that_value_null() + { + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(new { EndpointDetails = new { HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = false }, "KnownEndpoints/no-name"); + await session.StoreAsync(new { EndpointDetails = new { Name = "Sales.Orders", HostId = Guid.NewGuid() }, Monitored = false }, "KnownEndpoints/no-host"); + await session.StoreAsync(new { Monitored = false }, "KnownEndpoints/no-details"); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var endpoints = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToDictionary(row => row.SourceId, row => (KnownEndpoint)row.Document); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints["KnownEndpoints/no-name"].EndpointDetails.Name, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-host"].EndpointDetails.Host, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-details"].EndpointDetails, Is.Null); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs new file mode 100644 index 0000000000..9fb6081bb3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration; + +[TestFixture] +class MigrationCategoryReadersTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Every_reader_claims_a_category_the_registry_knows() + { + await using var source = (RavenMigrationSource)await OpenMigrationSource(); + + var unknown = source.SupportedCategoryIds.Where(id => MigrationCategoryRegistry.Find(id) is null).ToArray(); + + Assert.That(unknown, Is.Empty, "a reader for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_reader_derives_from_the_typed_base() + { + var readers = typeof(IMigrationCategoryReader).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryReader).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(readers, Is.Not.Empty, "the test proves nothing if it finds no reader"); + Assert.That(readers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryReader<>))).Select(type => type.Name), Is.Empty, + "a reader that implements the interface directly can declare a DocumentType it does not yield, which the pairing test cannot see"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs new file mode 100644 index 0000000000..358c3f9943 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +abstract class RavenMigrationSourceTestBase : RavenPersistenceTestBase +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + static readonly object SettingsGate = new(); + + protected async Task OpenMigrationSource() + { + var settings = (RavenPersisterSettings)PersistenceSettings; + (string Name, string Value)[] variables = + [ + ("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", settings.ConnectionString), + ("SERVICECONTROL_RAVENDB_DATABASENAME", settings.DatabaseName), + ("SERVICECONTROL_ERRORRETENTIONPERIOD", settings.ErrorRetentionPeriod.ToString()), + ("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", settings.ThroughputDatabaseName) + ]; + + IMigrationSource source; + + // Environment variables are process wide and these tests run in parallel, so without the gate a source + // reads whichever test set them last and opens another test's database. + lock (SettingsGate) + { + try + { + foreach (var (name, value) in variables) + { + Environment.SetEnvironmentVariable(name, value); + } + + source = new RavenPersistenceConfiguration().CreateSource(SettingsRoot); + } + finally + { + foreach (var (name, _) in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + } + } + + await source.Open(); + + return source; + } + + protected static async Task> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs new file mode 100644 index 0000000000..c34641d535 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -0,0 +1,145 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +class RavenMigrationSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_endpoint_settings_in_document_id_order_in_batches_of_the_requested_size() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var batches = await CollectBatches(source, EndpointSettingsCategory, batchSize: 2); + + Assert.Multiple(() => + { + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 2, 1 }), "A batch never exceeds the requested size, and the trailing partial batch is still yielded."); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.Ordered); + }); + } + + [Test] + public async Task Every_row_is_identified_by_the_document_id_the_persister_wrote_it_under() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, EndpointSettingsCategory)).SelectMany(batch => batch.Rows).ToList(); + + Assert.Multiple(() => + { + Assert.That(rows, Has.Count.EqualTo(5)); + Assert.That(rows.Select(row => row.SourceId), Has.All.StartWith("EndpointSettings/"), "The cursor the engine checkpoints is a source document id, so a row identified by anything else cannot be resumed after."); + }); + } + + [Test] + public async Task Resuming_after_a_cursor_yields_only_what_follows_it() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var firstBatch = (await CollectBatches(source, EndpointSettingsCategory, batchSize: 2))[0]; + var resumed = (await CollectBatches(source, EndpointSettingsCategory, resumeAfter: firstBatch.Cursor, batchSize: 2)).SelectMany(batch => batch.Rows).ToList(); + + Assert.That(resumed, Has.Count.EqualTo(3)); + Assert.That(resumed.Select(row => row.SourceId), Has.No.Member(firstBatch.Rows[0].SourceId)); + Assert.That(resumed.Select(row => row.SourceId), Is.Ordered); + } + + [Test] + public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_instead_of_starting_past_it() + { + // The cursor is saved in the target database, so a restored backup or a corrected database name leaves + // one naming a document this source does not have. + await SeedEndpointSettings(3); + await using var source = await OpenMigrationSource(); + + var refusal = Assert.ThrowsAsync(() => CollectBatches(source, EndpointSettingsCategory, resumeAfter: "EndpointSettings/zzz-from-another-database")); + + Assert.Multiple(() => + { + Assert.That(refusal.Message, Does.Contain(EndpointSettingsCategory.Id)); + Assert.That(refusal.Message, Does.Contain("EndpointSettings/zzz-from-another-database")); + Assert.That(refusal.Message, Does.Contain(DatabaseName), "the operator has to be told which database the cursor was looked for in"); + }); + } + + [Test] + public async Task Counts_every_row_the_category_would_read() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.EqualTo(5)); + } + + [Test] + public async Task Counts_a_category_with_nothing_in_it_as_zero() + { + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.Zero, "A category with nothing to copy has to be told apart from one this source declines to count."); + } + + [Test] + public async Task Reading_every_category_creates_no_index() + { + await using var source = await OpenMigrationSource(); + + Assert.That(source.SupportedCategoryIds, Is.SubsetOf(Seeds.Keys), "A category read with nothing in it cannot show whether its reader builds an index, so a new reader needs its seed adding here."); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await Seeds[categoryId](this); + } + + var before = await IndexNames(); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await CollectBatches(source, MigrationCategoryRegistry.Find(categoryId)); + } + + var after = await IndexNames(); + + Assert.Multiple(() => + { + Assert.That(after, Is.EquivalentTo(before), "A reader that queries without naming a static index has RavenDB build one and index the whole collection on the customer's live database."); + Assert.That(after, Has.None.StartWith("Auto/")); + }); + } + + // Keyed by category id so a reader added without seed data fails the index test by name instead of passing + // over an empty collection. + static readonly Dictionary> Seeds = new() + { + [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), + [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }) + }; + + string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; + + async Task IndexNames() => + await DocumentStore.Maintenance.SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); + + const int IndexNamePageSize = 1024; + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.Find("EndpointSettings"); + + async Task SeedEndpointSettings(int count) + { + for (var index = 0; index < count; index++) + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = $"Endpoint{index}", TrackInstances = true }); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs index 859e02c505..ba3f892ffe 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -88,6 +88,20 @@ public async Task Opening_the_source_creates_no_database() Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentThroughput)), Is.Null, "Opening a migration source must not create a database that was missing."); } + [Test] + public async Task Opening_the_source_refuses_a_missing_primary_database() + { + var absentPrimary = $"{databaseName}-absent"; + sourceSettings.DatabaseName = absentPrimary; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentPrimary).And.Contain("ServiceControl/RavenDB/DatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentPrimary)), Is.Null, "Opening a migration source must not create a primary database that was missing."); + } + [Test] public async Task Opening_the_source_writes_no_database_settings() { diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs index b6610b3ec8..e318f8b450 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs @@ -52,6 +52,8 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) persistence.AddInstaller(hostBuilder.Services); } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public async Task PostSetup(IHost host) { DocumentStore = await host.Services.GetRequiredService().GetDocumentStore(); diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs new file mode 100644 index 0000000000..8a90e4965c --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class EndpointSettingsKeyCollationTests : PersistenceTestBase +{ + readonly TargetWarnings targetWarnings = new(); + + public EndpointSettingsKeyCollationTests() => RegisterServices = services => services.AddSingleton(targetWarnings); + + [Test] + public async Task A_case_merge_inside_one_batch_logs_both_names_and_the_one_kept() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + + [Test] + public async Task A_case_merge_with_a_row_an_earlier_batch_wrote_is_logged_and_a_row_written_again_is_not() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales")); + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + Assert.That(targetWarnings.Messages, Has.Exactly(1).Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + + async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // Set on the column, because the merge follows the column's collation and the test server's default can be either. + var ignoreCase = $""" + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE Latin1_General_CI_AS NOT NULL; + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await dbContext.Database.ExecuteSqlRawAsync(ignoreCase); + } + + foreach (var name in knownEndpointNames) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + await ServiceProvider.GetRequiredService().Open(); + } + + async Task Write(params (string SourceId, string Name)[] rows) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = true }, new Dictionary()))], + rows[^1].SourceId); + var checkpoint = await ServiceProvider.GetRequiredService().Read(category.Id) + ?? new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + await ServiceProvider.GetRequiredService().Write(category, batch, checkpoint); + } + + [Test] + public async Task The_key_columns_own_collation_decides_a_merge_when_the_database_default_disagrees() + { + bool databaseIgnoresCase; + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var database = dbContext.Database; + + databaseIgnoresCase = await database + .SqlQuery($"SELECT CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 AS [Value]") + .SingleAsync() == 1; + + // Named from the model, because the table sits in the configured schema when the persister has one. + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // The column is given the opposite of the database default, because a test where the two agree cannot show which one decided. + var recollate = $""" + DECLARE @columnCollation sysname = IIF(CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 = 1, N'Latin1_General_CS_AS', N'Latin1_General_CI_AS'); + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + EXEC (N'ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE ' + @columnCollation + N' NOT NULL'); + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await database.ExecuteSqlRawAsync(recollate); + } + + foreach (var name in new[] { "Sales", "sales" }) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + var target = ServiceProvider.GetRequiredService(); + await target.Open(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [ + new MigrationRow("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }, new Dictionary()), + new MigrationRow("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false }, new Dictionary()) + ], + "EndpointSettings/2"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(databaseIgnoresCase ? 2 : 1), "the column's own collation decides: two keys where it respects case, one where it ignores case, whatever the database default says"); + Assert.That( + ServiceProvider.GetRequiredService().KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)).Equals("Sales", "sales"), + Is.EqualTo(!databaseIgnoresCase), + "the dry run predicts merges with this comparer, so it follows the same column the statement does"); + } + } + + // RecordingLoggerProvider lives in ServiceControl.Infrastructure.Tests, which this project does not reference. + sealed class TargetWarnings : ILoggerProvider + { + readonly System.Collections.Concurrent.ConcurrentQueue messages = new(); + + public IReadOnlyList Messages => [.. messages]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(EFCoreMigrationTarget).FullName ? new WarningLogger(messages) : Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance; + + public void Dispose() + { + } + + sealed class WarningLogger(System.Collections.Concurrent.ConcurrentQueue messages) : ILogger + { + public IDisposable BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Warning; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (logLevel == LogLevel.Warning) + { + messages.Enqueue(formatter(state, exception)); + } + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs index 9ded3182bc..2dab62abe4 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs @@ -50,7 +50,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -58,6 +58,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs new file mode 100644 index 0000000000..67d7c9c307 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +class EFMigrationCheckpointStoreTests : PersistenceTestBase +{ + IMigrationCheckpointStore Store => ServiceProvider.GetRequiredService(); + + [Test] + public async Task Read_returns_null_for_a_category_that_has_never_run() + { + Assert.That(await Store.Read("EndpointSettings"), Is.Null); + } + + [Test] + public async Task Upsert_then_Read_round_trips_every_field() + { + var checkpoint = new MigrationCheckpoint( + "EndpointSettings", MigrationCategoryState.CompleteWithErrors, "cursor-1", 5, 2, 40, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable", AlreadyPresentCount: 33, + StartedWindowSeconds: (long)TimeSpan.FromDays(14).TotalSeconds); + + var saved = await Store.Upsert(checkpoint); + var stored = await Store.Read("EndpointSettings"); + + // Record equality compares SkipReasons by reference, and a round-tripped dictionary is a new instance. + using (Assert.EnterMultipleScope()) + { + Assert.That(stored! with { SkipReasons = null, Version = 0 }, Is.EqualTo(checkpoint with { SkipReasons = null })); + Assert.That(stored.SkipReasons, Is.EquivalentTo(checkpoint.SkipReasons)); + Assert.That(stored.Version, Is.EqualTo(1), "the first save leaves the row at version 1"); + // The engine saves from what Upsert hands back, so a returned version that is not the stored one + // makes the next save conflict against a row nothing else touched. + Assert.That(saved.Version, Is.EqualTo(stored.Version)); + } + } + + [Test] + public async Task A_second_Upsert_for_the_same_category_updates_rather_than_duplicates() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 0, 40, null, Now, Now, null, null)); + var saved = await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "cursor-2", 9, 1, 40, null, Now, Now, Now, null, Version: 1)); + + var all = await Store.ReadAll(); + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(all, Has.Count.EqualTo(1)); + Assert.That(saved.Version, Is.EqualTo(stored!.Version), "an update hands back the version it landed on, not the one it was saved from"); + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(9)); + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + } + } + + [Test] + public async Task A_checkpoint_saved_as_Abandoned_stores_its_counts_its_reasons_and_when_it_was_abandoned() + { + var reasons = new Dictionary { [MigrationSkipReason.BodyUnreadable] = 400 }; + var halted = Now.AddMinutes(-5); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Halted, "g-1", 12, 400, 412, reasons, Now, Now, halted, "body storage unavailable")); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Abandoned, "g-1", 12, 400, 412, reasons, Now, Now, Now, "body storage unavailable", Version: 1)); + + var stored = await Store.Read("GroupComments"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Abandoned)); + // State says which terminal state was reached and SettledAt says when, so abandoning a halted + // category moves the one timestamp on rather than filling a second column beside it. + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Not.EqualTo(halted)); + Assert.That(stored.SkippedCount, Is.EqualTo(400)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 400 })); + Assert.That(stored.LastError, Is.EqualTo("body storage unavailable")); + } + } + + [Test] + public async Task A_restart_after_a_halt_clears_the_fields_the_previous_save_set() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Halted, "cursor-1", 5, 2, 40, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable")); + + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 2, 40, + null, Now, Now, null, null, Version: 1)); + + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.SettledAt, Is.Null, "a resumed category must not look finished"); + Assert.That(stored.LastError, Is.Null); + Assert.That(stored.SkipReasons, Is.Null); + } + } + + [Test] + public void An_upsert_for_a_category_that_has_never_run_is_refused_when_it_carries_a_version() + { + var conflict = Assert.ThrowsAsync(() => Store.Upsert( + new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 1, 0, 40, null, Now, Now, null, null, Version: 1))); + + Assert.That(conflict!.Message, Does.Contain("EndpointSettings")); + } + + [Test] + public async Task ReadAll_returns_every_category_that_has_run() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "e-1", 1, 0, 1, null, Now, Now, Now, null)); + await Store.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.InProgress, "k-1", 4, 0, 9, null, Now, Now, null, null)); + + var all = await Store.ReadAll(); + + Assert.That(all.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "EndpointSettings", "KnownEndpoints" })); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..2826d26011 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs @@ -0,0 +1,143 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class EndpointSettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + IMigrationCheckpointStore CheckpointStore => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.EndpointSettings); + + [Test] + public async Task Every_setting_in_a_batch_arrives_with_its_track_instances_value() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false }), + ("EndpointSettings/3", new EndpointSettings { Name = "Shipping", TrackInstances = true })), + CheckpointAfter("EndpointSettings/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(3), "Copied is what the insert added, not the size of the batch"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + Assert.That(await TrackInstancesFor("Billing"), Is.False); + Assert.That(await TrackInstancesFor("Shipping"), Is.True); + } + } + + [Test] + public async Task A_name_already_in_the_target_is_left_alone_and_counted_as_already_present() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf(("EndpointSettings/7", new EndpointSettings { Name = "Sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/7")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a row the target already holds lost nothing, so it is not a skip"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + } + } + + [Test] + public async Task A_setting_whose_endpoint_is_not_known_is_copied_and_left_to_the_heartbeat_sync() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = string.Empty, TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Retired", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero, "whether a setting outlives its endpoint is the heartbeat sync's rule, not the migration's"); + Assert.That(await TrackInstancesFor(string.Empty), Is.True); + Assert.That(await TrackInstancesFor("Retired"), Is.False); + } + } + + [Test] + public async Task A_batch_that_copies_nothing_still_saves_the_cursor() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Billing", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = false }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + var stored = await CheckpointStore.Read(MigrationCategoryIds.EndpointSettings); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(stored.Cursor, Is.EqualTo("EndpointSettings/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + await Target.Write(EndpointSettingsCategory, BatchOf(("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true })), CheckpointAfter("EndpointSettings/1")); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.EndpointSettings.AsNoTracking().SingleAsync()); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); + + async Task TrackInstancesFor(string name) => + (await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync()).Single(settings => settings.Name == name).TrackInstances; + + [Test] + public async Task Two_names_in_one_batch_differing_only_in_case_are_each_copied_or_already_present() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2), "SQL Server's default collation makes these one key and PostgreSQL makes them two; either way neither row may throw or go uncounted"); + Assert.That(result.Skipped, Is.Zero, "a merge onto a row the batch writes loses nothing the target lacks, and a skip would count toward the halt threshold"); + Assert.That(await Target.Count(EndpointSettingsCategory), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); + Assert.That(await TrackInstancesFor("Sales"), Is.True, "the first row in document-id order wins where the key makes the two one"); + } + } + +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs new file mode 100644 index 0000000000..41cf3f5bd5 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs @@ -0,0 +1,199 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class KnownEndpointsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + [Test] + public async Task A_known_endpoint_is_written_with_its_monitored_flag() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + var stored = (await MonitoringDataStore.GetAllKnownEndpoints()).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(stored.Monitored, Is.True); + Assert.That(stored.EndpointDetails.Name, Is.EqualTo("Sales.Orders")); + } + } + + [Test] + public async Task A_known_endpoint_already_in_the_target_keeps_its_flag_and_is_counted_as_already_present() + { + var details = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }; + await MonitoringDataStore.CreateIfNotExists(details); + + var endpoint = new KnownEndpoint { EndpointDetails = details, HostDisplayName = "SALES01", Monitored = true }; + var sourceId = $"KnownEndpoints/{details.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Single().Monitored, Is.False, "insert-if-absent never updates, so the flag the target already had stays"); + } + } + + [Test] + public async Task A_known_endpoint_with_no_name_or_no_host_is_skipped_as_required_value_missing() + { + var named = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" } }; + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", named), ("KnownEndpoints/2", nameless), ("KnownEndpoints/3", hostless)), CheckpointAfter("KnownEndpoints/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EquivalentTo(new[] { "KnownEndpoints/2", "KnownEndpoints/3" })); + Assert.That(result.SkipReasons[MigrationSkipReason.RequiredValueMissing], Is.EqualTo(2), "KnownEndpoints.Name and Host are NOT NULL, and a throw here would halt a required category"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { "Sales.Orders" })); + } + } + + [Test] + public async Task A_batch_of_nothing_but_skips_saves_the_cursor_and_writes_no_rows() + { + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", nameless), ("KnownEndpoints/2", hostless)), CheckpointAfter("KnownEndpoints/2")); + + var stored = await ServiceProvider.GetRequiredService().Read(MigrationCategoryIds.KnownEndpoints); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero, "no key was looked up, so no row may be counted as one the target already held"); + Assert.That(result.Skipped, Is.EqualTo(2)); + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty); + Assert.That(stored.Cursor, Is.EqualTo("KnownEndpoints/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public void A_batch_reporting_more_copied_and_skipped_rows_than_it_held_is_refused() + { + // The guard only counts rows, so what the documents hold cannot change its answer. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + + var exception = Assert.Throws(() => EFCoreMigrationTarget.AlreadyPresentIn(KnownEndpointsCategory, batch, copied: 2, skipped: 1)); + + Assert.That(exception.Message, Does.Contain("copied 2").And.Contain("skipped 1").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + [Test] + public async Task A_batch_whose_checkpoint_save_fails_leaves_no_rows_behind() + { + // The rows save first and the checkpoint second, so this is the only order in which the two can part + // company: a stale version fails the checkpoint after the endpoint row is already in the transaction. + var store = ServiceProvider.GetRequiredService(); + await store.Upsert(CheckpointAfter("KnownEndpoints/0")); + + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + Assert.ThrowsAsync(async () => + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty, "the rows and the checkpoint commit together, so a failed checkpoint takes the rows with it"); + Assert.That((await store.Read(MigrationCategoryIds.KnownEndpoints)).Cursor, Is.EqualTo("KnownEndpoints/0"), "the stored cursor must still describe the rows the target actually holds"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.KnownEndpoints.AsNoTracking().SingleAsync()); + } + + + [Test] + public async Task A_batch_spanning_several_statements_counts_what_each_statement_inserted() + { + var rows = Enumerable.Range(0, 500) + .Select(index => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = $"Endpoint{index}", HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = index % 2 == 0 }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint)) + .ToArray(); + var batch = BatchOf(rows); + + var first = await Target.Write(KnownEndpointsCategory, batch, CheckpointAfter(batch.Cursor)); + // The engine carries the committed checkpoint into the next write, and the version guard refuses anything else. + var second = await Target.Write(KnownEndpointsCategory, batch, first.Saved with { Cursor = batch.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(500)); + Assert.That(second.Copied, Is.Zero, "every key is present on the second write, so no statement may report a row it did not insert"); + Assert.That(second.AlreadyPresent, Is.EqualTo(500)); + } + } + + + [Test] + public void A_batch_whose_writer_neither_prepared_nor_skipped_a_row_is_refused() + { + // The already-present count is a subtraction, so it only catches a writer that over-reports. A row dropped in silence under-reports, and without this guard it is counted as a row the target already held. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + var prepared = new PreparedBatch((_, _) => Task.FromResult(1), PreparedRowCount: 1, Skips: [], Merges: []); + + var exception = Assert.Throws(() => EFCoreMigrationTarget.AccountForEveryRow(KnownEndpointsCategory, batch, prepared)); + + Assert.That(exception.Message, Does.Contain("prepared 1").And.Contain("skipped 0").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs new file mode 100644 index 0000000000..7fa6186c1e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; + +class MigrationCategoryWritersTests : PersistenceTestBase +{ + [Test] + public void Every_writer_claims_a_category_the_registry_knows() + { + var unknown = ServiceProvider.GetRequiredService().SupportedCategoryIds + .Where(id => MigrationCategoryRegistry.Find(id) is null) + .ToArray(); + + Assert.That(unknown, Is.Empty, "a writer for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_writer_derives_from_the_typed_base() + { + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(writers, Is.Not.Empty, "the test proves nothing if it finds no writer"); + Assert.That(writers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryWriter<>))).Select(type => type.Name), Is.Empty, + "a writer that implements the interface directly skips the named cast and can declare a DocumentType it does not take"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); + + [Test] + public async Task Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type() + { + var target = ServiceProvider.GetRequiredService(); + await target.Open(); + + using (Assert.EnterMultipleScope()) + { + foreach (var (categoryId, document) in target.SupportedCategoryIds.SelectMany(id => new (string, object)[] { (id, new object()), (id, null) })) + { + var category = MigrationCategoryRegistry.Find(categoryId)!; + var batch = new MigrationBatch([new MigrationRow("Wrong/1", document, new Dictionary())], "Wrong/1"); + var checkpoint = new MigrationCheckpoint(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + var exception = Assert.ThrowsAsync(() => target.Write(category, batch, checkpoint)); + + Assert.That(exception!.Message, Does.Contain(categoryId).And.Contain("Wrong/1").And.Contain(target.DocumentTypes[categoryId].FullName), "an engine halt quotes this message, so it has to say which pair disagrees"); + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs new file mode 100644 index 0000000000..d3ede8f8ec --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections; +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using NUnit.Framework; + +static class MigrationEntityCoverage +{ + public static void AssertEveryMappedPropertyIsSet(IModel model, TEntity entity, params string[] legitimatelyDefault) + { + var entityType = model.FindEntityType(typeof(TEntity)) + ?? throw new ArgumentException($"{typeof(TEntity).Name} is not an entity in the EF Core model.", nameof(entity)); + + // Reflection rather than a new() constraint, which an entity with required members cannot satisfy. + var defaults = Activator.CreateInstance(typeof(TEntity)); + + // Every property in this model that is not ValueGenerated.Never is an identity column the database assigns. + var unset = entityType.GetProperties() + .Where(property => property.ValueGenerated == ValueGenerated.Never && !legitimatelyDefault.Contains(property.Name)) + .Where(property => SameValue(property.GetGetter().GetClrValue(entity), property.GetGetter().GetClrValue(defaults))) + .Select(property => property.Name) + .ToArray(); + + Assert.That(unset, Is.Empty, $"{typeof(TEntity).Name} has mapped properties equal to a default-constructed {typeof(TEntity).Name}. Set each from the source document, feed the test a document whose values all differ from those defaults, or name the property in legitimatelyDefault if RavenDB genuinely holds that value."); + } + + // Collections compare by item, and an empty one is unset whether or not the entity initialises it. + static bool SameValue(object actual, object defaultValue) => + actual is IEnumerable items and not string + ? !items.Cast().Any() || (defaultValue is IEnumerable defaultItems && items.Cast().SequenceEqual(defaultItems.Cast())) + : Equals(actual, defaultValue); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs new file mode 100644 index 0000000000..d5d0fedf42 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DbContexts; + +class MigrationEntityCoverageClaimTests : PersistenceTestBase +{ + [Test] + public void Every_property_the_database_fills_in_is_a_key_it_assigns_on_insert() + { + using var scope = ServiceProvider.CreateScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var databaseFilled = model.GetEntityTypes() + .SelectMany(entityType => entityType.GetProperties()) + .Where(property => property.ValueGenerated != ValueGenerated.Never) + .ToArray(); + + var notKeysAssignedOnInsert = databaseFilled + .Where(property => !property.IsPrimaryKey() || property.ValueGenerated != ValueGenerated.OnAdd) + .Select(property => $"{property.DeclaringType.ClrType.Name}.{property.Name} is {property.ValueGenerated}") + .ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(databaseFilled, Is.Not.Empty, "the model no longer has a single property the database fills in, so this test is watching nothing. Either the identity keys have gone, or the model was never built."); + Assert.That(notKeysAssignedOnInsert, Is.Empty, "MigrationEntityCoverage checks only the properties that are ValueGenerated.Never, on the claim that every other one is a key the database assigns on insert. These properties break that claim, so the coverage check silently stops watching them and a migration can copy a row with them left unset."); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs new file mode 100644 index 0000000000..da87ec7dcd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +class MigrationTargetCoverageTests : PersistenceTestBase +{ + [Test] + public async Task Every_category_the_target_supports_has_a_batch_size_and_a_count() + { + var target = ServiceProvider.GetRequiredService(); + + Assert.That(target.SupportedCategoryIds, Is.SupersetOf(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }), "the test proves nothing if the target supports no category"); + + foreach (var id in target.SupportedCategoryIds) + { + var category = MigrationCategoryRegistry.Find(id)!; + + Assert.That(await target.BatchSizeFor(category, CancellationToken.None), Is.Positive, $"{id}: every supported category has a batch size"); + Assert.That(await target.Count(category, CancellationToken.None), Is.Zero, id); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs new file mode 100644 index 0000000000..caffb35e0e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs @@ -0,0 +1,226 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationTargetReadinessTests : PersistenceTestBase +{ + IMigrationTargetReadiness Readiness => ServiceProvider.GetRequiredService(); + + [Test] + public void The_target_contributes_the_three_checks_only_it_can_make() => + Assert.That( + Readiness.ContributedChecks().Select(check => check.GetType()), + Is.EqualTo(new[] { typeof(SchemaIsCurrentCheck), typeof(TargetHoldsNoServiceControlDataCheck), typeof(BodyStorageIsWritableCheck) })); + + [Test] + public void Every_contributed_check_passes_against_a_migrated_database() + { + using (Assert.EnterMultipleScope()) + { + foreach (var check in Readiness.ContributedChecks()) + { + Assert.DoesNotThrowAsync(() => check.Run(), $"a healthy target must pass every contributed check, and it failed '{check.Name}'"); + } + } + } + + [Test] + public async Task A_target_holding_a_row_is_refused_while_no_checkpoint_exists() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName()).And.Contain("--setup")); + } + + // A plain start writes this row first, so a key someone later ignores fails here before it reaches a customer. + [Test] + public async Task A_settings_row_counts_as_data() + { + await ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2030, 1, 1)); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName())); + } + + [Test] + public async Task A_target_with_a_required_checkpoint_row_is_not_judged() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + [Test] + public async Task A_row_under_an_id_this_build_does_not_know_counts_as_a_started_copy() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint("SomeCategoryFromANewerBuild", MigrationCategoryState.InProgress); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + // --migration-abandon writes this row before any copy has run, so on a database that already served on SQL it must not switch the check off. + [Test] + public async Task An_abandoned_optional_row_alone_does_not_skip_the_check() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.Abandoned); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName())); + } + + [Test] + public void Every_mapped_entity_but_the_checkpoint_is_judged() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var mapped = dbContext.Model.GetEntityTypes().Select(type => type.ClrType).Where(type => type != typeof(MigrationCheckpointEntity)); + + Assert.That(TargetHoldsNoServiceControlDataCheck.Tables.Select(entry => entry.Entity), Is.EquivalentTo(mapped)); + } + + IMigrationStartupCheck TargetCheck => Readiness.ContributedChecks().OfType().Single(); + + Task SeedCheckpoint(string categoryId, MigrationCategoryState state) => + ServiceProvider.GetRequiredService().Upsert(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + + string TableName() + { + using var scope = ServiceProvider.CreateScope(); + + return scope.ServiceProvider.GetRequiredService().Model.FindEntityType(typeof(T))!.GetTableName()!; + } + + [Test] + public async Task The_schema_check_refuses_a_database_whose_migrations_have_not_been_applied() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService(); + + foreach (var row in await history.GetAppliedMigrationsAsync()) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + var exception = Assert.ThrowsAsync(() => Readiness.ContributedChecks().OfType().Single().Run()); + + Assert.That(exception.Message, Does.Contain(dbContext.Database.GetMigrations().First()).And.Contain("--setup")); + } + + [Test] + public async Task The_body_storage_check_leaves_no_probe_body_behind() + { + var bodyStorage = ServiceProvider.GetRequiredService(); + + await Readiness.ContributedChecks().Single(check => check.Name == "message body storage is writable").Run(); + + Assert.That(await bodyStorage.ReadBody("migration-writable-probe"), Is.Null, "a probe left in the store is a body the source never had, which a count comparison reads as an extra rather than a loss"); + } + + [Test] + public void The_body_storage_check_refuses_a_store_that_cannot_write() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + var check = new BodyStorageIsWritableCheck(new FileSystemBodyStoragePersistence( + new FileSystemBodyStorageSettings { StoragePath = Path.Combine(parentThatIsAFile, "bodies") })); + + Assert.CatchAsync(() => check.Run()); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + // The copy runs before any hosted service starts, so a target that only works once one has started is broken exactly when the migration needs it. + [Test] + public async Task The_target_answers_from_a_container_whose_hosted_services_have_not_started() + { + var (host, context) = await BuildHostWithoutStarting(); + + try + { + await using var scope = host.Services.GetRequiredService().CreateAsyncScope(); + + Assert.That(await scope.ServiceProvider.GetRequiredService().EndpointSettings.LongCountAsync(), Is.Zero); + Assert.DoesNotThrowAsync(() => host.Services.GetRequiredService().ContributedChecks().Single(check => check.Name == "message body storage is writable").Run()); + } + finally + { + await context.TearDown(); + host.Dispose(); + } + } + + // PersistenceTestBase already started a host in SetUp, so this builds a second one over its own test database. + static async Task<(IHost Host, PersistenceTestsContext Context)> BuildHostWithoutStarting() + { + var context = new PersistenceTestsContext(); + var hostBuilder = Host.CreateApplicationBuilder(); + + await context.Setup(hostBuilder); + var host = hostBuilder.Build(); + await context.InstallSchema(host); + + return (host, context); + } + + [Test] + public async Task The_host_opened_record_is_absent_until_written_and_stays_after_a_second_write() + { + var readiness = ServiceProvider.GetRequiredService(); + + Assert.That(await readiness.HasHostOpened(), Is.False, "a target nothing has opened on must not claim otherwise"); + + await readiness.RecordHostOpened(); + var firstOpened = Now; + + AdvanceClock(TimeSpan.FromDays(7)); + await readiness.RecordHostOpened(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await readiness.HasHostOpened(), Is.True); + // The stored instant is what tells an operator the clean abort is over, so a later start must not move it forward. + Assert.That(await ReadHostOpenedAt(), Is.EqualTo(firstOpened)); + } + } + + async Task ReadHostOpenedAt() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + return await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs new file mode 100644 index 0000000000..5261d9fffd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTableTests : PersistenceTestBase +{ + [Test] + public async Task The_table_round_trips_every_column_written_directly_through_the_DbContext() + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "EndpointSettings", + State = MigrationCategoryState.InProgress, + Cursor = "cursor-1", + CopiedCount = 3, + SkippedCount = 1, + SourceTotal = 40, + SkipReasons = new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 }, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = null, + LastError = null, + AlreadyPresentCount = 36, + Version = 7 + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(3)); + Assert.That(stored.SkippedCount, Is.EqualTo(1)); + Assert.That(stored.SourceTotal, Is.EqualTo(40)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(stored.AlreadyPresentCount, Is.EqualTo(36)); + Assert.That(stored.Version, Is.EqualTo(7)); + Assert.That(stored.StartedAt, Is.EqualTo(Now)); + Assert.That(stored.LastProgressAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Null); + Assert.That(stored.LastError, Is.Null); + } + } + + [Test] + public async Task Every_timestamp_column_comes_back_as_a_Utc_DateTime() + { + Assert.That(Now.Kind, Is.EqualTo(DateTimeKind.Utc), "the engine stamps timings from TimeProvider.GetUtcNow().UtcDateTime, so the test clock has to match"); + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "KnownEndpoints", + State = MigrationCategoryState.Complete, + Cursor = null, + CopiedCount = 1, + SkippedCount = 0, + SourceTotal = 1, + SkipReasons = null, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = Now, + LastError = null + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "KnownEndpoints"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.StartedAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.LastProgressAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.SettledAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public void The_skip_reasons_JSON_is_keyed_by_reason_name_not_by_its_number() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 })!; + + // Enum.Parse accepts "0" as readily as "BodyUnreadable", so a round trip cannot tell the two encodings apart. + Assert.That(json, Does.Contain("BodyUnreadable")); + } + + [Test] + public void A_blank_group_comment_skip_round_trips_by_name() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary { [MigrationSkipReason.BlankGroupComment] = 3 })!; + var reasons = (IReadOnlyDictionary)converter.ConvertFromProvider(json)!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(json, Does.Contain("BlankGroupComment")); + Assert.That(reasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BlankGroupComment] = 3 })); + } + } + + [Test] + public void A_reason_name_this_build_does_not_know_is_read_as_Unknown_rather_than_throwing() + { + // The rows a newer build wrote are what a downgraded instance meets, and the read path that would + // throw is ReadAll, which is what decides whether the host may start. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var reasons = (IReadOnlyDictionary)converter.ConvertFromProvider( + """{"BodyUnreadable":2,"SomethingFromTheFuture":3,"AlsoFromTheFuture":1}""")!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(reasons[MigrationSkipReason.BodyUnreadable], Is.EqualTo(2)); + Assert.That(reasons[MigrationSkipReason.Unknown], Is.EqualTo(4), "several unknown names collapse onto the one member, so their counts are summed rather than one of them winning"); + Assert.That(reasons, Has.Count.EqualTo(2)); + } + } + + [Test] + public void The_table_carries_exactly_the_columns_the_checkpoint_needs() + { + // The one thing the one-way door makes permanent. Adding, dropping or renaming one after this + // ships costs a migration against customer data, so it fails here rather than in a customer's + // database. Property names rather than column names, because each provider spells the columns + // its own way and the spelling is a convention the generated migrations already pin. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var entityType = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(entityType.GetProperties().Select(property => property.Name), Is.EquivalentTo(new[] + { + "CategoryId", "State", "Cursor", "CopiedCount", "SkippedCount", "SourceTotal", + "SkipReasons", "StartedAt", "LastProgressAt", "SettledAt", "LastError", "AlreadyPresentCount", "Version", "StartedWindowSeconds" + })); + + // Underscores stripped so one assertion covers MigrationCheckpoints and migration_checkpoints. + Assert.That(entityType.GetTableName()!.Replace("_", ""), Is.EqualTo("MigrationCheckpoints").IgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs new file mode 100644 index 0000000000..706f41f046 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs @@ -0,0 +1,178 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTransactionTests : PersistenceTestBase +{ + static MigrationCheckpoint Checkpoint(DateTime now, string cursor, long copied, long version = 0) => + new("EndpointSettings", MigrationCategoryState.InProgress, cursor, copied, 0, 40, null, now, now, null, null, Version: version); + + async Task SeedCheckpoint(string cursor, long copied) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + await dbContext.UpsertCheckpoint(Checkpoint(Now, cursor, copied)); + } + + [Test] + public async Task A_failure_before_commit_leaves_neither_the_row_nor_the_checkpoint() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.False); + } + } + + [Test] + public async Task A_successful_commit_lands_the_row_and_the_checkpoint_together() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.True); + } + } + + [Test] + public async Task An_existing_checkpoint_is_advanced_by_a_commit_inside_the_callers_transaction() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task An_existing_checkpoint_keeps_its_previous_cursor_when_the_transaction_rolls_back() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_save_built_from_an_out_of_date_copy_is_refused_rather_than_silently_winning() + { + var store = ServiceProvider.GetRequiredService(); + await store.Upsert(Checkpoint(Now, "cursor-1", 1)); + + // Both read the row at the same version, as a background copy and a required copy on two hosts would. + var first = (await store.Read("EndpointSettings"))!; + var stale = (await store.Read("EndpointSettings"))!; + + await store.Upsert(first with { Cursor = "cursor-first", CopiedCount = 2 }); + + Assert.ThrowsAsync(() => store.Upsert(stale with { Cursor = "cursor-stale", CopiedCount = 99 })); + + var stored = (await store.Read("EndpointSettings"))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Cursor, Is.EqualTo("cursor-first")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + Assert.That(stored.Version, Is.EqualTo(first.Version + 1)); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs index 15d9b76981..5e238f2d22 100644 --- a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs @@ -10,6 +10,13 @@ public interface IPersistenceTestsContext { Task Setup(IHostApplicationBuilder hostBuilder); + /// + /// Puts the schema in place on the built host, before it is started, the way --setup does in + /// production. The EF Core persisters refuse to start against a database whose schema predates the + /// build, and that check runs before any hosted service, so migrating after the start is too late. + /// + Task InstallSchema(IHost host); + Task PostSetup(IHost host); Task TearDown(); diff --git a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs index f065c190f6..dc5ae1178e 100644 --- a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs @@ -53,6 +53,7 @@ public async Task SetUp() host = hostBuilder.Build(); + await PersistenceTestsContext.InstallSchema(host); await host.StartAsync(); await PersistenceTestsContext.PostSetup(host); } diff --git a/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs new file mode 100644 index 0000000000..7bf02d735b --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Answers from the saved checkpoints, read once before the host opens rather than queried live. +/// +public sealed class CheckpointMigrationState(IMigrationCheckpointStore? checkpointStore = null) : IMigrationState +{ + volatile bool anyCategoryIncomplete; + IReadOnlyCollection selectedCategoryIds = []; + + public bool AnyCategoryIncomplete => anyCategoryIncomplete; + + /// + /// Reads every checkpoint and fixes the answer for the life of the host. + /// + /// The required categories this instance copies, which are the only ones that hold back the services that delete or overwrite copied data. They are passed in because the store holds a row only for a category some run has already been given. + public async Task Seed(IReadOnlyCollection selectedIds, CancellationToken cancellationToken = default) + { + // Only a persister that can be migrated into keeps checkpoints, so no store means no migration has run. + if (checkpointStore is null) + { + return; + } + + selectedCategoryIds = selectedIds; + + Recompute(await checkpointStore.ReadAll(cancellationToken)); + } + + /// + /// Works the answer out again from the checkpoints given, without reading the store. + /// + /// Every checkpoint the store holds. A selected category with no checkpoint among them has not started, which is not finished. + public void Recompute(IReadOnlyList checkpoints) => + anyCategoryIncomplete = selectedCategoryIds.Any(id => + checkpoints.FirstOrDefault(checkpoint => checkpoint.CategoryId == id)?.State.IsFinished() != true); +} diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs index 92d6a73ce7..534e120b7d 100644 --- a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -1,8 +1,19 @@ namespace ServiceControl.Persistence.DataMigration; +/// +/// The rule that stops a category early because it has lost too much to keep copying. +/// public static class HaltThreshold { - // Both must be exceeded: the floor ignores a few bad rows in a small category, the percentage a small share of a large one. + /// + /// Whether the skipped rows are enough to stop the category. Both limits have to be passed, so the floor + /// ignores a few bad rows in a small category and the percentage ignores a small share of a large one. + /// + /// Rows skipped as faults. A skip the product would have dropped anyway does not belong here. + /// Rows dealt with over the same stretch as , copied, skipped and already present alike. + /// The share of skipped rows, as a percentage, that has to be passed. + /// The number of skipped rows that has to be passed before the percentage counts at all. + /// True when both limits are passed, which means the category stops. public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) { if (skippedCount <= minimumFloor || totalCount == 0) diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs index bc71283d01..0731525dab 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -6,18 +6,61 @@ namespace ServiceControl.Persistence.DataMigration; using System.Threading; using System.Threading.Tasks; +/// +/// Where one category has got to. says which of these +/// let the host open, and which wait for the operator. A +/// start picks up every category that is neither. +/// public enum MigrationCategoryState { + /// No run has read a row of this category yet. NotStarted, + + /// + /// A run is copying this category, or a run stopped without settling it. On an optional category, a LastError + /// means an exception stopped it or a start could not open the source, and the next start resumes it. + /// InProgress, + + /// + /// The category reached the end of the source with no fault skips. Harmless ones stay counted. + /// Complete, + + /// + /// The category reached the end of the source with fault skips, so it is Failed until the operator retries or abandons it. + /// CompleteWithErrors, + + /// + /// The category stopped early. It waits for --migration-retry or --migration-abandon, and no start re-reads it. + /// Halted, + + /// + /// The operator gave up on a Failed or started required category, on any optional one, or on one whose rows + /// depend on such a category. It is final: nothing copies it again, even in a later migration. + /// Abandoned, + + /// + /// This category follows a category that is not Done or Abandoned, so it did not run. A restart clears it once that one is. + /// Blocked } -/// A category's saved progress: where a restart carries on from, and what the status and verify commands report. +/// +/// A category's saved progress: where a restart carries on from, and what the status and verify commands report. +/// Every count is the total across every run, not this run alone. +/// +/// The point the last committed batch reached. A restart reads the source after it. Null means nothing has been read. +/// No longer written: the engine does not count the source, so this is null on every row it saves. The column stays only because dropping it is a schema change. +/// How many rows each reason skipped. The counts here add up to . +/// The moment the category stopped running, whatever state it stopped in. Read it beside , because a halt settles too. +/// Why the category stopped or did not run, in the words the operator is shown: a halt, a category it must follow that is not finished, an exception that left an optional category copying, or a source a start could not open, on an optional category still copying or not started. It stays until a start runs the category again. +/// Rows the target already held, so they were neither copied nor skipped. They still count as accounted for. +/// The optimistic concurrency token, which is the guard against two writers. A store sets it on save and refuses a checkpoint carrying a value the stored row no longer holds. +/// The window, in whole seconds, an optional category started with. It is null until the copier writes it, and always null for a required category. public sealed record MigrationCheckpoint( string CategoryId, MigrationCategoryState State, @@ -28,14 +71,22 @@ public sealed record MigrationCheckpoint( IReadOnlyDictionary? SkipReasons, DateTime? StartedAt, DateTime? LastProgressAt, - // The moment the category stopped running, whatever state it stopped in. Read it beside State: a halt settles too. DateTime? SettledAt, string? LastError, long AlreadyPresentCount = 0, - // The optimistic concurrency token. A store sets it on save and refuses one carrying a value the stored row no longer holds. - long Version = 0) + long Version = 0, + long? StartedWindowSeconds = null) { - /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, so the saved counts are the real ones. + /// + /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, + /// so the saved counts are the real ones. + /// + /// Rows this batch wrote. + /// Rows this batch could not write. Every one of them needs a reason. + /// Rows this batch found the target already held. + /// How many rows each reason skipped in this batch. + /// A copy with this batch's counts and reasons added to the totals. + /// The reasons do not add up to , which would leave the verify command unable to account for a row. public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, IReadOnlyDictionary? skipReasons) { var explained = skipReasons?.Values.Sum() ?? 0; @@ -70,11 +121,26 @@ public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, I } } +/// +/// Where the checkpoints are kept. The target database holds them, so progress and the rows it describes +/// commit together. +/// public interface IMigrationCheckpointStore { + /// + /// Every checkpoint the store holds. A run saves a not-started row for every category it will copy before it + /// copies any of them, so a category is absent when no run has recorded it. + /// Task> ReadAll(CancellationToken cancellationToken = default); + + /// + /// One category's checkpoint, or null when no run has recorded it. + /// Task Read(string categoryId, CancellationToken cancellationToken = default); - /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. Throws when the stored row has moved on. + /// + /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. + /// + /// The stored row has moved on, which means another writer saved it, so this save is refused. Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs index bd19e623aa..7e969ffa5f 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -5,21 +5,41 @@ namespace ServiceControl.Persistence.DataMigration; using System.Threading; using System.Threading.Tasks; +/// +/// Implemented by a persister that can be read as the old database a migration copies from. Nothing here +/// writes to the source, because the old database has to stay usable if the migration is thrown away. +/// public interface IMigrationSource : IAsyncDisposable { - /// Connects to the source read-only. Every other member throws until this has run. + /// + /// Connects to the source read-only. Every other member except throws + /// until this has run. + /// Task Open(CancellationToken cancellationToken = default); - /// What the source report and dry run print about the source. + /// + /// The checks this source wants run before the copy starts, in the order they must run. Call it after Open. + /// + IReadOnlyList ContributedChecks(); + + /// + /// What the source report and the dry run print about the source. + /// Task Describe(CancellationToken cancellationToken = default); - /// A count of everything the source holds, including data no category copies. + /// + /// A count of everything the source holds, including data no category copies. + /// Task> Inventory(CancellationToken cancellationToken = default); - /// How many rows the source holds for one category, for progress and verify. + /// + /// How many rows the source holds for one category, which --migration-verify reports. + /// Task Count(MigrationCategory category, CancellationToken cancellationToken = default); - /// Reads a category in batches, after the checkpoint cursor if provided, or from the start when it is null. Throws on a cursor it never issued. + /// + /// Reads a category in batches, after the checkpoint cursor, or from the start when it is null. Throws on a cursor it never issued. + /// /// A ceiling, not a target: returning fewer costs nothing, returning more fails the target's write. IAsyncEnumerable Read( MigrationCategory category, @@ -27,6 +47,21 @@ IAsyncEnumerable Read( int batchSize, CancellationToken cancellationToken = default); - /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + /// + /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + /// Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); + + /// + /// The categories this source can read. A category outside this set is never handed to the engine. + /// Answers before Open and does not change across it. + /// + IReadOnlyCollection SupportedCategoryIds { get; } + + /// + /// The type of in every row this source reads, for each category in + /// . The target's writer for the same category must take that type. + /// Answers before Open and does not change across it. + /// + IReadOnlyDictionary DocumentTypes { get; } } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs new file mode 100644 index 0000000000..bd08c1df27 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// One check the host runs before the copy starts. The host, the source and the target each contribute their own. +/// +public interface IMigrationStartupCheck +{ + /// + /// What the check is called in the refusal the host prints, so it is phrased to finish the sentence + /// "Migration startup check '...' failed". + /// + string Name { get; } + + /// + /// Runs the check. Throws when it fails, which stops the host before anything is copied. + /// + Task Run(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs new file mode 100644 index 0000000000..212e8694a5 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// Whether this instance is still part way through copying its old database in. +/// +public interface IMigrationState +{ + /// + /// True while any required category is unfinished. Services that delete or overwrite copied data stand down while it + /// is true. Optional categories never make it true, because they copy beside those services. + /// + bool AnyCategoryIncomplete { get; } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index 78026fc571..1e457a27a8 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -1,27 +1,64 @@ namespace ServiceControl.Persistence.DataMigration; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -/// Implemented by a persister that can be the new database a migration copies into. +/// +/// Implemented by a persister that can be the new database a migration copies into. +/// public interface IMigrationTarget { - /// The most rows a source may return in one batch for this category. The target picks it because its own database sets the limit, and a batch over it fails the write. - int BatchSizeFor(MigrationCategory category); + /// + /// Makes the target ready to write. The host calls it before the copy and after the target's own checks have passed. + /// + Task Open(CancellationToken cancellationToken = default); - /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. Extend checkpointToExtend with this batch's own outcome through and save the result, so what lands is the real split rather than a guess the next save has to correct. - /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Not yet counting anything the target does. + /// + /// The most rows a source may return in one batch for this category. + /// The target picks it because its own database sets the limit, and a bigger batch fails the write. + /// + Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default); + + /// + /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. + /// Add this batch's own outcome with and save that, so the stored counts are never a guess. + /// + /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Nothing the target does is counted yet. Task Write( MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default); - /// How many rows the target holds for one category, for progress and verify. Counts only that category, even where two categories share a table. + /// + /// How many rows the target holds for one category, for progress and verify. + /// Counts only that category, even where two categories share a table. + /// Task Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// + /// The categories this target can write. A category outside this set is never handed to the engine. + /// Answers before Open and does not change across it. + /// + IReadOnlyCollection SupportedCategoryIds { get; } + + /// + /// The type of this target takes, for each category in + /// . A row holding any other type fails its batch with an + /// . Answers before Open and does not change across it. + /// + IReadOnlyDictionary DocumentTypes { get; } } -/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. -/// How many of Skipped the target would have deleted anyway, such as a row already past retention. Counted and reported like any skip, but never counted toward the halt threshold. -public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null, int BenignSkipped = 0); +/// +/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +/// +/// The checkpoint as the target stored it, carrying the version that save landed on. The engine carries on from this one, never from the one it passed in. +/// Rows this batch wrote. The same number must show up as the rise in the saved copied count, because the halt threshold reads one and status and verify read the other. +/// Rows this batch could not write, benign ones included. +/// The source ids of those rows, so the engine can name each one in the log. +/// Rows the target already held. They were neither copied nor skipped, and they still count as accounted for. +/// How many rows each reason skipped. The counts have to add up to Skipped, or the checkpoint refuses the batch. +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs new file mode 100644 index 0000000000..8cde500f8f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// +/// The target's own startup checks, plus a marker kept in the target database saying a host has already started on it. +/// +public interface IMigrationTargetReadiness +{ + /// + /// The checks this target wants run before the copy starts, in the order they must run. + /// + IReadOnlyList ContributedChecks(); + + /// + /// Stamps the marker the first time a host opens on the target, and leaves that first stamp alone on every start after it. + /// + Task RecordHostOpened(CancellationToken cancellationToken = default); + + /// + /// Whether a host has already opened on the target, which is what makes discarding a partial copy a loss rather than a clean abort. + /// + Task HasHostOpened(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs index 431730ba82..7a36324b40 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs @@ -3,9 +3,17 @@ namespace ServiceControl.Persistence.DataMigration; using System; using System.Collections.Generic; +/// +/// When a category is copied. Required categories are copied with ServiceControl closed, because the host must +/// not serve a half copied instance. Optional ones are copied in the background once it is open. +/// public enum MigrationCategoryKind { Required, Optional } /// One kind of data, such as endpoint settings, copied as a unit and resumed from its own cursor. +/// The name in , which is also the key of its checkpoint row. +/// Whether its rows have message bodies, which the engine fetches separately when the source does not attach them. +/// The order within one kind, counting from 1. Required and optional both start at 1, so the two lists are never sorted together. +/// The category that has to finish first, or null when nothing has to. A category whose predecessor is unfinished is recorded as instead of running. public sealed record MigrationCategory( string Id, MigrationCategoryKind Kind, @@ -16,7 +24,11 @@ public sealed record MigrationCategory( /// A message body read from the source, as bytes plus its content type. public sealed record MigrationBody(ReadOnlyMemory Content, string ContentType); -/// One item read from the source. Body is null unless the source attached it or the engine fetched it. +/// One item read from the source. +/// The row's identifier in the old database. It is what the engine logs when the row is skipped, and what takes. +/// The row itself, as the object the source read. The target's writer for that category is what knows the type. +/// Facts about the row that the document does not carry. Empty when the source has none to add. +/// The message body, or null when the source did not attach one and the engine has not fetched it. public sealed record MigrationRow( string SourceId, object Document, @@ -24,4 +36,6 @@ public sealed record MigrationRow( MigrationBody? Body = null); /// Rows read from the source together, plus the cursor to resume after them. +/// The rows, which can be empty when every document in this stretch turned into no row. The cursor past them still has to be saved. +/// Where the source got to. A later read handed this back carries on after it. public sealed record MigrationBatch(IReadOnlyList Rows, string Cursor); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs index 6f15bcdf24..fadc585355 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -1,5 +1,9 @@ namespace ServiceControl.Persistence.DataMigration; +/// +/// The name of every category a migration can copy. The name is what the checkpoint row is keyed on, so renaming +/// one strands the progress already saved under the old name. says when each one is copied. +/// public static class MigrationCategoryIds { public const string KnownEndpoints = nameof(KnownEndpoints); @@ -18,7 +22,6 @@ public static class MigrationCategoryIds public const string EventLog = nameof(EventLog); public const string CustomChecks = nameof(CustomChecks); public const string FailedErrorImports = nameof(FailedErrorImports); - public const string FailedMessageEdits = nameof(FailedMessageEdits); public const string ArchivedAndResolvedFailedMessages = nameof(ArchivedAndResolvedFailedMessages); public const string GroupComments = nameof(GroupComments); } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs index 69665e34e1..13e8a0fbca 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -4,9 +4,23 @@ namespace ServiceControl.Persistence.DataMigration; using System.Linq; using ServiceControl.MessageFailures; +/// +/// Every category a migration can copy, and when each one is copied. This list is the whole of what a migration +/// covers: a source or a target that cannot handle a category says so through its own supported ids, and the +/// engine never invents one. +/// public static class MigrationCategoryRegistry { + /// + /// The failed message statuses that are copied before the host opens. A message in one of these is waiting on + /// somebody, so an operator who cannot see it after the cutover has lost work. + /// public static readonly IReadOnlyList UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + + /// + /// The failed message statuses that are copied in the background. These are history, so the instance is usable + /// while they are still arriving. + /// public static readonly IReadOnlyList ArchivedAndResolvedStatuses = [FailedMessageStatus.Archived, FailedMessageStatus.Resolved]; public static readonly IReadOnlyList All = @@ -24,14 +38,13 @@ public static class MigrationCategoryRegistry new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 14), + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Required, CarriesBodies: true, Order: 15), + new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Required, CarriesBodies: false, Order: 16, MustFollow: MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), // Optional, copied in the background once the host is open. new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), - new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 2), - new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 3), - new(MigrationCategoryIds.FailedMessageEdits, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 4), - new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 5), - new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 6, MustFollow: MigrationCategoryIds.ArchivedAndResolvedFailedMessages), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 2), ]; public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs new file mode 100644 index 0000000000..0a6ae5bb2c --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class MigrationCategoryStateExtensions +{ + /// + /// Whether the category is done with, so a restart passes over it and it no longer holds the host back: Done, + /// or Abandoned by the operator. A Failed category () is not finished. It waits for + /// --migration-retry or --migration-abandon, and a category that must follow it stays blocked until then. + /// + public static bool IsFinished(this MigrationCategoryState state) => + state is MigrationCategoryState.Complete + or MigrationCategoryState.Abandoned; + + /// + /// Whether the category is Failed: it stopped early, or reached the end having skipped rows to faults. It waits + /// for the operator to run --migration-retry or --migration-abandon, and no start copies it until then. + /// + public static bool IsFailed(this MigrationCategoryState state) => + state is MigrationCategoryState.Halted + or MigrationCategoryState.CompleteWithErrors; +} + +public static class MigrationSkipReasonExtensions +{ + /// + /// Whether the running product would have dropped this row anyway, which makes the skip no loss. The halt + /// threshold and the settle rule never count a harmless skip, so no number of them can stop a category or + /// leave it Failed. The list is fixed. is not on it, because a + /// newer build's reason read back here may be a real loss. + /// + public static bool IsBenign(this MigrationSkipReason reason) => + reason is MigrationSkipReason.PastRetention + or MigrationSkipReason.BlankGroupComment; + + /// + /// Whether no retry can fix a row skipped for this reason, because the row itself can never be stored. + /// + public static bool IsPermanent(this MigrationSkipReason reason) => reason is MigrationSkipReason.RequiredValueMissing; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index 3503321796..bf975e1e21 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -7,6 +7,15 @@ namespace ServiceControl.Persistence.DataMigration; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +/// +/// Copies one category at a time from the source to the target, and records where it got to after every batch. +/// The engine knows nothing about either database: what a row is, how it is written and how big a batch can be +/// all come from the source and the target. A category that fails comes back as a Failed checkpoint rather than +/// an exception, and no later run copies it until the operator retries or abandons it. An optional category an +/// exception stopped comes back still copying instead, with the error on its row. A shutdown and a checkpoint +/// conflict come out as exceptions, because neither is the category's fault, and so does a failure to save the +/// checkpoint a category stopped or settled on, because then there is no row to record it on. +/// public sealed class MigrationEngine( IMigrationSource source, IMigrationTarget target, @@ -15,8 +24,13 @@ public sealed class MigrationEngine( MigrationEngineOptions options, ILogger logger) { + /// How many times a message body is read before the row is skipped as unreadable. public const int MaxBodyReadAttempts = 3; + /// + /// The categories of one kind to copy, in the order to copy them. Optional categories the operator did not + /// ask for are left out. + /// public IReadOnlyList SelectCategories(MigrationCategoryKind kind) => MigrationCategoryRegistry.All .Where(c => c.Kind == kind) @@ -24,12 +38,29 @@ public IReadOnlyList SelectCategories(MigrationCategoryKind k .OrderBy(c => c.Order) .ToArray(); + /// + /// Copies the categories one after another and returns where each one ended, in the same order. A category + /// that halts does not stop the ones after it. Before copying anything it saves a not-started checkpoint for + /// every category that has none, so a copy stopped between two categories still lists the ones it never reached. + /// + /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. + /// The host is shutting down. + /// Saving the checkpoint a category stopped or settled on failed. Its stored row is the last one saved, and the categories after it were not run. // Runs in the order given without re-sorting: required and optional orders both start at 1, so // sorting a mixed list would put an optional category in front of a required one. public async Task> RunCategories( IReadOnlyList categories, CancellationToken cancellationToken = default) { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(NotStarted(category), cancellationToken); + } + } + var results = new List(categories.Count); foreach (var category in categories) @@ -40,12 +71,19 @@ public async Task> RunCategories( return results; } + /// + /// Copies one category, carrying on from its saved cursor, and returns the checkpoint it ended on. A category + /// already finished or Failed is returned untouched without reading the source. + /// + /// The stored checkpoint, whose state says how it ended and whose LastError says why it stopped. + /// Another writer saved this category's checkpoint, which means a second instance is copying into the same database. The state is left as that writer set it. + /// The host is shutting down. The last committed batch saved its own counts, so the stored checkpoint is already correct and a restart carries on from it. + /// Saving the checkpoint the category stopped or settled on failed, such as a halt the store could not write. The stored row is the last one saved, so it still reads as copying from its last committed batch. public async Task RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) { var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); - // Halted is deliberately not one of them: a halt says "stopped, and here is why", and a restart after the cause is fixed has to be able to pick it up again. - if (checkpoint.State is MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned) + if (checkpoint.State.IsFinished() || checkpoint.State.IsFailed()) { return checkpoint; } @@ -53,7 +91,7 @@ public async Task RunCategoryAsync(MigrationCategory catego if (category.MustFollow is { } mustFollowId) { var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); - if (predecessor is not { State: MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned }) + if (predecessor?.State.IsFinished() != true) { var predecessorState = predecessor?.State.ToString() ?? "not started"; var blocked = checkpoint with @@ -61,32 +99,40 @@ public async Task RunCategoryAsync(MigrationCategory catego State = MigrationCategoryState.Blocked, LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" }; + logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", category.Id, mustFollowId, predecessorState); + return await checkpointStore.Upsert(blocked, cancellationToken); } } - if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Halted or MigrationCategoryState.Blocked) + // A LastError on a row still copying is cleared here, because this start is trying it again. + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Blocked || checkpoint.LastError is not null) { checkpoint = await checkpointStore.Upsert(checkpoint with { State = MigrationCategoryState.InProgress, StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, SettledAt = null, LastError = null }, cancellationToken); } var isFirstBatch = true; - // Per run, not the persisted totals: the skips that tripped a halt stay on the row, so - // counting them again would re-halt a restart whose cause has been fixed. + // Per run, not the persisted totals: the skips an earlier run made stay on the row, so counting them + // again would stop a run resumed from its cursor on its first batch. var skippedThisRun = 0L; var processedThisRun = 0L; + var rowsReadThisRun = 0L; + // Saved after the try rather than inside it, so a failed save is not caught below as the category's own error. + MigrationCheckpoint? halt = null; try { - var batchSize = target.BatchSizeFor(category); + var batchSize = await target.BatchSizeFor(category, cancellationToken); + await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) { if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) @@ -94,10 +140,11 @@ public async Task RunCategoryAsync(MigrationCategory catego await Pause(options.ThrottlePause, cancellationToken); } isFirstBatch = false; + rowsReadThisRun += batch.Rows.Count; var batchToWrite = batch; - // Stays off checkpoint until the write commits: the catch persists checkpoint, and a restart - // re-reads an uncommitted batch and would count these skips again. + // Kept out of checkpoint until the write commits: the catch below saves checkpoint, and a + // restart re-reads the uncommitted batch and would count these skips twice. var bodySkips = 0; if (category.CarriesBodies) { @@ -111,43 +158,52 @@ public async Task RunCategoryAsync(MigrationCategory catego } } - // Prior totals, the new cursor, and the rows this engine already skipped. The target adds its own - // outcome inside the transaction that writes the rows, so nothing provisional is ever stored. + // The target adds its own outcome inside the transaction that writes the rows, + // so nothing provisional is ever stored. var checkpointToExtend = checkpoint with { Cursor = batch.Cursor, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, SkippedCount = checkpoint.SkippedCount + bodySkips, SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [MigrationSkipReason.BodyUnreadable] = bodySkips }) }; var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); + + // The target has committed this row, so every halt below settles from it. A halt settling from the + // older version would be refused by the store as a conflict and lose its reason. checkpoint = result.Saved; - foreach (var id in result.SkippedIds) + // The result states the batch's outcome twice, as its own counts and as deltas on the checkpoint + // it committed. The halt threshold reads the first and status and verify read the second. + var committed = (result.Saved.CopiedCount - checkpointToExtend.CopiedCount, result.Saved.SkippedCount - checkpointToExtend.SkippedCount, result.Saved.AlreadyPresentCount - checkpointToExtend.AlreadyPresentCount); + if (committed != (result.Copied, result.Skipped, result.AlreadyPresent)) { - logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); + var mismatch = $"The target reported copying {result.Copied}, skipping {result.Skipped} and finding {result.AlreadyPresent} already present in category {category.Id}, but the checkpoint it committed moved by {committed}."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, mismatch); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = mismatch }; + break; } - // A negative fault count would silently disarm the halt threshold for the rest of the run. - if (result.BenignSkipped > result.Skipped) + foreach (var id in result.SkippedIds) { - throw new InvalidOperationException($"The target reported {result.BenignSkipped} benign skips in category {category.Id} out of {result.Skipped} skipped rows. Benign skips are a subset of the skipped rows."); + logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); } processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; - // Rows the target would have deleted anyway are not faults, so they never halt a category. - skippedThisRun += bodySkips + result.Skipped - result.BenignSkipped; + skippedThisRun += bodySkips + FaultSkips(result.SkipReasons); if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) { - var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows. Fix the cause and restart to resume from the cursor, or abandon the category to accept the loss."; + var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows."; logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); - return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }; + break; } } } - // A shutdown is not a halt, and there is nothing to reconcile: the last committed batch stored its - // real split with its own rows, so the row on disk is already correct and resumable. + // A shutdown is not a halt: the last committed batch saved its counts with its own rows, + // so the checkpoint on disk is already correct and resumable. catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; @@ -161,13 +217,38 @@ public async Task RunCategoryAsync(MigrationCategory catego { var position = checkpoint.Cursor is null ? "at the start" : $"at cursor {checkpoint.Cursor}"; var reason = $"{ex.GetType().Name} {position}: {ex.Message}"; + + // An optional category copies while ServiceControl serves, so an error there waits for the next start rather than for the operator. + if (category.Kind == MigrationCategoryKind.Optional) + { + logger.LogError(ex, "Optional category {CategoryId} stopped at cursor {Cursor} and stays copying until the next start", category.Id, checkpoint.Cursor); + return await checkpointStore.Upsert(checkpoint with { LastError = $"{reason.TrimEnd('.')}. It stays copying and the next start resumes it from the cursor." }, cancellationToken); + } + logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); } - return await Settle(checkpoint with { State = checkpoint.SkippedCount > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); + if (halt is not null) + { + return await Settle(halt, cancellationToken); + } + + // The one check a category runs on itself, judged over this run because the counts on a resumed row cover earlier runs. + if (rowsReadThisRun != processedThisRun) + { + var reason = $"Halted: {category.Id} read {rowsReadThisRun} rows in this run but the target accounted for {processedThisRun} of them as copied, skipped or already present."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + return await Settle(checkpoint with { State = FaultSkips(checkpoint.SkipReasons) > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); } + // Harmless skips are rows the product would have removed anyway, so they neither stop a category nor leave it Failed. + static long FaultSkips(IReadOnlyDictionary? skipReasons) => + skipReasons is null ? 0 : skipReasons.Where(reason => !reason.Key.IsBenign()).Sum(reason => reason.Value); + // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); @@ -230,8 +311,8 @@ Task Settle(MigrationCheckpoint settled, CancellationToken static bool IsDefect(Exception exception) => exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; - // A configured pause of zero means "do not throttle", and a timer that is never going to be - // waited on is worse than no timer: against a fake clock nobody advances, it never completes. + // Zero means no throttling, and it must return without waiting: against a fake clock + // nobody advances, even a zero-length wait never finishes. Task Pause(TimeSpan duration, CancellationToken cancellationToken) => duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs index 8433836893..8dee699d9f 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -2,10 +2,16 @@ namespace ServiceControl.Persistence.DataMigration; using System; using System.Collections.Generic; -using System.Linq; using ServiceControl.Configuration; -/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories to copy. +/// +/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories +/// to copy and how far back. +/// +/// How long to wait between batches of an optional category, which is what keeps the background copy off the instance's back. Zero waits not at all. +/// The share of skipped rows, as a percentage, that stops a category. +/// The number of skipped rows that has to be passed before the percentage counts, so a handful of bad rows in a small category is not a halt. +/// The optional categories to copy. An optional category outside this list is never copied. public sealed record MigrationEngineOptions( TimeSpan ThrottlePause, int HaltThresholdPercent, @@ -17,28 +23,68 @@ public sealed record MigrationEngineOptions( /// How long to wait before trying again to read a message body that failed. Not read from settings. public TimeSpan BodyRetryBackoff { get; init; } = DefaultBodyRetryBackoff; - /// Reads the options from settings. Throws if the optional categories setting names one that doesn't exist or isn't optional. - public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace) + /// + /// How far back the event log copy goes, measured against when each event was raised. + /// + public TimeSpan EventLogWindow { get; init; } + + /// + /// How far back the archived and resolved failed messages copy goes, measured against when each message was + /// archived or resolved. + /// + public TimeSpan ArchivedAndResolvedFailedMessagesWindow { get; init; } + + /// + /// Reads the options from the instance's settings. Anything not set takes the default from + /// , except the two windows, which default to the retention periods passed in. + /// An optional category is selected when its window is greater than zero. + /// + /// The settings root the migration keys are read under. + /// The instance's event retention period, used as the event log window when none is set. + /// The instance's error retention period, used as the archived and resolved window when none is set. + /// A window setting is not a time span of zero or more, which the operator has to see before the copy starts. + public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace, TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) { var throttleMilliseconds = SettingsReader.Read(settingsRootNamespace, MigrationSettings.ThrottlePauseMillisecondsKey, MigrationSettings.DefaultThrottlePauseMilliseconds); var haltPercent = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdPercentKey, MigrationSettings.DefaultHaltThresholdPercent); var haltMinimum = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdMinimumKey, MigrationSettings.DefaultHaltThresholdMinimum); - var optionalCategories = SettingsReader.Read(settingsRootNamespace, MigrationSettings.OptionalCategoriesKey, string.Empty); + var eventLogWindow = ReadWindow(settingsRootNamespace, MigrationSettings.EventLogWindowKey, eventRetentionPeriod); + var archivedAndResolvedWindow = ReadWindow(settingsRootNamespace, MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey, errorRetentionPeriod); + + var selectedIds = new List(); + + if (eventLogWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.EventLog); + } - var selectedIds = optionalCategories - .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToArray(); + if (archivedAndResolvedWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + } - var unknown = selectedIds - .Where(id => MigrationCategoryRegistry.Find(id) is not { Kind: MigrationCategoryKind.Optional }) - .ToArray(); + return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds) + { + EventLogWindow = eventLogWindow, + ArchivedAndResolvedFailedMessagesWindow = archivedAndResolvedWindow + }; + } + + static TimeSpan ReadWindow(SettingsRootNamespace settingsRootNamespace, string key, TimeSpan defaultWindow) + { + var value = SettingsReader.Read(settingsRootNamespace, key); + + if (value is null) + { + return defaultWindow; + } - if (unknown.Length > 0) + if (!TimeSpan.TryParse(value, out var window) || window < TimeSpan.Zero) { throw new InvalidOperationException( - $"{MigrationSettings.OptionalCategoriesKey} names categories that do not exist or are not optional: {string.Join(", ", unknown)}"); + $"{key} is '{value}', which is not a time span of zero or more. Set it like a retention period, such as 7.00:00:00 for seven days, or to 0 to leave that category behind."); } - return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds); + return window; } } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs index 0de8ddb892..6102b22058 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -1,15 +1,31 @@ namespace ServiceControl.Persistence.DataMigration; -/// The migration's setting names, relative to the instance's settings root, and their defaults. explains the tuning ones. +/// +/// The migration's setting names, relative to the instance's settings root, and their defaults. +/// explains the tuning ones. +/// public static class MigrationSettings { public const string ThrottlePauseMillisecondsKey = "Migration/ThrottlePauseMilliseconds"; public const string HaltThresholdPercentKey = "Migration/HaltThresholdPercent"; public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; - /// A comma-separated list of the optional categories to copy, such as "EventLog, CustomChecks". - public const string OptionalCategoriesKey = "Migration/OptionalCategories"; + /// + /// How far back the event log copy goes, as a time span. Unset means the event retention period, and zero + /// turns the category off. + /// + public const string EventLogWindowKey = "Migration/EventLogWindow"; + /// + /// How far back the archived and resolved failed messages copy goes, as a time span. Unset means the error + /// retention period, and zero turns the category off. + /// + public const string ArchivedAndResolvedFailedMessagesWindowKey = "Migration/ArchivedAndResolvedFailedMessagesWindow"; + /// + /// Turns the migration on: the required copy runs before the host opens. + /// + public const string EnabledKey = "Migration/Enabled"; public const int DefaultThrottlePauseMilliseconds = 100; public const int DefaultHaltThresholdPercent = 5; public const int DefaultHaltThresholdMinimum = 100; + public const bool DefaultEnabled = false; } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs index 4129eb939d..11d03ffa52 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -1,9 +1,30 @@ namespace ServiceControl.Persistence.DataMigration; +/// +/// Why one row was not copied. Every skipped row carries one, so the verify command can account for the +/// difference between the two databases. says which of +/// these are no loss. +/// public enum MigrationSkipReason { + /// The message body could not be read from the source, after the engine had tried more than once. BodyUnreadable, + + /// The row is older than the retention period, so the instance would have deleted it soon anyway. PastRetention, - // Never written by a copier. A database a newer build wrote still reads rather than throwing where the host decides whether to start. + + /// The source row has no value for something the target column requires. + RequiredValueMissing, + + /// + /// The group comment is null or blank, and the product keeps no row for a blank comment. + /// + BlankGroupComment, + + /// + /// A reason a newer build wrote and this one cannot name, so an older host still starts instead of throwing. + /// Nothing in this build writes it. It counts as a fault, because harmless would let this build settle a + /// category Done over rows the newer build lost; a false Failed from it is cleared by one retry. + /// Unknown } diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index 08a73eb1c3..4d87e98f76 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -8,25 +8,44 @@ namespace ServiceControl.Persistence /// public abstract class PersistenceSettings { + /// + /// Whether this host starts the database and nothing else, so an operator can repair or inspect it. + /// Only the RavenDB persister supports it. There the embedded server and RavenDB Studio start as + /// usual, and the host registers nothing that ingests or serves data. On any other persister the + /// host refuses to start in maintenance mode. + /// public bool MaintenanceMode { get; set; } - //HINT: This needs to be here so that ServerControl instance can add an instance specific metadata to tweak the DatabasePath value + + /// + /// The directory that holds the database files, or null when this host holds none. The RavenDB + /// persister fills it in from its DbPath setting, and its disk space checks measure the drive that + /// this path names. The SQL persisters leave it null, because their files live on the database server. + /// public string? DatabasePath { get; set; } /// - /// Whether this host owns the background deletion of data past its retention period. Only one - /// host in a deployment should, so error ingestion only hosts turn it off. + /// Whether this host deletes data that is past its retention period. Only one host in a deployment + /// must do this, so a host that only ingests errors turns it off. The SQL persisters start a + /// background sweeper when it is true. RavenDB expires documents on its own and does not read this. /// public bool RunRetentionSweep { get; set; } = true; + /// + /// Whether message bodies are indexed for search. The RavenDB persister decides this as it ingests a + /// message, so a change only reaches messages ingested after it. The SQL persisters always index the + /// body, so the value does not change what they do. + /// public bool EnableFullTextSearchOnBodies { get; set; } = true; /// - /// Wall-clock limit for the message view queries, see . + /// The wall clock limit for a message query, see . It is also how long + /// this instance waits for a remote instance to answer. /// public TimeSpan QueryTimeout { get; set; } = QueryTimeLimit.Default; /// - /// The setting is read from, as named in the timeout error. + /// The name of the setting that comes from. The timeout error names it, so + /// an operator can see which setting to change. /// public const string QueryTimeoutSettingName = "ServiceControl/" + QueryTimeLimit.SettingName; } diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 14a86520d3..d1b03f3fa2 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -113,6 +113,10 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi return embeddedDatabase; } + // Start only queues the server up, so connecting by the configured URL instead reaches whatever already holds the port. + public async Task WaitUntilReady(CancellationToken cancellationToken = default) => + (await EmbeddedServer.Instance.GetServerUriAsync(cancellationToken)).ToString(); + void Start(ServerOptions serverOptions) { this.serverOptions = serverOptions; diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs index f9ff055506..03183720a3 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs @@ -10,8 +10,8 @@ public class EmbeddedDatabaseConfiguration(string serverUrl, string dbName, stri public bool RunInMemory { get; set; } - // Makes RavenDB throw on a query that would create an auto index, instead of creating it. Tests use it to fail - // on dynamic queries, which must go through a static index. + // Makes RavenDB throw on a query that would create an auto index, instead of creating it. Tests use it to catch + // dynamic queries, and the migration source uses it because creating an index would be a write. public bool DisableAutoIndexes { get; set; } } } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index b7303ff3a8..9cabb918e0 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,6 +63,7 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", + "MigrationEnabled": false, "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt index b77ea84b2b..1e54a214bc 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -10,9 +10,8 @@ Required 9: LicensingThroughput, after LicensingEndpoints Required 10: LicensingReportMasks Required 11: LicensedEndpointDetails Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Required 14: CustomChecks +Required 15: FailedErrorImports, with bodies +Required 16: GroupComments, after UnresolvedAndRetryIssuedFailedMessages Optional 1: EventLog -Optional 2: CustomChecks -Optional 3: FailedErrorImports, with bodies -Optional 4: FailedMessageEdits -Optional 5: ArchivedAndResolvedFailedMessages, with bodies -Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages \ No newline at end of file +Optional 2: ArchivedAndResolvedFailedMessages, with bodies \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt index 6c96bee11f..d0b5240bad 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -1,6 +1,4 @@ -Configured: GroupComments, EventLog, ArchivedAndResolvedFailedMessages +Configured: EventLog Runs as: Optional 1: EventLog -Optional 5: ArchivedAndResolvedFailedMessages, with bodies -Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages -Never copied: CustomChecks, FailedErrorImports, FailedMessageEdits \ No newline at end of file +Never copied: ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs new file mode 100644 index 0000000000..7a71eb1c2d --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs @@ -0,0 +1,111 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class CheckpointMigrationStateTests +{ + [Test] + public async Task An_instance_that_has_never_migrated_has_no_store_and_nothing_incomplete() + { + var state = new CheckpointMigrationState(); + + await state.Seed(["EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [Test] + public async Task Selected_categories_that_all_finished_leave_the_guard_off() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint("EndpointSettings", MigrationCategoryState.Complete)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.Blocked)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task One_selected_category_short_of_finished_holds_the_guard_on(MigrationCategoryState unfinished) + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint("EndpointSettings", unfinished)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.True); + } + + [Test] + public async Task A_selected_category_with_no_row_at_all_has_not_started() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.True); + } + + [Test] + public async Task An_unselected_category_that_never_ran_does_not_hold_the_guard_on_forever() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, null, 3, 0, 3, null, null, null, null, null)); + await store.Upsert(new MigrationCheckpoint("EventLog", MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [Test] + public void Exactly_complete_and_abandoned_are_finished() => + Assert.That( + Enum.GetValues().Where(state => state.IsFinished()), + Is.EquivalentTo(new[] { MigrationCategoryState.Complete, MigrationCategoryState.Abandoned })); + + [Test] + public void Exactly_halted_and_complete_with_errors_are_failed() => + Assert.That( + Enum.GetValues().Where(state => state.IsFailed()), + Is.EquivalentTo(new[] { MigrationCategoryState.Halted, MigrationCategoryState.CompleteWithErrors })); + + [Test] + public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsBenign()), + Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), + "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); + + [Test] + public void Unknown_stays_the_last_skip_reason() => + Assert.That(Enum.GetValues().Last(), Is.EqualTo(MigrationSkipReason.Unknown)); + + [Test] + public void Exactly_the_reasons_no_retry_can_fix_are_permanent() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsPermanent()), + Is.EquivalentTo(new[] { MigrationSkipReason.RequiredValueMissing })); + + static MigrationCheckpoint Checkpoint(string categoryId, MigrationCategoryState state) => + new(categoryId, state, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs new file mode 100644 index 0000000000..e381874eac --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs @@ -0,0 +1,232 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// No acceptance test can reach the watchdog: the host copies on the real clock and nobody waits half an hour. +[TestFixture] +class ClosedWindowProgressTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_category_that_commits_nothing_for_the_stall_limit_is_stopped() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that committed nothing for longer than the limit was never stopped"); + await progress.DisposeAsync(); + } + + // A call that ignores its token never returns, so the row stays in progress and every later poll sees the same stall. + [Test] + public async Task A_category_already_stopped_is_reported_once_however_long_it_stays_stuck() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(logger.Entries.Count(entry => entry.Level == LogLevel.Error), Is.EqualTo(1), "the same stall was reported again on the next poll"); + } + + [Test] + public async Task A_resumed_row_carrying_the_previous_runs_stamp_is_not_a_stall() + { + // The row a killed run left behind keeps its last stamp, so an operator restarting hours later + // would otherwise have a healthy copy cancelled on the first tick, every time. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the window runs from when this category's run began, not from a stamp the previous run left"); + } + + [Test] + public async Task A_row_other_than_the_running_category_is_not_judged() + { + // Both were attempted, but only KnownEndpoints is running, and it has no row yet. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.EndpointSettings, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, [MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings]); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the running category was stopped over a row nobody is copying"); + } + + [Test] + public async Task Committing_nothing_for_exactly_the_stall_limit_is_not_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the limit is the point at which a copy has not yet stalled"); + } + + [Test] + public async Task A_category_that_has_committed_nothing_since_its_run_began_is_stopped() + { + // The engine stamps a category when it marks it running, so the window covers the first read, which is + // where a copy that never gets going actually hangs. When the category first started does not matter. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that has committed nothing since its run began was never stopped"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_row_an_older_build_left_without_a_stamp_is_still_watched() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: null, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "an unstamped row was left unwatched for ever"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_category_this_run_finished_is_not_stopped_for_having_gone_quiet() + { + // The watch moves to the next category only when that one starts, so a settled row can still be the one watched. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a category that finished was stopped"); + } + + [Test] + public async Task A_halted_category_this_run_attempted_is_not_stopped_for_having_gone_quiet() + { + // A halt settles the row and leaves its stamp behind. The gate is what refuses the host over it, not the watchdog. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a halted category was stopped"); + } + + // The watch has no total limit, so a copy that keeps committing outlives any length of run. + [Test] + public async Task A_category_still_committing_batches_is_never_stopped_however_long_it_takes() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var committed = await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + // Four times the limit, committing a batch every poll, which a total timeout would have killed long ago. + for (var elapsed = TimeSpan.Zero; elapsed < StallLimit * 4; elapsed += PollInterval) + { + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + committed = await store.Upsert(committed with { LastProgressAt = clock.GetUtcNow().UtcDateTime }); + } + + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a copy committing a batch every poll was stopped, so the watch is a deadline rather than a stall detector"); + } + + [Test] + public async Task A_poll_that_fails_leaves_the_watch_running_and_says_so() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var storeFailure = new InvalidOperationException("the checkpoint store is unreachable"); + store.FailReadAll(1, storeFailure); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a watch that stopped at the first blip never noticed the stall that followed"); + await progress.DisposeAsync(); + Assert.That(logger.Entries.Where(entry => entry.Level == LogLevel.Error).Select(entry => entry.Exception), Has.Member(storeFailure), "a watch that has gone deaf has to say so"); + } + + static async Task StartWatching(PollObservingCheckpointStore store, TimerRecordingTimeProvider clock, string runningCategoryId, CancellationTokenSource stop, string[]? attemptedCategoryIds = null, ILogger? logger = null) + { + var progress = new MigrationStartup.ClosedWindowProgress(store, clock, logger ?? new CapturingLogger(), attemptedCategoryIds ?? [runningCategoryId]); + progress.Watch(runningCategoryId, clock.GetUtcNow().UtcDateTime, stop); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watch never started its timer, so advancing the clock would tick nothing"); + + return progress; + } + + static MigrationCheckpoint InProgress(string categoryId, DateTime? lastProgressAt, DateTime? startedAt = null) => + new(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, startedAt ?? lastProgressAt, lastProgressAt, null, null); + + static MigrationCheckpoint Settled(string categoryId, MigrationCategoryState state, DateTime lastProgressAt) => + new(categoryId, state, null, 0, 0, null, null, lastProgressAt, lastProgressAt, lastProgressAt, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs new file mode 100644 index 0000000000..c8774a9e9a --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class CopyableCategoryIdsTests +{ + [Test] + public void A_category_only_one_side_supports_counts_as_not_copyable() + { + var source = new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }; + var target = new[] { MigrationCategoryIds.KnownEndpoints }; + + Assert.That(MigrationStartup.CopyableCategoryIds(source, target), Is.EquivalentTo(new[] { MigrationCategoryIds.KnownEndpoints }), + "a category with a reader and no writer would otherwise be attempted and fail partway through a customer's copy"); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs index 66341cbf20..ebc2dd8253 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -13,6 +13,7 @@ public sealed class InMemoryMigrationSource : IMigrationSource { readonly Dictionary> rowsByCategory = []; readonly Dictionary bodyFailures = []; + readonly Dictionary bodyFailuresInTurn = []; readonly Dictionary bodyReadAttempts = []; readonly Dictionary bodies = []; @@ -24,13 +25,22 @@ public sealed class InMemoryMigrationSource : IMigrationSource public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); - /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + /// + /// Fails the body reads for this row with one exception per attempt, in order, so a test can tell one attempt's error from another's. An attempt past the end of the list reads the body normally. + /// + public void FailBodyReadsInTurn(string sourceId, params Exception[] failuresInAttemptOrder) => bodyFailuresInTurn[sourceId] = failuresInAttemptOrder; + + /// + /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + /// public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + public IReadOnlyList ContributedChecks() => []; + public Task Describe(CancellationToken cancellationToken = default) => Task.FromResult(Description); public Task> Inventory(CancellationToken cancellationToken = default) => @@ -38,7 +48,7 @@ public Task> Inventory(Cancellation [.. rowsByCategory.Select(pair => new MigrationSourceInventoryEntry("memory", pair.Key, pair.Value.Count))]); public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => - Task.FromResult((long)(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + Task.FromResult(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0); public async IAsyncEnumerable Read( MigrationCategory category, @@ -85,8 +95,19 @@ public async IAsyncEnumerable Read( throw failures.Failure; } + if (bodyFailuresInTurn.TryGetValue(sourceId, out var failuresInTurn) && attempt <= failuresInTurn.Length) + { + throw failuresInTurn[attempt - 1]; + } + return bodies.GetValueOrDefault(sourceId); } + // Every registered category, not just the seeded ones: Seed writes rowsByCategory after construction, + // and the interface promises an answer that does not change across Open. + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index cdfaff9777..84c171e912 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -2,6 +2,7 @@ namespace ServiceControl.UnitTests.Migration.Fakes; using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using ServiceControl.Persistence.DataMigration; @@ -10,30 +11,60 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint { readonly Dictionary> writtenKeysByCategory = []; readonly Dictionary> writtenRowsByCategory = []; + readonly Dictionary> rowsHandedToWriteByCategory = []; readonly HashSet preExistingKeys = []; - readonly Dictionary rejectedKeys = []; + readonly Dictionary rejectedKeys = []; public int DefaultBatchSize { get; set; } = 3; public string NoBatchSizeFor { get; set; } public int? FailOnCallNumber { get; set; } - /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + /// + /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + /// public Exception FailWith { get; set; } - /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + /// + /// How many rows each write leaves out of the outcome it reports and commits, without failing, so the two still agree with each other. + /// + public int UnderReportBy { get; set; } + + /// + /// Runs at the start of every write, before any simulated failure, with the checkpoint the engine is extending, so a test can watch a stamp move between batches. + /// + public Action BeforeWrite { get; set; } + + /// + /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + /// public (int CallNumber, CancellationTokenSource Source)? CancelOnCall { get; set; } + /// + /// Cancels the token on this call and throws instead of writing, so the stop surfaces from the write itself. + /// public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } + /// + /// Waits on this call until the token is cancelled and then throws, as a write stuck on a database that stopped answering does. + /// + public int? HangOnCall { get; set; } int callCount; public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); - public void RejectKey(string sourceId, MigrationSkipReason reason, bool benign = false) => rejectedKeys[sourceId] = (reason, benign); + public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; public IReadOnlyList WrittenRows(string categoryId) => writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; - public int BatchSizeFor(MigrationCategory category) => - category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : DefaultBatchSize; + /// + /// Every row the engine sent to a write that ran, in the order it sent them, including the rows this fake then skipped or found already present. It is the only way to see the same row sent twice, because WrittenRows de-duplicates as the real targets do. Empty for a category never written to. + /// + public IReadOnlyList RowsHandedToWrite(string categoryId) => + rowsHandedToWriteByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : Task.FromResult(DefaultBatchSize); public async Task Write( MigrationCategory category, @@ -42,6 +73,12 @@ public async Task Write( CancellationToken cancellationToken = default) { callCount++; + BeforeWrite?.Invoke(checkpointToExtend); + + if (HangOnCall == callCount) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + } if (StopOnCall is { } stop && stop.CallNumber == callCount) { @@ -62,22 +99,22 @@ public async Task Write( var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; var rows = writtenRowsByCategory.TryGetValue(category.Id, out var existingRows) ? existingRows : writtenRowsByCategory[category.Id] = []; + // Recorded here rather than at the top of the method: a write that threw above never committed, so + // the restart that sends its batch again is doing the right thing. + var handedOver = rowsHandedToWriteByCategory.TryGetValue(category.Id, out var existingHandedOver) ? existingHandedOver : rowsHandedToWriteByCategory[category.Id] = []; + handedOver.AddRange(batch.Rows); + var copied = 0; var alreadyPresent = 0; - var benignSkipped = 0; var skippedIds = new List(); var skipReasons = new Dictionary(); foreach (var row in batch.Rows) { - if (rejectedKeys.TryGetValue(row.SourceId, out var rejection)) + if (rejectedKeys.TryGetValue(row.SourceId, out var reason)) { skippedIds.Add(row.SourceId); - skipReasons[rejection.Reason] = skipReasons.GetValueOrDefault(rejection.Reason) + 1; - if (rejection.Benign) - { - benignSkipped++; - } + skipReasons[reason] = skipReasons.GetValueOrDefault(reason) + 1; continue; } @@ -91,15 +128,21 @@ public async Task Write( copied++; } - // Extended and saved in the same operation as the rows, as the real targets do, so what lands - // is this batch's real split rather than a provisional one the next save has to correct. + copied -= UnderReportBy; + + // The real targets extend and save the checkpoint in the transaction that writes the rows, + // so this fake saves it here too. var saved = await checkpointStore.Upsert( checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), cancellationToken); - return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons, benignSkipped); + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); } diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs new file mode 100644 index 0000000000..147fac3a8f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs @@ -0,0 +1,64 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +// Records when each poll read the rows, so a test can wait for a poll that saw the state it is judging. +// Only the stall watchdog calls ReadAll, so every ReadAll is one of its polls. +public sealed class PollObservingCheckpointStore(TimeProvider clock) : IMigrationCheckpointStore +{ + readonly InMemoryMigrationCheckpointStore inner = new(); + readonly SemaphoreSlim polled = new(0); + readonly ConcurrentQueue polledAt = new(); + Exception? readAllFailure; + int readAllFailuresLeft; + + public void FailReadAll(int times, Exception failure) + { + readAllFailuresLeft = times; + readAllFailure = failure; + } + + /// + /// Waits until a poll has read the rows at the clock's current time, and fails the test if none does within ten + /// seconds. Polls at earlier times are passed over. + /// + public async Task WaitForAPollAtTheCurrentTime() + { + var now = clock.GetUtcNow().UtcDateTime; + + while (true) + { + Assert.That(await polled.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, $"no poll read the checkpoints at {now:O}; a watch that stopped polling never reaches one"); + + if (polledAt.TryDequeue(out var at) && at == now) + { + return; + } + } + } + + public Task> ReadAll(CancellationToken cancellationToken = default) + { + polledAt.Enqueue(clock.GetUtcNow().UtcDateTime); + polled.Release(); + + if (readAllFailuresLeft > 0) + { + readAllFailuresLeft--; + return Task.FromException>(readAllFailure!); + } + + return inner.ReadAll(cancellationToken); + } + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => inner.Upsert(checkpoint, cancellationToken); +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs index fbe2bebe97..ae309d6c8b 100644 --- a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -61,6 +61,16 @@ public void A_hair_past_the_proportion_halts_once_the_floor_is_behind_it() Assert.That(exceeded, Is.True); } + [Test] + public void A_large_category_losing_a_small_share_does_not_halt_once_past_the_floor() + { + // 101 of 5,000,000 is 0.002%. The floor is already behind it, so only the proportion can stop a halt + // here, and a rule that halted on the floor alone would stop a healthy copy of a huge category. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + [Test] public void A_skip_count_with_nothing_processed_never_halts_and_never_divides_by_zero() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs index 9b5e637ec3..c82e610f09 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -12,12 +12,12 @@ namespace ServiceControl.UnitTests.Migration; class MigrationCategoryRegistryTests { [Test] - public void Contains_all_eighteen_categories_with_unique_ids() + public void Contains_all_seventeen_categories_with_unique_ids() { using (Assert.EnterMultipleScope()) { - Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); - Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(17)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(17)); } } @@ -35,7 +35,8 @@ public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() "KnownEndpoints", "EndpointSettings", "MessageRedirects", "Subscriptions", "NotificationSettings", "TrialEndDate", "RetryOperations", "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", - "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages" + "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages", + "CustomChecks", "FailedErrorImports", "GroupComments" })); } @@ -50,8 +51,7 @@ public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() Assert.That(optionalIds, Is.EqualTo(new[] { - "EventLog", "CustomChecks", "FailedErrorImports", - "FailedMessageEdits", "ArchivedAndResolvedFailedMessages", "GroupComments" + "EventLog", "ArchivedAndResolvedFailedMessages" })); } @@ -71,7 +71,7 @@ public void Three_categories_declare_a_MustFollow() ["EndpointSettings"] = "KnownEndpoints", // A group comment whose failed messages have not arrived reads as an orphan to the // retention sweeper, which deletes it. - ["GroupComments"] = "ArchivedAndResolvedFailedMessages" + ["GroupComments"] = "UnresolvedAndRetryIssuedFailedMessages" })); } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs index 3df3d4e123..8b981b0665 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -61,4 +61,23 @@ public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); } + + [Test] + public void MigrationCheckpoint_StartedWindowSeconds_defaults_to_null() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.StartedWindowSeconds, Is.Null); + } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs new file mode 100644 index 0000000000..4d82cac0f8 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Reflection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEnabledSettingsTests +{ + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", null); + } + + [Test] + public void Migration_defaults_to_off() + { + Assert.That(NewSettings().MigrationEnabled, Is.False); + } + + [Test] + public void MigrationEnabled_is_read_from_the_environment() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + + Assert.That(NewSettings().MigrationEnabled, Is.True); + } + + [Test] + public void Only_MigrationEngineOptions_parses_the_engine_settings_keys() + { + var parsers = typeof(Settings).Assembly.GetTypes() + .Concat(typeof(MigrationEngineOptions).Assembly.GetTypes()) + .Where(type => type.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static) + .Any(method => method.Name is "FromSettings" or "Read" or "Resolve" + && method.GetParameters() is [{ ParameterType.Name: nameof(SettingsRootNamespace) }, ..])) + .Select(type => type.Name) + .ToArray(); + + Assert.That(parsers, Is.EquivalentTo(new[] { nameof(MigrationEngineOptions) }), + "Migration/ThrottlePauseMilliseconds and its neighbours have exactly one parser. A second one drifts its defaults and its refusal message away from this one, and nothing fails until a customer types a window wrong."); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs index dc938d6543..dbc94ad95a 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -114,10 +114,11 @@ public async Task The_configured_backoff_is_waited_between_body_read_attempts() // millisecond and the message is skipped for an outage it would have survived. var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; var source = new InMemoryMigrationSource(); - source.Seed(category.Id, Row("msg-1")); - var body = new MigrationBody(new byte[] { 1 }, "text/plain"); - source.SetBody("msg-1", body); - source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("body store unreachable")); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // Every attempt on msg-1 fails, so the guard against a wait after the final one is the only thing + // keeping the third timer from being created. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var clock = new TimerRecordingTimeProvider(); @@ -127,7 +128,8 @@ public async Task The_configured_backoff_is_waited_between_body_read_attempts() var runTask = engine.RunCategoryAsync(category); - // Two failures, so a wait after each before the attempt that succeeds. + // A wait after the first two failures only. A third would never complete, because the clock is + // not advanced again and the run below would time out waiting for it. for (var waitNumber = 1; waitNumber <= MigrationEngine.MaxBodyReadAttempts - 1; waitNumber++) { Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"backoff {waitNumber} never started"); @@ -139,20 +141,24 @@ public async Task The_configured_backoff_is_waited_between_body_read_attempts() using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1), "msg-1 was given up on, so all three attempts ran"); Assert.That(clock.DueTimes, Is.EqualTo(new[] { backoff, backoff }), "no wait after the final attempt, which has nothing left to retry"); - Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); } } [Test] public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() { + // The first failure of a retry run is usually a transient connect error. The last one says what the + // body store was doing when the message was given up on, and it is all the operator gets. var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("msg-1")); - var failure = new TimeoutException("body store unreachable"); - source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, failure); + var failures = Enumerable.Range(1, MigrationEngine.MaxBodyReadAttempts) + .Select(attempt => new TimeoutException($"body store unreachable on attempt {attempt}")) + .ToArray(); + source.FailBodyReadsInTurn("msg-1", failures); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; @@ -161,7 +167,7 @@ public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attem await engine.RunCategoryAsync(category); - Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failure)); + Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failures[^1])); } [Test] diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs index 9c74a7b605..15cf13aaad 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -3,6 +3,7 @@ namespace ServiceControl.UnitTests.Migration; using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using NUnit.Framework; @@ -13,9 +14,9 @@ namespace ServiceControl.UnitTests.Migration; [TestFixture] class MigrationEngineCategorySelectionTests { - static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalIds, out InMemoryMigrationCheckpointStore checkpointStore) + static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalIds) { - checkpointStore = new InMemoryMigrationCheckpointStore(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, selectedOptionalIds); return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -33,7 +34,7 @@ public void Every_category_runs_in_a_fixed_order() .Where(category => category.Kind == MigrationCategoryKind.Optional) .Select(category => category.Id) .ToArray(); - var engine = BuildEngine(everyOptionalId, out _); + var engine = BuildEngine(everyOptionalId); var runOrder = engine.SelectCategories(MigrationCategoryKind.Required) .Concat(engine.SelectCategories(MigrationCategoryKind.Optional)) @@ -45,8 +46,8 @@ public void Every_category_runs_in_a_fixed_order() [Test] public void Only_configured_optional_categories_are_selected() { - string[] configured = [MigrationCategoryIds.GroupComments, MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages]; - var engine = BuildEngine(configured, out _); + string[] configured = [MigrationCategoryIds.EventLog]; + var engine = BuildEngine(configured); var selected = engine.SelectCategories(MigrationCategoryKind.Optional); var left = MigrationCategoryRegistry.All @@ -63,18 +64,27 @@ .. selected.Select(Describe), } [Test] - public void A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() + public async Task A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() { - var engine = BuildEngine([], out var checkpointStore); - var previousRun = new MigrationCheckpoint("EventLog", MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); - checkpointStore.Upsert(previousRun).GetAwaiter().GetResult(); + // Dropping a category from the configuration must not restart, reset or delete what it already + // copied: those rows are in the target, and a row wound back to the start copies every one again. + var stillConfigured = MigrationCategoryIds.EventLog; + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var source = new InMemoryMigrationSource(); + source.Seed(stillConfigured, new MigrationRow("check-1", new object(), new Dictionary())); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, [stillConfigured]); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var previousRun = new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(previousRun); - var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + var results = await engine.RunCategories(engine.SelectCategories(MigrationCategoryKind.Optional)); + var deselected = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); using (Assert.EnterMultipleScope()) { - Assert.That(selected.Select(c => c.Id), Does.Not.Contain("EventLog")); - Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun with { Version = 1 })); + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { stillConfigured }), "the run touched only the configured category"); + Assert.That(deselected, Is.EqualTo(previousRun with { Version = 1 }), "the row is still at the version the seeding save left it"); } } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs index aafb9d4935..6135b96aa5 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -40,6 +40,105 @@ public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skippe } } + [Test] + public async Task A_first_run_leaves_the_source_total_null_and_every_batch_moves_LastProgressAt() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var startedAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var betweenBatches = TimeSpan.FromMinutes(1); + var clock = new FakeTimeProvider(startedAt); + var stamps = new List(); + // The engine stamps the checkpoint before the target sees it, so moving the clock in here is what + // makes a stamp that never moves show up as two identical entries. + target.BeforeWrite = extending => + { + stamps.Add(extending.LastProgressAt); + clock.Advance(betweenBatches); + }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SourceTotal, Is.Null, "counting the source is a full pass over it, which on a big category could trip the stall watchdog before a row is copied"); + Assert.That(stamps, Is.EqualTo(new DateTime?[] { startedAt.UtcDateTime, (startedAt + betweenBatches).UtcDateTime }), "the stall watchdog stops a copy whose LastProgressAt stops moving, so every batch has to move it"); + Assert.That(checkpoint.LastProgressAt, Is.EqualTo((startedAt + betweenBatches).UtcDateTime), "the saved row carries the last batch's stamp"); + } + } + + [Test] + public async Task A_category_that_read_fewer_rows_than_the_source_holds_still_settles_complete() + { + // The row says the source held 10 and the source yields 2. The engine judges only its own accounting, + // so a read that ends early is not something it can see, and only --migration-verify shows it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("d"), Row("e")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, 10, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task An_optional_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + [Test] + public async Task A_required_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + // The target commits a checkpoint and a result that agree with each other, so only the rows read can show the one it lost. + static Task RunWithOneRowUnaccountedFor(MigrationCategory category) + { + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 4, UnderReportBy = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + return engine.RunCategoryAsync(category); + } + [Test] public async Task A_category_with_no_rows_finishes_Complete_without_a_cursor() { @@ -84,8 +183,7 @@ public async Task The_moment_a_category_settles_comes_from_the_injected_clock() [Test] public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_second_run() { - // Finished with a few skips is finished. Re-reading it would copy the whole category again and - // count its skips a second time. + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("a"), Row("b")); @@ -106,6 +204,29 @@ public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_secon } } + [Test] + public async Task A_category_already_Halted_is_left_alone_on_a_second_run() + { + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, "a", 1, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + [Test] public async Task A_category_already_Complete_is_left_alone_on_a_second_run() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs index fb26628421..9d02e7bb4d 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -35,6 +35,7 @@ public async Task A_failing_write_halts_the_category_and_records_the_error_inste using (Assert.EnterMultipleScope()) { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); Assert.That(checkpoint.LastError, Does.Contain("Simulated failure")); // The first batch committed, so the cursor is real and a later run resumes from it. @@ -78,40 +79,43 @@ public void A_cancelled_run_is_a_shutdown_rather_than_a_failure_and_is_not_swall } [Test] - public async Task A_halted_category_is_re_attempted_on_the_next_run_and_resumes_from_its_cursor() + public async Task A_halted_category_is_left_alone_until_the_operator_puts_it_back_in_progress() { - // A halt that no restart can clear would leave abandoning the category as the only way out of - // a fault the customer has already repaired. var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; - var firstRun = BuildEngine(source, checkpointStore, target); - var halted = await firstRun.RunCategoryAsync(category); + var halted = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); - // Stopped on its first write, so the saved row is the restarted one rather than the completed one. + // The cause is fixed, but nothing re-runs a Failed category until the operator says so. target.FailOnCallNumber = null; - using var stopping = new CancellationTokenSource(); - target.StopOnCall = (3, stopping); - Assert.ThrowsAsync(() => BuildEngine(source, checkpointStore, target).RunCategoryAsync(category, stopping.Token)); - var restarted = await checkpointStore.Read(category.Id); + var untouched = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); - target.StopOnCall = null; - var lastRun = BuildEngine(source, checkpointStore, target); - var finished = await lastRun.RunCategoryAsync(category); + // The reset --migration-retry writes: a re-read from the start that keeps only when the category first started. + await checkpointStore.Upsert(untouched with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); using (Assert.EnterMultipleScope()) { - Assert.That(restarted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); - Assert.That(restarted.SettledAt, Is.Null, "a copy running again does not keep the time its halt settled at"); + Assert.That(untouched, Is.EqualTo(halted), "a start re-ran a Failed category the operator had not put back"); Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); - Assert.That(finished.LastError, Is.Null, "a cleared halt does not leave a stale error on the row"); - Assert.That(finished.CopiedCount, Is.EqualTo(4)); - var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); - Assert.That(writtenIds, Is.EquivalentTo(new[] { "a", "b", "c", "d" })); - Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates across the halt"); + Assert.That(finished.LastError, Is.Null, "a retried category does not keep the error it failed with"); + Assert.That(finished.SourceTotal, Is.Null); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(new[] { "a", "b", "c", "d" })); + // Rows a and b were committed before the failure, so the re-read finds them already present. + Assert.That((finished.CopiedCount, finished.AlreadyPresentCount), Is.EqualTo((2L, 2L))); } } @@ -130,6 +134,8 @@ public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_acros var firstRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); var halted = await firstRun.RunCategoryAsync(category); + // The row a crash just before the failing batch commits leaves, which a start resumes from its cursor. + await checkpointStore.Upsert(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null }); target.FailOnCallNumber = null; var secondRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); var finished = await secondRun.RunCategoryAsync(category); @@ -191,7 +197,7 @@ public async Task A_target_with_no_batch_size_for_a_category_halts_it_and_the_ne } [Test] - public void A_halt_whose_save_fails_still_logs_the_exception_that_caused_it() + public void A_write_failure_halt_logs_its_exception_before_it_settles() { var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); @@ -207,7 +213,7 @@ public void A_halt_whose_save_fails_still_logs_the_exception_that_caused_it() } [Test] - public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() + public void A_threshold_halt_logs_its_reason_before_it_settles() { var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); @@ -225,6 +231,78 @@ public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); } + [Test] + public void An_optional_threshold_halt_whose_save_fails_throws_rather_than_staying_copying() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + var stored = checkpointStore.Read(category.Id).GetAwaiter().GetResult(); + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.LastError, Is.Null, "the failed save of a halt was recorded as the category's own error, which leaves a Failed category reading as still copying"); + } + } + + [Test] + public async Task An_optional_category_that_throws_stays_copying_with_the_error_and_its_cursor() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + foreach (var row in new[] { checkpoint, stored! }) + { + Assert.That(row.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("b")); + Assert.That(row.CopiedCount, Is.EqualTo(2)); + Assert.That(row.SettledAt, Is.Null); + Assert.That(row.LastError, Does.Contain("Simulated failure")); + Assert.That(row.LastError, Does.Not.Contain("--migration-abandon").And.Not.Contain("--migration-retry")); + } + } + } + + [Test] + public async Task An_optional_category_an_exception_left_copying_resumes_on_the_next_start_and_clears_its_error() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + target.FailOnCallNumber = null; + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "the error stays on the row only until a start tries again"); + Assert.That(finished.CopiedCount, Is.EqualTo(4)); + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b", "c", "d" }), "the resume did not carry on from the cursor"); + } + } + [Test] public async Task A_checkpoint_conflict_leaves_the_other_writer_alone_instead_of_halting_over_it() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index 47a77f85d7..0ece00a601 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -58,7 +58,7 @@ public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_t var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) { - target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); } var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -67,7 +67,7 @@ public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_t using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((800L, 200L))); Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(800), "the last row was reached, so nothing halted partway"); } @@ -78,7 +78,7 @@ public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_t public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_alone(int everyNthIsAFault, MigrationCategoryState expected) { // A real archive copy loses rows both ways at once: retention takes some, unreadable bodies take - // others. This is the only shape where the subtraction has to do arithmetic rather than pick a side. + // others. This is the only shape where the sum has to leave some reasons out. var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, [.. Enumerable.Range(1, 5_000).Select(i => Row($"row-{i}"))]); @@ -87,7 +87,7 @@ public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_ // A fifth of the category is past retention either way, which on its own is four times the threshold. foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % 5 == 0)) { - target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); } // The offset keeps the faults clear of the benign rows: 4% of the category in one case, 10% in the other. foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % everyNthIsAFault == 3)) @@ -160,30 +160,167 @@ public async Task Rows_already_present_in_the_target_never_count_toward_the_halt } [Test] - public async Task A_restart_after_a_threshold_halt_counts_only_its_own_skips_and_keeps_the_earlier_ones() + public async Task A_resumed_run_counts_only_its_own_skips_and_keeps_the_earlier_ones() { var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; var source = new InMemoryMigrationSource(); source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); var checkpointStore = new InMemoryMigrationCheckpointStore(); - var failingTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + // What a shutdown after the sixth batch leaves: 120 fault skips in 600 rows, past both the floor and 5% if counted again. + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, "row-600", 480, 120, null, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 120 }, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); + Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); + } + } + + [Test] + public async Task A_category_smaller_than_the_floor_that_loses_every_row_ends_Failed_rather_than_Done() + { + // Ninety rows is under the hundred-row floor, so the threshold the engine checks after every batch + // can never fire, however many rows are lost. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.RequiredValueMissing); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.State.IsFinished(), Is.False, "a required category that copied nothing must not let the host open"); + Assert.That(checkpoint.CopiedCount, Is.Zero); + } + } + + [Test] + public async Task A_halted_category_is_returned_untouched_and_its_source_is_not_read() + { + // Failed waits for the operator, so a start that read the category again would copy rows nobody asked for. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, null, 0, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.RowsHandedToWrite(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_small_category_losing_rows_the_product_would_drop_anyway_still_completes() + { + // Harmless skips are rows the target would have deleted anyway, so no number of them may stop a + // category or leave it Failed, however small the category is. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } + + [Test] + public async Task A_run_that_ends_with_one_fault_skip_settles_CompleteWithErrors_however_small_the_share() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + target.RejectKey("row-500", MigrationSkipReason.RequiredValueMissing); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one row in a thousand is far under the threshold, and it is still a row the product wanted"); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((999L, 1L))); + } + } + + [Test] + public async Task A_run_whose_only_skips_are_harmless_settles_Complete_and_keeps_its_reasons() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + target.RejectKey("a", MigrationSkipReason.PastRetention); + target.RejectKey("b", MigrationSkipReason.PastRetention); + target.RejectKey("c", MigrationSkipReason.BlankGroupComment); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((1L, 3L)), "a harmless skip is still a skip, and status and verify count it"); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary + { + [MigrationSkipReason.PastRetention] = 2, + [MigrationSkipReason.BlankGroupComment] = 1 + })); + } + } + + [Test] + public async Task The_threshold_halt_names_its_counts_and_no_command() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) { - failingTarget.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); } var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); - var halted = await new MigrationEngine(source, failingTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); - // The cause is fixed: the remaining rows now write cleanly. - var fixedTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; - var finished = await new MigrationEngine(source, fixedTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); using (Assert.EnterMultipleScope()) { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); // 120 skips in 600 rows is the first point past both the floor and 5%. - Assert.That((halted.State, halted.CopiedCount, halted.SkippedCount), Is.EqualTo((MigrationCategoryState.Halted, 480L, 120L))); - Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); - Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); + Assert.That(checkpoint.LastError, Does.Contain("120 of 600").And.Contain("5%").And.Contain("100 rows")); + // The engine cannot know which commands a category may take, so the readers of the row add them. + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon").And.Not.Contain("restart to resume")); } } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs index 0ede2f09b3..71e62dac7b 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -10,6 +10,8 @@ namespace ServiceControl.UnitTests.Migration; class MigrationEngineOptionsTests { static readonly SettingsRootNamespace Namespace = new("ServiceControl"); + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); [TearDown] public void ClearEnvironmentVariables() @@ -17,20 +19,25 @@ public void ClearEnvironmentVariables() Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", null); Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", null); Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", null); - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); } + static MigrationEngineOptions Read() => MigrationEngineOptions.FromSettings(Namespace, EventRetention, ErrorRetention); + [Test] public void Defaults_match_the_contract_when_nothing_is_configured() { - var options = MigrationEngineOptions.FromSettings(Namespace); + var options = Read(); using (Assert.EnterMultipleScope()) { Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); - Assert.That(options.SelectedOptionalCategoryIds, Is.Empty); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages }), "both optional categories are copied unless turned off"); + Assert.That(options.EventLogWindow, Is.EqualTo(EventRetention), "the event log window defaults to the event retention period"); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention), "the archived and resolved window defaults to the error retention period"); Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); } } @@ -41,38 +48,41 @@ public void Reads_configured_values_from_environment_variables() Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", "2500"); Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", "10"); Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", "50"); - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EventLog, CustomChecks"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "2.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "5.12:00:00"); - var options = MigrationEngineOptions.FromSettings(Namespace); + var options = Read(); using (Assert.EnterMultipleScope()) { Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(2500))); Assert.That(options.HaltThresholdPercent, Is.EqualTo(10)); Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); - Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { "EventLog", "CustomChecks" })); + Assert.That(options.EventLogWindow, Is.EqualTo(TimeSpan.FromDays(2))); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(TimeSpan.FromDays(5.5))); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages })); } } - [Test] - public void Refuses_an_unknown_optional_category_id() + [TestCase("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", MigrationCategoryIds.ArchivedAndResolvedFailedMessages)] + [TestCase("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", MigrationCategoryIds.EventLog)] + public void A_zero_window_turns_its_category_off(string variable, string stillSelected) { - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "NoSuchCategory"); + Environment.SetEnvironmentVariable(variable, "0"); - var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + var options = Read(); - Assert.That(ex.Message, Does.Contain("NoSuchCategory")); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { stillSelected })); } - [Test] - public void Refuses_a_required_category_id_named_as_optional() + [TestCase("a week")] + [TestCase("-1.00:00:00")] + public void Refuses_a_window_that_is_not_a_time_span_of_zero_or_more(string value) { - // EndpointSettings is required, not optional: naming it here is a customer mistake, not - // a valid way to force it. Required categories are never a matter of configuration. - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EndpointSettings"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", value); - var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + var ex = Assert.Throws(() => Read()); - Assert.That(ex.Message, Does.Contain("EndpointSettings")); + Assert.That(ex.Message, Does.Contain(value).And.Contain(MigrationSettings.EventLogWindowKey), "the refusal has to name both the value and the setting holding it for the customer to fix it"); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs index a7f2c99e25..056805e938 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -67,13 +67,14 @@ public async Task EndpointSettings_does_not_start_before_KnownEndpoints_complete [TestCase(MigrationCategoryState.InProgress)] [TestCase(MigrationCategoryState.Halted)] - public async Task GroupComments_does_not_start_before_the_archive_completes(MigrationCategoryState archiveState) + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task GroupComments_does_not_start_before_the_unresolved_failed_messages_complete(MigrationCategoryState predecessorState) { var comments = MigrationCategoryRegistry.Find("GroupComments")!; var source = new InMemoryMigrationSource(); source.Seed(comments.Id, Row("GroupComment/g-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); var target = new InMemoryMigrationTarget(checkpointStore); var engine = BuildEngine(source, checkpointStore, target); @@ -82,7 +83,7 @@ public async Task GroupComments_does_not_start_before_the_archive_completes(Migr using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); - Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); Assert.That(target.WrittenRows(comments.Id), Is.Empty); } } @@ -90,15 +91,14 @@ public async Task GroupComments_does_not_start_before_the_archive_completes(Migr [Test] public async Task A_blocked_category_runs_once_the_category_it_follows_settles() { - // Every real migration starts group comments blocked, so a block nothing can clear would strand - // the last category and leave the migration unable to end. + // A block nothing can clear would strand group comments and keep the host closed for good. var comments = MigrationCategoryRegistry.Find("GroupComments")!; - var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var unresolved = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; var source = new InMemoryMigrationSource(); source.Seed(comments.Id, Row("GroupComment/g-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - var archiveRunning = new MigrationCheckpoint(archive.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); - var saved = await checkpointStore.Upsert(archiveRunning); + var predecessorRunning = new MigrationCheckpoint(unresolved.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + var saved = await checkpointStore.Upsert(predecessorRunning); var target = new InMemoryMigrationTarget(checkpointStore); var blocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); @@ -133,7 +133,7 @@ public async Task A_blocked_category_has_not_settled_because_it_has_not_finished [TestCase(MigrationCategoryState.NotStarted)] [TestCase(MigrationCategoryState.Blocked)] - public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState archiveState) + public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState predecessorState) { // A missing row reads as "not started"; a row that exists says what it actually holds, which is // how an operator tells a category waiting its turn from one waiting on a chain. @@ -141,7 +141,7 @@ public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_s var source = new InMemoryMigrationSource(); source.Seed(comments.Id, Row("GroupComment/g-1")); var checkpointStore = new InMemoryMigrationCheckpointStore(); - await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, null, 0, 0, null, null, null, null, null, null)); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, null, 0, 0, null, null, null, null, null, null)); var target = new InMemoryMigrationTarget(checkpointStore); var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); @@ -149,12 +149,11 @@ public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_s using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); - Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); } } [TestCase(MigrationCategoryState.Complete)] - [TestCase(MigrationCategoryState.CompleteWithErrors)] [TestCase(MigrationCategoryState.Abandoned)] public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finished_or_abandoned(MigrationCategoryState endpointsState) { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs index d21760171f..3a2dca44d7 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -32,8 +32,19 @@ public async Task A_restart_keeps_the_moment_the_category_first_started() var halted = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(firstStart), options, NullLogger.Instance) .RunCategoryAsync(category); - // A day later, the cause is fixed and the host is started again. + // A day later, the cause is fixed and the operator puts the category back as --migration-retry does. target.FailOnCallNumber = null; + await checkpointStore.Upsert(halted with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); var restart = firstStart.AddDays(1); var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(restart), options, NullLogger.Instance) .RunCategoryAsync(category); @@ -42,7 +53,7 @@ public async Task A_restart_keeps_the_moment_the_category_first_started() { Assert.That(halted.StartedAt, Is.EqualTo(firstStart.UtcDateTime)); Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); - Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "the restart carries on a copy that started a day ago"); + Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "a retried category keeps the moment it first started"); Assert.That(finished.SettledAt, Is.EqualTo(restart.UtcDateTime)); } } @@ -82,9 +93,9 @@ public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_an { Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); Assert.That(finalCheckpoint.CopiedCount, Is.EqualTo(6)); - var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); - Assert.That(writtenIds, Is.EquivalentTo(allIds), "no gaps"); - Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates"); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(allIds), "no gaps"); + // The target de-duplicates, as the real ones do, so what it kept can never show a row sent twice. + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.Unique, "no duplicates: the restart resumes past the rows the first run committed"); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs index e4e8393de0..d08b17f548 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -40,6 +40,80 @@ public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them( } } + [Test] + public async Task Every_category_it_is_given_has_a_row_before_the_first_one_copies_anything() + { + // A copy stopped between two categories must not leave a table that reads as finished, because the + // gates that keep a host off an unfinished copy look only at the rows that exist. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + IReadOnlyList? rowsAtFirstWrite = null; + var target = new InMemoryMigrationTarget(checkpointStore) { BeforeWrite = _ => rowsAtFirstWrite ??= checkpointStore.ReadAll().GetAwaiter().GetResult() }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + // Three, so recording only the next category would still leave the last one missing. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!, MigrationCategoryRegistry.Find("MessageRedirects")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + await engine.RunCategories(given); + + Assert.That(rowsAtFirstWrite, Is.Not.Null, "the copy never wrote a batch"); + using (Assert.EnterMultipleScope()) + { + Assert.That(rowsAtFirstWrite!.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "KnownEndpoints", "EndpointSettings", "MessageRedirects" })); + Assert.That(rowsAtFirstWrite!.Where(c => c.CategoryId != "KnownEndpoints").Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.NotStarted)); + } + } + + [Test] + public async Task A_halted_category_is_left_untouched_and_the_next_category_still_runs() + { + var firstStarted = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Halted, "KnownEndpoints-1", 1, 0, null, null, firstStarted, firstStarted, firstStarted, "Halted: earlier run")); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1"), Row("KnownEndpoints-2")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed("MessageRedirects", Row("MessageRedirects-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("MessageRedirects")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty, "a Failed category waits for the operator"); + Assert.That(results[0], Is.EqualTo(halted), "the halted category's row was changed"); + Assert.That(await checkpointStore.Read("KnownEndpoints"), Is.EqualTo(halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows("MessageRedirects").Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_finished_category_is_not_copied_again() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Complete, "KnownEndpoints-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + [Test] public async Task Running_no_categories_copies_nothing_and_reports_nothing() { diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 2e07501603..374a06de68 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -25,10 +25,10 @@ public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkp source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; - // One reason exists, so this pins the total rather than the split between reasons. - target.RejectKey("a", MigrationSkipReason.BodyUnreadable); - target.RejectKey("b", MigrationSkipReason.BodyUnreadable); - target.RejectKey("d", MigrationSkipReason.BodyUnreadable); + // Two reasons over two batches: a and b land in the first, d in the second, so the saved map has to merge both. + target.RejectKey("a", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("b", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("d", MigrationSkipReason.PastRetention); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -37,8 +37,8 @@ public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkp using (Assert.EnterMultipleScope()) { Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); - Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); - Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); } } @@ -71,7 +71,7 @@ public async Task A_batch_that_loses_rows_two_different_ways_records_both_reason source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; - target.RejectKey("msg-2", MigrationSkipReason.PastRetention, benign: true); + target.RejectKey("msg-2", MigrationSkipReason.PastRetention); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); @@ -112,42 +112,82 @@ public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_t } [Test] - public async Task A_target_counting_more_benign_skips_than_skipped_rows_halts_the_category() + public async Task A_target_whose_reported_counts_disagree_with_the_checkpoint_it_committed_never_finishes_the_category() { var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; var source = new InMemoryMigrationSource(); - source.Seed(category.Id, Row("a")); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); var checkpointStore = new InMemoryMigrationCheckpointStore(); - var target = new OverCountedBenignTarget(checkpointStore); + var target = new MiscountedCopyTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); - var checkpoint = await engine.RunCategoryAsync(category); + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); using (Assert.EnterMultipleScope()) { - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); - Assert.That(checkpoint.LastError, Does.Contain("2 benign skips").And.Contain("out of 1 skipped")); + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.LastError, Does.Contain("but the checkpoint it committed moved by"), "the halt has to name which two accounts disagreed, or the operator goes looking for the wrong problem"); + Assert.That(stored.State.IsFinished(), Is.False, "the halt threshold never saw the 150 rows the target dropped, so the category settled finished and the host opened on half a category"); + } + } + + [Test] + public async Task An_optional_category_whose_reported_counts_disagree_with_the_checkpoint_it_committed_settles_halted() + { + // An optional category stays copying after an exception, and this check must not become one of those. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new MiscountedCopyTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored.LastError, Does.Contain("reported copying 10").And.Contain("but the checkpoint it committed moved by")); + Assert.That(stored.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); } } - sealed class OverCountedBenignTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + // Commits half of every batch as skipped and reports the whole batch copied. The saved counts still add up + // to the source total, so nothing later in the run can notice, and the halt threshold sees a clean copy. + sealed class MiscountedCopyTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget { - public int BatchSizeFor(MigrationCategory category) => 10; + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) { - var reasons = new Dictionary { [MigrationSkipReason.PastRetention] = batch.Rows.Count }; - var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, reasons), cancellationToken); - return new MigrationWriteResult(saved, 0, batch.Rows.Count, [], 0, reasons, BenignSkipped: batch.Rows.Count + 1); + var skipped = batch.Rows.Count / 2; + var reasons = new Dictionary { [MigrationSkipReason.RequiredValueMissing] = skipped }; + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(batch.Rows.Count - skipped, skipped, 0, reasons), cancellationToken); + return new MigrationWriteResult(saved, batch.Rows.Count, 0, []); } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); } sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget { - public int BatchSizeFor(MigrationCategory category) => 10; + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) { @@ -156,5 +196,9 @@ public async Task Write(MigrationCategory category, Migrat } public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs new file mode 100644 index 0000000000..638efaaf1b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationIsReleasedCheckTests +{ + [Test] + public void A_build_without_the_whole_migration_refuses_MigrationEnabled() + { + var exception = Assert.ThrowsAsync(() => new MigrationIsReleasedCheck().Run()); + + Assert.That(exception.Message, Does.Contain("does not yet carry the whole migration").And.Contain(MigrationSettings.EnabledKey), + "the customer has to learn that nothing will be copied, and which setting to turn back off"); + } + + // A RavenDB target fails the pair check, so this only passes while the release check runs before it. + [Test] + public void An_unreleased_build_refuses_before_any_other_check() + { + var settings = new Settings(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); + using var services = new ServiceCollection().BuildServiceProvider(); + + var exception = Assert.ThrowsAsync(() => MigrationStartup.RunRequiredCopy(services, settings)); + + Assert.That(exception.Message, Does.Contain("this build carries the whole migration").And.Not.Contain("supported pair")); + } + + [Test] + public void The_test_marker_lets_a_copy_run_on_an_unreleased_build() + { + Assert.DoesNotThrowAsync(() => new MigrationIsReleasedCheck(new AllowUnreleasedMigration()).Run()); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs new file mode 100644 index 0000000000..c25bf44d0c --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; + +[TestFixture] +class MigrationPairIsSupportedCheckTests +{ + [OneTimeSetUp] + public static void TheTreeIsBuilt() => + Assert.That( + PersistenceFactory.SqlPersistenceNames.Where(name => PersistenceManifestLibrary.Find(name) is null), + Is.Empty, + "These persistence manifests did not resolve, so this fixture cannot tell a supported pair from an unsupported one. Build the tree first with: dotnet build src --configuration Release -graph"); + + [TestCase("SQLServer")] + [TestCase("PostgreSQL")] + public void RavenDB_to_a_SQL_persister_is_supported(string target) => + Assert.DoesNotThrowAsync(() => Check(target).Run()); + + [Test] + public void A_RavenDB_target_is_refused_naming_the_target_setting() + { + var exception = Assert.ThrowsAsync(() => Check("RavenDB").Run()); + + Assert.That(exception.Message, Does.Contain("ServiceControl/PersistenceType")); + } + + static MigrationPairIsSupportedCheck Check(string target) => + new(new Settings(transportType: "LearningTransport", persisterType: target, errorRetentionPeriod: TimeSpan.FromDays(10))); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs new file mode 100644 index 0000000000..2767013bb3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs @@ -0,0 +1,75 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationStartupCheckRunnerTests +{ + [Test] + public async Task Checks_run_in_the_order_they_were_given() + { + var ran = new List(); + + await MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the second", _ => ran.Add("second")), + Check("the third", _ => ran.Add("third")) + ]); + + Assert.That(ran, Is.EqualTo(new[] { "first", "second", "third" }), "the databases are opened by checks that sit behind the ones refusing the configuration outright"); + } + + [Test] + public void The_first_check_to_refuse_stops_the_ones_behind_it() + { + var ran = new List(); + var failure = new InvalidOperationException("ServiceControl/RetryHistoryDepth is 0"); + + var exception = Assert.ThrowsAsync(() => MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the migration target is ready", _ => throw failure), + Check("the last", _ => ran.Add("last")) + ])); + + using (Assert.EnterMultipleScope()) + { + Assert.That(ran, Is.EqualTo(new[] { "first" }), "a check behind a refusal would run against a configuration already known to be wrong, and the two that open a database are at the end of the list"); + Assert.That(exception!.Message, Does.Contain("the migration target is ready").And.Contain(failure.Message), "the operator needs to know which check refused and why"); + Assert.That(exception.InnerException, Is.SameAs(failure), "the original carries the stack and the detail the summary leaves out"); + } + } + + [Test] + public void A_host_being_stopped_is_not_a_check_refusing() + { + using var stopping = new CancellationTokenSource(); + stopping.Cancel(); + + var exception = Assert.ThrowsAsync(() => MigrationStartupCheckRunner.Run( + [Check("the first", token => token.ThrowIfCancellationRequested())], stopping.Token)); + + Assert.That(exception!.Message, Does.Not.Contain("Migration startup check"), "a stop dressed up as a refusal sends the operator after a setting that was never wrong"); + } + + static IMigrationStartupCheck Check(string name, Action run) => new FakeCheck(name, run); + + sealed class FakeCheck(string name, Action run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) + { + run(cancellationToken); + return Task.CompletedTask; + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs new file mode 100644 index 0000000000..6f4393ab40 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class OptionalCategoryWindowsAreValidCheckTests +{ + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); + + [TearDown] + public void ClearWindows() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); + } + + [Test] + public async Task Unset_windows_pass_and_default_to_the_retention_periods_it_was_given() + { + var check = new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention); + + await check.Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(check.Options.EventLogWindow, Is.EqualTo(EventRetention)); + Assert.That(check.Options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention)); + } + } + + [Test] + public void A_window_that_is_not_a_time_span_is_refused_and_named() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "a week"); + + var exception = Assert.ThrowsAsync(async () => + await new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention).Run()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain("a week")); + Assert.That(exception.Message, Does.Contain(MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs new file mode 100644 index 0000000000..cc598257ab --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs @@ -0,0 +1,80 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RecordSourceOutageTests +{ + static readonly MigrationCategory EventLog = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + static readonly MigrationCategory ArchivedAndResolvedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + + readonly InMemoryMigrationCheckpointStore checkpointStore = new(); + + [Test] + public async Task A_category_still_copying_keeps_its_state_and_gains_the_error() + { + await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress, "EventLogItem/40", 40, 0, null, null, null, null, null, null)); + + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("EventLogItem/40")); + Assert.That(row.CopiedCount, Is.EqualTo(40)); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [Test] + public async Task A_category_with_no_row_gets_a_not_started_row_carrying_the_error() + { + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + Assert.That(row, Is.Not.Null, "status and verify read only the rows that exist, so a category with no row would show no error"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(row.CopiedCount, Is.Zero); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.AlreadyPresentCount, Is.Zero); + Assert.That(row.StartedAt, Is.Null, "a category that never started must not look as if it had"); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task A_settled_category_is_left_alone(MigrationCategoryState state) + { + var before = await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, state, null, 0, 0, null, null, null, null, null, "earlier")); + + await RecordOutage(); + + var after = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(after!.Version, Is.EqualTo(before.Version), "a settled row is never saved again"); + Assert.That(after.State, Is.EqualTo(state)); + Assert.That(after.LastError, Is.EqualTo("earlier")); + } + } + + Task RecordOutage() => + MigrationStartup.RecordSourceOutage(checkpointStore, [EventLog, ArchivedAndResolvedFailedMessages], new InvalidOperationException("RavenDB is unreachable")); +} diff --git a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs new file mode 100644 index 0000000000..2077493bff --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs @@ -0,0 +1,395 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RequiredCopyGateTests +{ + [TestCase(MigrationCategoryState.Complete)] + public void A_finished_required_category_lets_the_host_open(MigrationCategoryState state) => + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.NotStarted)] + // Blocked happens because EndpointSettings must follow KnownEndpoints. + [TestCase(MigrationCategoryState.Blocked)] + public void A_required_category_that_is_not_finished_keeps_the_host_closed(MigrationCategoryState state) + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings).And.Contain(state.ToString())); + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + public void An_unfinished_row_under_an_id_this_build_does_not_know_keeps_the_host_closed(MigrationCategoryState state) + { + var unknown = Checkpoint(state, UnknownCategoryId); + + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy([unknown, Checkpoint(MigrationCategoryState.Complete)], [EndpointSettings]); + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete), .. outside], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is ").And.Contain(state.ToString()), + "an id this build does not know is required until it is known to be optional, as the ingestion gate already decides"); + } + + [Test] + public void An_unfinished_unknown_row_makes_the_copy_not_settled_and_the_refusal_names_it() + { + var unknown = Checkpoint(MigrationCategoryState.InProgress, UnknownCategoryId); + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), unknown }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.False); + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is InProgress")); + }); + } + + [Test] + public void A_finished_unknown_row_leaves_the_copy_settled_and_the_refusal_silent() + { + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), Checkpoint(MigrationCategoryState.Abandoned, UnknownCategoryId) }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.True); + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + }); + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + public void A_finished_row_under_an_id_this_build_does_not_know_holds_nothing(MigrationCategoryState state) => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(state, UnknownCategoryId)], [EndpointSettings]), Is.Empty); + + [Test] + public void An_unfinished_row_known_to_be_optional_holds_nothing() => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(MigrationCategoryState.InProgress, MigrationCategoryIds.EventLog)], [EndpointSettings]), Is.Empty); + + [Test] + public void A_category_that_reported_no_checkpoint_at_all_keeps_the_host_closed() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings), + "an empty result reads as success unless the gate checks what it asked for against what came back"); + } + + [Test] + public void Of_two_attempted_categories_the_one_that_reported_nothing_is_the_one_named() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [KnownEndpoints, EndpointSettings], + [Checkpoint(MigrationCategoryState.Complete, MigrationCategoryIds.KnownEndpoints)], + NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} reported no checkpoint at all").And.Not.Contain(MigrationCategoryIds.KnownEndpoints), + "comparing counts instead of ids would let a two-category run through with one category's fate unknown"); + } + + [Test] + public void The_refusal_reads_to_its_end_as_a_sentence_and_says_how_to_get_back() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Halted)], NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a Failed category waits for the operator, so a refusal naming neither command leaves them restarting for ever"); + Assert.That(exception.Message, Does.Not.Contain("the copy resumes from its last committed batch"), "a restart no longer copies a Failed category"); + Assert.That(exception.Message, Does.Contain("both with ServiceControl stopped. Nothing has opened on SQLServer yet") + .And.EndWith("returns the instance to RavenDB with no loss."), + "the operator reads the whole message, and the rollback instructions are at the end of it"); + }); + } + + [Test] + public void Every_failed_required_category_is_named_with_its_skips_and_both_commands() + { + var withErrors = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) with + { + CopiedCount = 40, + SkippedCount = 5, + SkipReasons = new Dictionary + { + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 3 + } + }; + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.MessageRedirects) with { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [MessageRedirects, UnresolvedAndRetryIssuedFailedMessages], [halted, withErrors], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in new[] { MigrationCategoryIds.MessageRedirects, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages }) + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {categoryId} once the cause is fixed").And.Contain($"--migration-abandon {categoryId} to keep what was copied"), + $"one start gives every required category its go, so the refusal has to name every Failed one, {categoryId} included"); + } + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.MessageRedirects} is Failed (Halted)").And.Contain("the target did not answer")); + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages} is Failed (CompleteWithErrors) after copying 40 and skipping 5")); + Assert.That(exception.Message, Does.Contain("RequiredValueMissing 2, no retry can fix"), "a row no retry can store is a reason to abandon, and the operator has to be told which"); + Assert.That(exception.Message, Does.Contain("BodyUnreadable 3").And.Not.Contain("BodyUnreadable 3, no retry can fix"), "an unreadable body may read on the next try"); + } + } + + [Test] + public void A_category_waiting_behind_a_failed_one_is_named_as_waiting() + { + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.KnownEndpoints); + var blocked = Checkpoint(MigrationCategoryState.Blocked) with { LastError = "Blocked: EndpointSettings must follow KnownEndpoints, which is Halted" }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([KnownEndpoints, EndpointSettings], [halted, blocked], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Blocked").And.Contain("must follow KnownEndpoints")); + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.KnownEndpoints}")); + Assert.That(exception.Message, Does.Not.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Not.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a category that never got its go has nothing to retry; it runs once the one ahead of it is dealt with"); + } + } + + [Test] + public void A_stalled_category_is_refused_with_the_limit_its_row_records() + { + var stalled = Checkpoint(MigrationCategoryState.Halted) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [stalled], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"committed nothing for {MigrationStartup.ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes") + .And.Contain("looking for a setting to change; run --migration-retry EndpointSettings"), + "the stall is on the row, and the refusal is where the operator reads it"); + } + + // The last thing said before the cutover is one way, and the only place a skipped total appears at all. + [Test] + public void A_failed_category_that_skipped_rows_is_told_a_retry_re_reads_them() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 6, new Dictionary + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 1 + })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning), "rows left behind are not an informational matter"); + Assert.That(entry.Message, Does.Contain("6 skipped"), "the total is the number the operator decides on"); + Assert.That(entry.Message, Does.Contain("PastRetention 3").And.Contain("RequiredValueMissing 2").And.Contain("BodyUnreadable 1"), "a total with no reasons cannot be acted on"); + Assert.That(entry.Message, Does.Contain("stay only in the source database").And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them"), + "without this the operator either waits for a copy that is already over or never learns the rows can still come across"); + Assert.That(entry.Message, Does.Contain("No retry can fix the RequiredValueMissing ones"), "the retry brings the unreadable bodies across but leaves these behind again"); + }); + } + + [Test] + public void A_failed_category_whose_faults_no_retry_can_fix_is_pointed_at_abandon() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 5, new Dictionary + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2 + })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("no retry can fix them").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a retry re-reads the whole category and ends Failed on the same rows"); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "harmless skips are not a reason to retry either"); + }); + } + + [Test] + public void A_category_that_failed_with_only_harmless_skips_is_still_offered_a_retry() + { + var logger = new CapturingLogger(); + + var stoppedByAnException = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 3, new Dictionary { [MigrationSkipReason.PastRetention] = 3 }) + with + { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stoppedByAnException], logger, RunStartedAt); + + Assert.That(logger.Entries.Single().Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them").And.Not.Contain("--migration-abandon"), + "no fault skip stands in the way of a retry, so the exception's cause is what the operator fixes"); + } + + [Test] + public void A_done_category_with_harmless_skips_is_not_offered_a_retry() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 3, new Dictionary { [MigrationSkipReason.PastRetention] = 3 })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("3 skipped").And.Contain("PastRetention 3")); + Assert.That(entry.Message, Does.Contain("harmless").And.Contain("stay only in the source database")); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "a Done category cannot be retried, so offering it sends the operator to a command that refuses"); + }); + } + + [Test] + public void A_blocked_category_is_left_to_the_refusal_rather_than_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Blocked, copied: 0, skipped: 0, null)], logger, RunStartedAt); + + Assert.That(logger.Entries, Is.Empty); + } + + // Every refused start reports it again, directly above the refusal that calls it Failed. + [Test] + public void A_failed_category_that_skipped_nothing_is_not_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning)); + Assert.That(entry.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Failed (Halted) after 10 copied").And.Not.Contain("nothing skipped")); + }); + } + + [Test] + public void A_category_that_skipped_nothing_says_so_without_raising_a_warning() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null)], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information), "a clean copy warning about nothing trains the operator to ignore the warning that matters"); + Assert.That(entry.Message, Does.Contain("nothing skipped")); + }); + } + + // Restarting a migrated instance used to reprint the whole copy summary in the present tense, so an operator + // restarting to change a setting read "7 copied" and had no way to tell the copy had not run again. + [Test] + public void A_category_an_earlier_run_finished_is_not_reported_as_copied_again() + { + var logger = new CapturingLogger(); + + var alreadyDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([alreadyDone], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("already finished before this start"), "without this the line is indistinguishable from a copy that just ran"); + Assert.That(entry.Message, Does.Contain("This start copied nothing"), "the operator needs to know nothing was written to a target that is already serving"); + Assert.That(entry.Message, Does.Contain("7"), "the historical total is still worth stating, just not as this run's work"); + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information)); + }); + } + + // A category this run actually settled must keep the ordinary wording, or the fix above would silence every report. + [Test] + public void A_category_this_run_finished_is_still_reported_as_copied() + { + var logger = new CapturingLogger(); + + var justDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([justDone], logger, RunStartedAt); + + Assert.That(logger.Entries.Single().Message, Does.Contain("7 copied").And.Not.Contain("already finished")); + } + + // A Failed row comes back untouched on every start, so every refused restart reports it again. + [Test] + public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() + { + var logger = new CapturingLogger(); + + var failedEarlier = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 2, new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 }) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([failedEarlier], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Not.Contain("already finished"), "the refusal right after this line calls the same category Failed"); + Assert.That(entry.Message, Does.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}").And.Not.Contain("--migration-retry"), + "the advice has to show on every refused start, not only the first, and no retry can store a row missing a required value"); + }); + } + + static readonly DateTime RunStartedAt = new(2026, 9, 20, 12, 0, 0, DateTimeKind.Utc); + + static MigrationCheckpoint Settled(MigrationCategoryState state, long copied, long skipped, IReadOnlyDictionary? skipReasons) => + new(MigrationCategoryIds.EndpointSettings, state, null, copied, skipped, null, skipReasons, null, null, null, null); + + static readonly MigrationCategory EndpointSettings = MigrationCategoryRegistry.Find(MigrationCategoryIds.EndpointSettings)!; + static readonly MigrationCategory KnownEndpoints = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + static readonly MigrationCategory MessageRedirects = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + static readonly MigrationCategory UnresolvedAndRetryIssuedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + + const string UnknownCategoryId = "SomeCategoryFromANewerBuild"; + + static MigrationCheckpoint Checkpoint(MigrationCategoryState state, string categoryId = MigrationCategoryIds.EndpointSettings) => + new(categoryId, state, null, 0, 0, null, null, null, null, null, null); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); +} diff --git a/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs new file mode 100644 index 0000000000..f3aa068120 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Migration.Checks; + +[TestFixture] +class RetryHistoryDepthIsSafeCheckTests +{ + [Test] + public void Passes_at_the_default_depth() => + Assert.DoesNotThrowAsync(() => new RetryHistoryDepthIsSafeCheck(10).Run()); + + [TestCase(0)] + [TestCase(-1)] + public void Refuses_a_depth_that_empties_the_table(int depth) + { + var exception = Assert.ThrowsAsync(() => new RetryHistoryDepthIsSafeCheck(depth).Run()); + + Assert.That(exception.Message, Does.Contain("RetryHistoryDepth").And.Contain("HistoricRetryOperations")); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs new file mode 100644 index 0000000000..2f3b033642 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs @@ -0,0 +1,194 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// The copy can stop three ways that are not the engine's to report: the stall watchdog stopping one category, +// the host shutting down, and a second instance writing to the same database. +[TestFixture] +class StoppedCopyExplanationTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_stall_settles_the_stalled_category_failed_and_the_next_category_still_runs() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { HangOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.CategoryId), Is.EqualTo(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects })); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted), "a stalled category is Failed, so it waits for the operator rather than for the next start"); + Assert.That(results[0].SettledAt, Is.EqualTo(clock.GetUtcNow().UtcDateTime)); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes").And.Not.Contain("--migration-"), + "the row says what happened, and the refusal is what names the commands"); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0]), "the stall was reported but never saved, so the next start would copy it again"); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "one stalled category ended the whole start"); + Assert.That(target.WrittenRows(MigrationCategoryIds.MessageRedirects).Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_host_stopping_mid_category_leaves_it_copying_and_the_stop_propagates() + { + var store = new InMemoryMigrationCheckpointStore(); + using var host = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(store) { DefaultBatchSize = 2, StopOnCall = (2, host) }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1"), Row("KnownEndpoints-2"), Row("KnownEndpoints-3"), Row("KnownEndpoints-4")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + var clock = new FakeTimeProvider(); + + var run = MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger(), host.Token); + + using (Assert.EnterMultipleScope()) + { + Assert.That(async () => await run, Throws.InstanceOf(), "a shutdown is not the category's fault, so it must not come back as a settled row"); + + var stopped = await store.Read(MigrationCategoryIds.KnownEndpoints); + + Assert.That(stopped!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown settled Halted makes every restart refuse until the operator runs --migration-retry"); + Assert.That(stopped.Cursor, Is.EqualTo("KnownEndpoints-2"), "the next start resumes from the first batch's cursor"); + Assert.That(stopped.CopiedCount, Is.EqualTo(2)); + Assert.That(stopped.SettledAt, Is.Null); + Assert.That(stopped.LastError, Is.Null); + Assert.That((await store.Read(MigrationCategoryIds.MessageRedirects))!.State, Is.EqualTo(MigrationCategoryState.NotStarted), "a stopping host started the next category"); + Assert.That(target.RowsHandedToWrite(MigrationCategoryIds.MessageRedirects), Is.Empty); + } + } + + [Test] + public async Task A_stale_in_progress_row_the_start_has_not_reached_is_not_judged_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var twoHoursAgo = clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2); + await store.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, null, 0, 0, null, null, twoHoursAgo, twoHoursAgo, null, null)); + var target = new InMemoryMigrationTarget(store) + { + DefaultBatchSize = 1, + // Each KnownEndpoints batch takes five minutes, and the next one waits until the watchdog has looked at the time. + BeforeWrite = checkpoint => + { + if (checkpoint.CategoryId == MigrationCategoryIds.KnownEndpoints) + { + clock.Advance(TimeSpan.FromMinutes(5)); + store.WaitForAPollAtTheCurrentTime().GetAwaiter().GetResult(); + } + } + }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, [.. Enumerable.Range(1, 8).Select(index => Row($"KnownEndpoints-{index}"))]); + source.Seed(MigrationCategoryIds.EndpointSettings, Row("EndpointSettings-1")); + + var results = await Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings), store, clock, new CapturingLogger())) + .WaitAsync(TimeSpan.FromSeconds(30)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.State), Is.All.EqualTo(MigrationCategoryState.Complete), + "forty minutes copying KnownEndpoints were judged against a row the start had not reached yet"); + Assert.That(target.WrittenRows(MigrationCategoryIds.KnownEndpoints), Has.Count.EqualTo(8)); + Assert.That(target.WrittenRows(MigrationCategoryIds.EndpointSettings).Select(row => row.SourceId), Is.EqualTo(new[] { "EndpointSettings-1" })); + } + } + + [Test] + public void A_cancelled_copy_stays_a_cancellation() => + Assert.That(async () => await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromCanceled>(new CancellationToken(canceled: true)), NewSettings()), + Throws.InstanceOf(), + "a shutdown is not a failure, so it must not come out as a refusal telling the operator to go looking at the source and the target"); + + [Test] + public void A_checkpoint_saved_by_another_instance_says_which_instance_to_stop() + { + var conflict = new MigrationCheckpointConflictException("Checkpoint KnownEndpoints was saved from version 3, but the stored row is at version 4."); + + var exception = Assert.ThrowsAsync(async () => await MigrationStartup.CopyOrExplainWhyItStopped( + Task.FromException>(conflict), + NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception!.Message, Does.Contain("a second ServiceControl pointed at the same SQLServer database"), + "the store's own message names a version, which tells nobody what is actually wrong"); + Assert.That(exception.Message, Does.Contain($"Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on"), + "this is the only refusal in the subsystem where doing nothing makes it worse"); + Assert.That(exception.Message, Does.Contain(conflict.Message).And.Contain("Nothing has opened on SQLServer yet")); + Assert.That(exception.InnerException, Is.SameAs(conflict)); + }); + } + + [Test] + public async Task A_copy_that_finished_is_returned_as_it_came_back() + { + IReadOnlyList copied = [new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, null, 3, 0, null, null, null, null, null, null)]; + + var finished = await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromResult(copied), NewSettings()); + + Assert.That(finished, Is.SameAs(copied)); + } + + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationCategory[] Categories(params string[] categoryIds) => [.. categoryIds.Select(categoryId => MigrationCategoryRegistry.Find(categoryId)!)]; + + static MigrationEngine Engine(InMemoryMigrationSource source, InMemoryMigrationTarget target, IMigrationCheckpointStore store, TimeProvider clock) => + new(source, target, store, clock, new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); + + // Refuses a cancelled token as the EF Core store does, so a settle made on the stalled category's token fails here too. + sealed class CancellationHonouringCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore inner = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.ReadAll(cancellationToken); + } + + public Task Read(string categoryId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.Read(categoryId, cancellationToken); + } + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.Upsert(checkpoint, cancellationToken); + } + } +} diff --git a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs index 744fa7bf19..627fee7dee 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs @@ -190,12 +190,10 @@ public async Task A_remote_timeout_does_not_cut_the_slower_remotes_short() factory.Register(settings.RemoteInstances[3], Delayed(delay, Healthy("msg-4"))); var api = new TestApi(settings, factory, Local("local-msg")); - var started = DateTime.UtcNow; var result = await api.Execute(Context(), "/api/messages"); - Assert.That(DateTime.UtcNow - started, Is.GreaterThanOrEqualTo(delay), "the composite must wait for the remotes that are still answering"); - Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"])); + Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"]), "the delay lives in the fake, so these three cannot arrive unless the composite waited for the remotes that are still answering"); Assert.That(result.IncompleteInstances, Is.EqualTo([new IncompleteInstance(settings.RemoteInstances[0].InstanceId, QueryFailure.TimedOut)])); Assert.That(result.QueryStats.TotalCount, Is.EqualTo(4)); } diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index c294001cf3..f1f6eee178 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -17,6 +17,7 @@ using global::ServiceControl.Operations.Metrics; using global::ServiceControl.Recoverability.Retrying.Metrics; using global::ServiceControl.Persistence; + using global::ServiceControl.Persistence.DataMigration; using global::ServiceControl.Transports; using Licensing; using Microsoft.AspNetCore.HttpLogging; @@ -107,6 +108,11 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S services.AddStartupChecks(); services.AddPersistence(settings); + + // The checkpoint store is optional: EF Core registers one, RavenDB does not. + services.TryAddSingleton(provider => + new CheckpointMigrationState(provider.GetService())); + services.AddMetrics(settings.PrintMetrics); hostBuilder.AddTelemetry(settings); services.AddServiceControlHealthChecks(); diff --git a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs index 00d38bb0b3..1890ba5242 100644 --- a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Hosting.Commands using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -13,8 +14,10 @@ namespace ServiceControl.Hosting.Commands using ServiceControl.ExternalIntegrations; using ServiceControl.Hosting.Https; using ServiceControl.Infrastructure.Health; + using ServiceControl.Migration; using ServiceControl.Monitoring; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Recoverability; /// @@ -25,15 +28,13 @@ namespace ServiceControl.Hosting.Commands /// class ErrorIngestionOnlyCommand : AbstractCommand { - static readonly string[] SupportedStorageNames = ["SQLServer", "PostgreSQL"]; - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) { EnsureStorageCanScaleOut(settings); var app = BuildHost(settings); - await app.RunAsync(settings.RootUrl); + await app.RunAsync(); } internal static WebApplication BuildHost(Settings settings, Action customize = null) @@ -47,12 +48,27 @@ internal static WebApplication BuildHost(Settings settings, Action + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService(), + "this error ingestion only host")); + + if (settings.MigrationEnabled) + { + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + } + customize?.Invoke(hostBuilder); var app = hostBuilder.Build(); app.MapServiceControlHealthChecks(); + // Set here rather than passed to RunAsync, so a caller that starts the host itself gets the + // configured address instead of Kestrel's default port. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + return app; } @@ -60,7 +76,7 @@ static void EnsureStorageCanScaleOut(Settings settings) { var manifest = PersistenceManifestLibrary.Find(settings.PersistenceType); - if (manifest == null || !SupportedStorageNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) + if (manifest == null || !PersistenceFactory.SqlPersistenceNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) { throw new Exception( $"--error-ingestion-only requires SQL Server or PostgreSQL storage, but this instance is configured to use '{settings.PersistenceType}'. Scaling out error ingestion is not supported for this storage type."); diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index d32da70fb1..c8970bdbac 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -17,6 +17,9 @@ using Recoverability; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; + using ServiceControl.Migration; + using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; class ImportFailedErrorsCommand : AbstractCommand { @@ -62,6 +65,15 @@ internal static IHost BuildHost(Settings settings) hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddErrorMessageEnricher(); + // Importing writes failed messages into the database, so on SQL it waits for an unfinished copy as an ingestion-only worker does. + if (PersistenceFactory.SqlPersistenceNames.Contains(PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name, StringComparer.OrdinalIgnoreCase)) + { + hostBuilder.Services.AddHostedService(provider => + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService(), + "--import-failed-errors")); + } + return hostBuilder.Build(); } } diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs index 21446e6c5e..40d832e124 100644 --- a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -11,6 +11,23 @@ namespace ServiceControl.Hosting.Commands class MigrationSourceReportCommand : AbstractCommand { public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + try + { + await Report(settings, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + await Console.Error.WriteLineAsync(e.Message); + Environment.ExitCode = 1; + } + } + + static async Task Report(Settings settings, CancellationToken cancellationToken) { await using var source = await PersistenceFactory.OpenMigrationSource(settings, cancellationToken); diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs index 33a6361511..436431f6f8 100644 --- a/src/ServiceControl/Hosting/Commands/RunCommand.cs +++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs @@ -1,9 +1,12 @@ namespace ServiceControl.Hosting.Commands { + using System; using System.Threading; using System.Threading.Tasks; using Infrastructure.WebApi; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -11,11 +14,19 @@ using ServiceControl; using ServiceControl.Hosting.Auth; using ServiceControl.Hosting.Https; + using ServiceControl.Migration; using ServicePulse; class RunCommand : AbstractCommand { - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + public override Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) => + Run(settings, customize: null, cancellationToken); + + /// + /// Builds and runs the full instance. When the migration is turned on, the required copy is added as the + /// first thing the host starts, so the instance opens only once the copy has finished. + /// + internal static async Task Run(Settings settings, Action customize, CancellationToken cancellationToken = default) { var endpointConfiguration = new EndpointConfiguration(settings.InstanceName); var assemblyScanner = endpointConfiguration.AssemblyScanner(); @@ -31,7 +42,17 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel hostBuilder.AddServiceControl(settings, endpointConfiguration); hostBuilder.AddServiceControlApi(settings.CorsSettings); - var app = hostBuilder.Build(); + customize?.Invoke(hostBuilder); + + if (settings.MigrationEnabled) + { + hostBuilder.Services.AddHostedService(provider => new RequiredCopyBeforeTheHostOpens(provider, settings)); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + } + + // A start that refuses would otherwise leave everything the host built undisposed. + await using var app = hostBuilder.Build(); + app.UseServiceControl(settings.ForwardedHeadersSettings, settings.HttpsSettings); if (settings.EnableIntegratedServicePulse) { @@ -39,7 +60,27 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel } app.UseServiceControlAuthentication(settings.OpenIdConnectSettings.Enabled); - await app.RunAsync(settings.RootUrl); + // WebApplication's RunAsync(url) takes no cancellation token, so set the url here and call IHost's RunAsync below. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + + // Read before RunAsync, which disposes the host before the filter below runs, so reading it there would + // throw and the catch would never fire. + var lifetime = app.Lifetime; + + // Starting the host is what marks the target as opened, through the RecordHostOpenedOnTarget hosted + // service registered above, so nothing here does it. + try + { + await app.RunAsync(cancellationToken); + } + // Stopping the service during the required copy cancels it inside host start, and that is a stop + // rather than a failure: every committed batch is durable and the next start resumes from the cursor. +#pragma warning disable PS0020 // The host cancels on its own lifetime token, not the caller's, so that is the one to filter on + catch (OperationCanceledException) when (settings.MigrationEnabled && lifetime.ApplicationStopping.IsCancellationRequested) +#pragma warning restore PS0020 + { + } } } } diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index 6fba5341f9..d616180539 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -23,20 +23,6 @@ This mode runs no setup, so combining it with --setup or --setup-and-run is refu Message bodies must be stored somewhere every host can read, so this mode should not be combined with file system body storage unless the path is a shared mount. -MIGRATION SOURCE REPORT - - ServiceControl.exe --migration-source-report - -Reports what a migration would read from the RavenDB source: the facts the source reports about -itself, with the setting each came from, and a row count for everything it holds. The source reads the instance's own -RavenDB settings, so keep them in place when switching PersistenceType. - -For a RavenDB source: an EXTERNAL server can be reported on while ServiceControl is running. An EMBEDDED source -cannot: ServiceControl starts its own RavenDB process against that data directory, and a second one -cannot attach to it. Stop the ServiceControl service first, run the report, and start it again. An -instance that does not ship the RavenDB server, such as the container image, cannot report on an -embedded source at all. - SERVICE INSTALL AND UNINSTALL AND CONFIGURATION OPTIONS As of Service Control 1.7 the command line uninstall and install switches have been removed. diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index c601c07b57..745f2bbde9 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,6 +16,7 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -185,6 +186,7 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } + public bool MigrationEnabled => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.EnabledKey, MigrationSettings.DefaultEnabled); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs new file mode 100644 index 0000000000..68ab286716 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Registered by a test host that needs a copy to run on a build that does not yet carry the whole migration. +/// +class AllowUnreleasedMigration; + +/// +/// Refuses every copy until the whole migration has shipped. A build that can copy the required categories but +/// not yet the background copy, the end-of-migration guard or verification would still commit an instance to the +/// target with no way to finish or check the move. The last phase of the migration deletes this check. +/// +class MigrationIsReleasedCheck(AllowUnreleasedMigration allowUnreleasedMigration = null) : IMigrationStartupCheck +{ + public string Name => "this build carries the whole migration"; + + public Task Run(CancellationToken cancellationToken = default) + { + if (allowUnreleasedMigration is null) + { + throw new Exception( + $"This build of ServiceControl does not yet carry the whole migration from RavenDB, so it will not copy anything. Set {MigrationSettings.EnabledKey} back to false and start ServiceControl again."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs new file mode 100644 index 0000000000..e3ffed392a --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses any target but SQL Server or PostgreSQL, since the source is always RavenDB. The seams underneath are +/// general enough to copy between any two persisters, and this is the check that says which pair is actually supported. +/// +class MigrationPairIsSupportedCheck(Settings settings) : IMigrationStartupCheck +{ + public string Name => "the source and target are the supported pair"; + + public Task Run(CancellationToken cancellationToken = default) + { + var targetName = PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name; + + if (!PersistenceFactory.SqlPersistenceNames.Contains(targetName, StringComparer.OrdinalIgnoreCase)) + { + throw new Exception( + $"Migrating from '{PersistenceFactory.MigrationSourcePersistenceType}' to '{settings.PersistenceType}' is not supported. The only supported migration is from RavenDB to SQL Server or PostgreSQL, so set {Settings.SettingsRootNamespace}/PersistenceType to {string.Join(" or ", PersistenceFactory.SqlPersistenceNames)} before setting {MigrationSettings.EnabledKey}."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs new file mode 100644 index 0000000000..d8f2ef87bc --- /dev/null +++ b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the engine options out of the settings, which refuses an optional category window that is not a time +/// span before anything is copied rather than part way through the copy. +/// +class OptionalCategoryWindowsAreValidCheck(TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) : IMigrationStartupCheck +{ + public string Name => "the optional category windows are valid"; + + /// + /// The options parsed from the settings, which is null until has returned. + /// + public MigrationEngineOptions Options { get; private set; } + + public Task Run(CancellationToken cancellationToken = default) + { + Options = MigrationEngineOptions.FromSettings(Settings.SettingsRootNamespace, eventRetentionPeriod, errorRetentionPeriod); + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs new file mode 100644 index 0000000000..00f5903bd4 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy that the instance's own retry history setting would throw away. At a depth of zero the SQL +/// persisters empty the history table whenever a retry completes, and the migrated rows would go with it. +/// +/// The instance's ServiceControl/RetryHistoryDepth. +class RetryHistoryDepthIsSafeCheck(int retryHistoryDepth) : IMigrationStartupCheck +{ + public string Name => "the retry history depth will not empty a migrated table"; + + public Task Run(CancellationToken cancellationToken = default) + { + // The depth at which RetryHistoryDataStore.TrimHistory deletes every row rather than trimming. + if (retryHistoryDepth <= 0) + { + throw new Exception( + $"ServiceControl/RetryHistoryDepth is {retryHistoryDepth}, and at that depth the SQL persisters delete every row of HistoricRetryOperations when a retry completes. The RetryOperations category copies that table, so the first completed retry after cutover would silently discard the migrated retry history. Set ServiceControl/RetryHistoryDepth to a positive number, or clear it to use the default of 10, before setting {MigrationSettings.EnabledKey}."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs new file mode 100644 index 0000000000..6433658d4b --- /dev/null +++ b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs @@ -0,0 +1,58 @@ +namespace ServiceControl.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses to start a host that writes failed messages into the database, an error ingestion only host or +/// --import-failed-errors, until every required category is Done or Abandoned. An optional category, which copies +/// in the background once the main host has opened, never holds it back, but a category under an id this build +/// does not know does. A refusal throws, which fails the start and stops the host. Start such a host only after the +/// main host has opened, because a database with no checkpoint row yet reads as one no migration has touched. +/// +/// Where the checkpoint rows are read. A database no migration has touched holds none, so the host starts as before. +/// Names the host in the refusal, such as "this error ingestion only host". +sealed class FinishedCopyBeforeAnIngestionNodeOpens( + IMigrationCheckpointStore checkpointStore, + string hostDescription) : IHostedLifecycleService +{ + // It runs with the migration off too, because a node someone forgot to flag would ingest into a part-copied + // database and turn abandoning it from a clean rollback into permanent loss. A database that never migrated has + // nothing unfinished, so its nodes start as before. + public async Task StartingAsync(CancellationToken cancellationToken = default) + { + var unfinished = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => !checkpoint.State.IsFinished()) + .ToArray(); + + if (unfinished.Length == 0) + { + return; + } + + var detail = string.Join(". ", unfinished.Select(checkpoint => checkpoint.State.IsFailed() + ? MigrationStartup.FailedDetail(checkpoint) + : $"{checkpoint.CategoryId} is Copying ({checkpoint.State})")); + + var advice = unfinished.Any(checkpoint => !checkpoint.State.IsFailed()) + ? "Let the instance running the copy finish it and start this host again." + : "Start this host again once they are settled."; + + throw new Exception($"A copy into this database has not finished, so {hostDescription} will not start and nothing has been lost. {detail}. {advice}"); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs new file mode 100644 index 0000000000..dc17bab1d1 --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -0,0 +1,607 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// +/// The required copy, from the checks that decide whether it can run at all to the refusal it throws when a +/// category did not finish. Everything here happens with ServiceControl closed, which is the only window in +/// which the copy can be thrown away at no cost. +/// +static class MigrationStartup +{ + // A category needs both a reader and a writer, so a half-implemented one is never attempted. + internal static IReadOnlyCollection CopyableCategoryIds(IReadOnlyCollection sourceSupports, IReadOnlyCollection targetSupports) => + [.. sourceSupports.Intersect(targetSupports, StringComparer.Ordinal)]; + + /// + /// Runs the startup checks, copies every required category this build can copy that is not already settled, + /// and then seeds the the host reads. Once every required category this build + /// copies is settled and no row under an unknown id is unfinished, a source that will not open is logged and + /// recorded on the optional categories' rows instead of refused, nothing is copied, and the host still opens. Throws when a check refuses or a category does not + /// finish, with a message telling the operator what to do and how to go back; the caller must let that stop + /// the host. Call it after the host is built and before it starts, because the copy has to finish before + /// anything else opens on the target. + /// + /// The built host's services, which is where the target, the checkpoint store and the migration state come from. + /// The instance settings, read for the source and target persistence types. + /// Cancelled when the host is shutting down, which ends the copy without a refusal message. + public static async Task RunRequiredCopy(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + await MigrationStartupCheckRunner.Run( + [ + new MigrationIsReleasedCheck(services.GetService()), + new MigrationPairIsSupportedCheck(settings) + ], cancellationToken); + + var loggerFactory = services.GetRequiredService(); + var logger = loggerFactory.CreateLogger(typeof(MigrationStartup)); + var target = services.GetRequiredService(); + var checkpointStore = services.GetRequiredService(); + var timeProvider = services.GetRequiredService(); + + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var copyable = CopyableCategoryIds(source.SupportedCategoryIds, target.SupportedCategoryIds); + + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + var engine = new MigrationEngine( + source, + target, + checkpointStore, + timeProvider, + options, + loggerFactory.CreateLogger()); + + // Optional categories copy beside the running services, so only the required ones hold those services back. + var holdingBack = engine.SelectCategories(MigrationCategoryKind.Required) + .Select(category => category.Id) + .ToArray(); + + var toCopy = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => copyable.Contains(category.Id)) + .ToArray(); + + var deferred = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => !copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + var checkpoints = await checkpointStore.ReadAll(cancellationToken); + var unfinishedOutsideTheCopy = UnfinishedRowsOutsideTheCopy(checkpoints, toCopy); + var requiredCopySettled = RequiredCopyIsSettled(checkpoints, toCopy); + + try + { + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Safe to open without the source, because every required category is settled and only the background copy of the optional ones reads it. + catch (Exception exception) when (requiredCopySettled) + { + // The runner's wrapper says ServiceControl will not start, which is false here and would land on the rows. + var cause = exception.InnerException ?? exception; + + logger.LogError(cause, + "The {SourcePersistenceType} migration source could not be opened, so ServiceControl opens without it: every required category this build copies is Done or Abandoned and no row under an id it does not know is unfinished, so only the background copy of the optional categories reads the source. The optional categories still copying stay as they are until a start that can open the source, and their checkpoints carry this error.", + PersistenceFactory.MigrationSourcePersistenceType); + + await RecordSourceOutage(checkpointStore, engine.SelectCategories(MigrationCategoryKind.Optional), cause, cancellationToken); + await SeedMigrationState(services, holdingBack, cancellationToken); + return; + } + catch (Exception exception) + { + throw new Exception($"{exception.Message} {NoWayOutWithoutTheSource(settings)}", exception.InnerException); + } + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + logger.LogInformation( + "Migration mode: copying {CopyCount} required categories before ServiceControl opens ({DeferredCount} not yet implemented: {Deferred})", + toCopy.Length, deferred.Length, string.Join(", ", deferred)); + + // Captured before the copy so the report can tell a category this run finished from one an earlier run did. + var runStartedAt = timeProvider.GetUtcNow().UtcDateTime; + + var finished = await CopyOrExplainWhyItStopped( + RunRequiredCategories(engine, toCopy, checkpointStore, timeProvider, logger, cancellationToken), + settings); + + ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt); + + RefuseIfAnyCategoryDidNotComplete(toCopy, finished, unfinishedOutsideTheCopy, settings); + + await SeedMigrationState(services, holdingBack, cancellationToken); + } + + static async Task SeedMigrationState(IServiceProvider services, IReadOnlyCollection holdingBack, CancellationToken cancellationToken) + { + if (services.GetRequiredService() is CheckpointMigrationState state) + { + await state.Seed(holdingBack, cancellationToken); + } + } + + /// + /// Runs every startup check in the order they have to run, and opens the target and the source as two of + /// them. The order is what the operator sees: a check that costs nothing comes before one that connects to a + /// database, and the source's own checks run last because they need it open. + /// + /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. + /// The options read from the settings, which the window check parsed on its way past. + /// A check refused. The message names the check and says what to do, and this start has copied nothing. + public static async Task RunChecksAndOpen(IServiceProvider services, Settings settings, IMigrationSource source, CancellationToken cancellationToken = default) + { + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + return options; + } + + /// + /// Runs every startup check that does not need the source, in the order they have to run, ending with + /// opening the target. A check that costs nothing comes before one that connects to a database. The target + /// is open when this returns, and the source has not been touched. + /// + /// The built host's services, which is where the target and its readiness checks come from. + /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. + /// Cancelled when the host is shutting down, which stops the checks without a refusal message. + /// The options read from the settings, which the window check parsed on its way past. + /// A check refused or the target did not open. The message names the check and says what to do, and this start has copied nothing. + /// The host is shutting down. + public static async Task RunChecksAndOpenTarget(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + var target = services.GetRequiredService(); + var readiness = services.GetRequiredService(); + var categories = new OptionalCategoryWindowsAreValidCheck(settings.EventsRetentionPeriod, settings.ErrorRetentionPeriod); + + await MigrationStartupCheckRunner.Run( + [ + categories, + new RetryHistoryDepthIsSafeCheck(settings.RetryHistoryDepth), + .. readiness.ContributedChecks(), + new Step("the migration target opens", target.Open) + ], cancellationToken); + + return categories.Options; + } + + /// + /// Records on each optional category's checkpoint that the source could not be opened, so status and verify + /// read the outage from the rows. The required copy calls it on a start that + /// opens without the source, and the background copier calls it when its own open fails. It changes no + /// state, cursor or count: a row still copying gains the error as its , + /// a category with no row gets a new not-started row carrying the error, and a row that is Done, Failed or + /// Abandoned is left alone. The next start that resumes a row clears the error. + /// + /// Where the rows are read and saved. + /// The optional categories this instance selected. + /// Why the source could not be opened. Its type and message go into the error. + /// Cancelled when the host is shutting down. + /// Another writer saved one of these rows since it was read. + internal static async Task RecordSourceOutage(IMigrationCheckpointStore checkpointStore, IReadOnlyList optionalCategories, Exception exception, CancellationToken cancellationToken = default) + { + var error = $"The {PersistenceFactory.MigrationSourcePersistenceType} migration source could not be opened: {exception.GetType().Name}: {exception.Message.TrimEnd('.', ' ')}. This category stays as it is, and the next start tries the source again."; + + foreach (var category in optionalCategories) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken); + + if (checkpoint is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, error), cancellationToken); + } + else if (!checkpoint.State.IsFinished() && !checkpoint.State.IsFailed()) + { + await checkpointStore.Upsert(checkpoint with { LastError = error }, cancellationToken); + } + } + } + + /// + /// Copies the required categories one after another under the stall watchdog and returns where each one ended, + /// in the same order. A category that commits nothing for is + /// stopped and settled Halted, and the categories after it still get their go. Before copying anything it saves + /// a not-started checkpoint for every category that has none, as does. + /// + /// Copies each category. + /// The categories to copy, in the order to copy them. + /// Where each category's row is seeded, watched and, after a stall, settled. + /// The clock the watchdog measures a stall on. + /// Receives the watchdog's progress lines and the stall. + /// The host's token. Cancelling it stops the copy and leaves the running category as its last committed batch left it, so the next start resumes it. + /// The checkpoint each category ended on. + /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. + /// The host is shutting down. + internal static async Task> RunRequiredCategories( + MigrationEngine engine, + IReadOnlyList categories, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + ILogger logger, + CancellationToken cancellationToken = default) + { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null), cancellationToken); + } + } + + await using var progress = new ClosedWindowProgress(checkpointStore, timeProvider, logger, [.. categories.Select(category => category.Id)], cancellationToken); + + var results = new List(categories.Count); + + foreach (var category in categories) + { + using var categoryCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + progress.Watch(category.Id, timeProvider.GetUtcNow().UtcDateTime, categoryCancellation); + + try + { + results.Add(await engine.RunCategoryAsync(category, categoryCancellation.Token)); + } + // Only the watchdog cancels this source without the host's token, so the filter tells a stall from a shutdown. +#pragma warning disable PS0020 + catch (OperationCanceledException) when (categoryCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) +#pragma warning restore PS0020 + { + // The host's token, because the stall has already cancelled the category's and a store call on that would fail. + var stalled = await checkpointStore.Read(category.Id, cancellationToken); + + results.Add(await checkpointStore.Upsert(stalled with + { + State = MigrationCategoryState.Halted, + SettledAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = StallExplanation(category.Id) + }, cancellationToken)); + } + } + + return results; + } + + /// + /// Waits for the copy and turns a second instance writing checkpoints to the same database into a refusal the + /// operator can act on. A shutdown and every other failure come out as they are. + /// + /// The copy, already running. + /// Read for the persistence type the refusal names. + /// What the copy returned. + /// A checkpoint saved by another instance. The message says what to do and how to go back. + /// The host is shutting down. +#pragma warning disable PS0018 // The copy it waits on already runs under the host's token, so a token here would have nothing to cancel. + internal static async Task> CopyOrExplainWhyItStopped( + Task> copy, + Settings settings) +#pragma warning restore PS0018 + { + try + { + return await copy; + } + // The engine never turns a conflict into a halt, so without this the copy ends on the checkpoint store's + // own message and none of the advice every other refusal carries. + catch (MigrationCheckpointConflictException exception) + { + throw new Exception( + $"The required copy stopped because another writer saved a migration checkpoint for this instance, which is a second ServiceControl pointed at the same {settings.PersistenceType} database. ServiceControl will not start. {exception.Message} Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on; the copy resumes from its last committed batch. " + + RollbackAdvice(settings), exception); + } + } + + /// + /// Throws unless every attempted category finished. A category that reported no checkpoint at all counts as + /// outstanding too, because nothing says how much of it was copied. + /// + /// A category did not finish. The message names each one with its state and counts, names the commands that move each Failed one on, and says how to go back. + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList attempted, + IReadOnlyList finished, + Settings settings) + { + var outstanding = finished + .Where(checkpoint => !checkpoint.State.IsFinished()) + .Select(checkpoint => checkpoint.State.IsFailed() ? FailedDetail(checkpoint) : CopyingDetail(checkpoint)) + .ToList(); + + var reported = finished.Select(checkpoint => checkpoint.CategoryId).ToHashSet(StringComparer.Ordinal); + + outstanding.AddRange(attempted + .Where(category => !reported.Contains(category.Id)) + .Select(category => $"{category.Id} reported no checkpoint at all, so whether it copied anything is unknown")); + + if (outstanding.Count == 0) + { + return; + } + + throw new Exception( + $"The required copy did not finish, so ServiceControl will not start and nothing has been lost. {string.Join(". ", outstanding)}. " + + RollbackAdvice(settings)); + } + + /// + /// Says whether the host may open without the source: every category this start copies is finished, and no row + /// outside the copy is unfinished. + /// + /// Every checkpoint row in the target. + /// The categories this start copies. + /// True when nothing required is still outstanding. + internal static bool RequiredCopyIsSettled(IReadOnlyList checkpoints, IReadOnlyList toCopy) => + UnfinishedRowsOutsideTheCopy(checkpoints, toCopy).Count == 0 + && toCopy.All(category => checkpoints.Any(checkpoint => checkpoint.CategoryId == category.Id && checkpoint.State.IsFinished())); + + /// + /// Throws unless every attempted category finished and no row outside the copy is unfinished, naming each + /// outstanding row in the same words. + /// + /// The categories this start copied. + /// Where each attempted category ended. + /// The rows found. + /// Read for the persistence type the refusal names. + /// Something is outstanding. The message names it and says how to go back. + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList attempted, + IReadOnlyList finished, + IReadOnlyList unfinishedOutsideTheCopy, + Settings settings) => + RefuseIfAnyCategoryDidNotComplete(attempted, [.. finished, .. unfinishedOutsideTheCopy], settings); + + /// + /// Finds the unfinished rows the copy will not run: a row under an id this build does not know, which counts as + /// required because a newer build may have written it for a category that must finish, and a required row for a + /// category this build cannot copy yet. A finished row, and a row known to be optional, hold nothing. + /// + /// Every checkpoint row in the target. + /// The categories this start copies, whose rows the copy judges itself. + /// The rows that keep the host closed, in the order they were read. + internal static IReadOnlyList UnfinishedRowsOutsideTheCopy(IReadOnlyList checkpoints, IReadOnlyList toCopy) => + [.. checkpoints + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => toCopy.All(category => category.Id != checkpoint.CategoryId)) + .Where(checkpoint => !checkpoint.State.IsFinished())]; + + internal static string FailedDetail(MigrationCheckpoint checkpoint) + { + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? $" ({string.Join("; ", counts.Select(reason => $"{reason.Key} {reason.Value}{(reason.Key.IsPermanent() ? ", no retry can fix" : "")}"))})" + : ""; + + return $"{checkpoint.CategoryId} is Failed ({checkpoint.State}) after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{reasons}{LastErrorClause(checkpoint)}; " + + $"run --migration-retry {checkpoint.CategoryId} once the cause is fixed, which copies the category again from the start, or --migration-abandon {checkpoint.CategoryId} to keep what was copied and give up the rest, both with ServiceControl stopped"; + } + + static string CopyingDetail(MigrationCheckpoint checkpoint) => + $"{checkpoint.CategoryId} is {checkpoint.State} after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{LastErrorClause(checkpoint)}"; + + // Trimmed because the refusal punctuates each category's sentence itself. + static string LastErrorClause(MigrationCheckpoint checkpoint) => + checkpoint.LastError is null ? "" : $": {checkpoint.LastError.TrimEnd('.', ' ')}"; + + /// + /// Logs one line per category saying what it copied and what it left behind. Skipped rows are warned about + /// one at a time while the copy runs, and nothing else states the total or says what becomes of them. + /// + /// When this start began, which is what tells a category this run finished from one an earlier run did. + internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList finished, ILogger logger, DateTime runStartedAt) + { + foreach (var checkpoint in finished) + { + // It did not run, and the refusal that follows names it with the category it waits for. + if (checkpoint.State == MigrationCategoryState.Blocked) + { + continue; + } + + // A category already finished when this run began is returned without being run, so its counts are + // an earlier run's. A Failed one is returned untouched too, but it is not finished, so it falls through. + if (checkpoint.State.IsFinished() && checkpoint.SettledAt is { } settledAt && settledAt < runStartedAt) + { + logger.LogInformation( + "{CategoryId}: already finished before this start, by a run that copied {Copied} and skipped {Skipped}. This start copied nothing.", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.SkippedCount); + continue; + } + + // An exception, a stall or unbalanced counts fail a category without a skip, and that is not a clean copy. + if (checkpoint.State.IsFailed() && checkpoint.SkippedCount == 0) + { + logger.LogWarning("{CategoryId} is Failed ({State}) after {Copied} copied and {AlreadyPresent} already present. The refusal that follows says why and what to run.", + checkpoint.CategoryId, checkpoint.State, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + continue; + } + + if (checkpoint.SkippedCount == 0) + { + logger.LogInformation("{CategoryId}: {Copied} copied, {AlreadyPresent} already present, nothing skipped", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + continue; + } + + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? string.Join(", ", counts.OrderByDescending(reason => reason.Value).Select(reason => $"{reason.Key} {reason.Value}")) + : "no reason recorded"; + + // Only a Failed category can be retried, so a Done one is never offered the command. + var whatBecomesOfThem = checkpoint.State.IsFailed() + ? WhatARetryCanDo(checkpoint) + : checkpoint.State == MigrationCategoryState.Complete + ? "They were left out as harmless, because ServiceControl would have removed them anyway, and they stay only in the source database." + : "They stay only in the source database."; + + logger.LogWarning( + "{CategoryId}: {Copied} copied, {AlreadyPresent} already present, {Skipped} skipped ({Reasons}). {WhatBecomesOfThem}", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount, checkpoint.SkippedCount, reasons, whatBecomesOfThem); + } + } + + // A retry re-reads the whole category, so it is offered only when some of the faults could come across on it. + static string WhatARetryCanDo(MigrationCheckpoint checkpoint) + { + var faults = checkpoint.SkipReasons?.Keys.Where(reason => !reason.IsBenign()).ToArray() ?? []; + var permanent = faults.Where(reason => reason.IsPermanent()).ToArray(); + + if (faults.Length > 0 && permanent.Length == faults.Length) + { + return $"They stay only in the source database, and no retry can fix them, so --migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up the rest."; + } + + var cannotFix = permanent.Length == 0 ? "" : $" No retry can fix the {string.Join(" or ", permanent)} ones."; + + return $"They stay only in the source database until --migration-retry {checkpoint.CategoryId} re-reads them once the cause is fixed.{cannotFix}"; + } + + internal static string StallExplanation(string stalledCategoryId) => + $"Halted: {stalledCategoryId} committed nothing for {ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes, so it was stopped. The limit is not configurable: check that the source and the target are both responding rather than looking for a setting to change."; + + static string RollbackAdvice(Settings settings) => + $"Nothing has opened on {settings.PersistenceType} yet, so setting {MigrationSettings.EnabledKey}=false and pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType} discards the partial copy and returns the instance to {PersistenceFactory.MigrationSourcePersistenceType} with no loss."; + + static string NoWayOutWithoutTheSource(Settings settings) => + $"If the {PersistenceFactory.MigrationSourcePersistenceType} source is already gone, a required category that is Failed or has started copying can be given up with --migration-abandon , with ServiceControl stopped. " + + $"A required category that never started cannot be abandoned: while one is outstanding ServiceControl has never opened on {settings.PersistenceType}, so pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType}, or starting over against an empty {settings.PersistenceType} database, loses nothing it has served."; + + static Step SourceOpens(IMigrationSource source) => new("the migration source opens", source.Open); + + /// + /// Makes opening the target or the source look like a startup check, so a failure to connect is reported in + /// the same words as a check that refused, and in its place in the order. + /// + sealed class Step(string name, Func run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) => run(cancellationToken); + } + + /// + /// Logs how far each category has got while the copy runs, and stops the running category when it commits + /// nothing for . Without it a copy that is waiting on a database nobody is watching + /// holds the instance closed for as long as the operator leaves it. + /// + internal sealed class ClosedWindowProgress : IAsyncDisposable + { + // Not configurable, and not a total timeout: a deadline would kill a copy that is working. + internal static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30); + internal static readonly TimeSpan StallLimit = TimeSpan.FromMinutes(30); + + readonly CancellationTokenSource cancellation; + readonly HashSet attempted; + readonly Task polling; + volatile RunningCategory running; + + public ClosedWindowProgress(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, IReadOnlyCollection attemptedCategoryIds, CancellationToken cancellationToken = default) + { + cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attempted = attemptedCategoryIds.ToHashSet(StringComparer.Ordinal); + polling = Poll(checkpointStore, timeProvider, logger, cancellation.Token); + } + + /// + /// Points the watchdog at the category that is starting now, in place of the one before it. Only this + /// category is judged from here on, from the later of its last committed batch and . + /// + /// The category starting now. + /// When this start began running it. A row an earlier start left carries an older stamp, which would read as a stall at once. + /// Cancelled when the category stalls. The category must run under its token, and nothing else should. + public void Watch(string categoryId, DateTime runStartedAt, CancellationTokenSource stop) => + running = new RunningCategory(categoryId, runStartedAt, stop); + + async Task Poll(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(PollInterval, timeProvider); + + try + { + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + try + { + // Only this run's categories, because a row an earlier run left in progress is not this run's to report. + var inProgress = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => checkpoint.State == MigrationCategoryState.InProgress && attempted.Contains(checkpoint.CategoryId)) + .ToArray(); + + foreach (var checkpoint in inProgress) + { + logger.LogInformation( + "{CategoryId}: {Copied} of {Total} copied, {Skipped} skipped, cursor {Cursor}", + checkpoint.CategoryId, + checkpoint.CopiedCount, + checkpoint.SourceTotal is { } total ? total.ToString() : "an unknown number of", + checkpoint.SkippedCount, + checkpoint.Cursor ?? "the start"); + } + + var watched = running; + + // A resumed row carries the previous run's stamp, so the window starts at whichever is + // later: that stamp, or the moment this category's run began. + var stalled = watched is not null && !watched.Stop.IsCancellationRequested && inProgress.Any(checkpoint => + checkpoint.CategoryId == watched.CategoryId + && timeProvider.GetUtcNow().UtcDateTime + - (checkpoint.LastProgressAt is { } lastProgress && lastProgress > watched.RunStartedAt ? lastProgress : watched.RunStartedAt) > StallLimit); + + if (stalled) + { + logger.LogError( + "{CategoryId} has committed nothing for {StallLimit}, so it is being stopped and settled Halted. The categories after it still get their go.", + watched.CategoryId, StallLimit); + await watched.Stop.CancelAsync(); + } + } + // The copy finished or the host is stopping: the outer catch ends the poll. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Log, don't throw: an error here would come out of DisposeAsync and hide what the copy itself failed with. + catch (Exception exception) + { + logger.LogError(exception, "The stall watchdog's poll failed, so a stalled copy will not be noticed until a later poll succeeds. The copy itself is unaffected and is still running."); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Either the copy finished and disposal cancelled the poll, or the host is shutting down. + } + } + + public async ValueTask DisposeAsync() + { + await cancellation.CancelAsync(); + + try + { + await polling; + } + finally + { + cancellation.Dispose(); + } + } + + sealed record RunningCategory(string CategoryId, DateTime RunStartedAt, CancellationTokenSource Stop); + } +} diff --git a/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs new file mode 100644 index 0000000000..4ccc0626a5 --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Runs startup checks in order and stops at the first one that refuses. +/// +static class MigrationStartupCheckRunner +{ + /// + /// Runs each check in turn, and wraps whatever a failing one throws in a message naming the check and + /// saying that this start has copied nothing, because every check runs before this start's copy. + /// + /// A check failed. The check's own message is kept, and its exception is the inner one. + public static async Task Run(IReadOnlyList checks, CancellationToken cancellationToken = default) + { + foreach (var check in checks) + { + try + { + await check.Run(cancellationToken); + } + // A shutdown is not a check failing, and saying it was would send the customer after the wrong thing. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + throw new Exception( + $"Migration startup check '{check.Name}' failed, so ServiceControl will not start, and this start has copied nothing. {exception.Message}", exception); + } + } + } +} diff --git a/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs new file mode 100644 index 0000000000..676eb714fa --- /dev/null +++ b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// +/// Calls when a host started with the migration on opens +/// on a database a migration has already written to. +/// +sealed class RecordHostOpenedOnTarget(IServiceProvider services) : IHostedLifecycleService +{ + // StartedAsync, not StartAsync: the web server binds its port during StartAsync, and a start that dies there + // served nothing. The stores are resolved here rather than injected, because a RavenDB host registers neither + // and is refused before it gets this far. + public async Task StartedAsync(CancellationToken cancellationToken = default) + { + var checkpointStore = services.GetRequiredService(); + + // Only a database a migration has written to gets the stamp, so the marker answers whether a host + // has opened since the copy began rather than whether one ever ran on this database. + if ((await checkpointStore.ReadAll(cancellationToken)).Count > 0) + { + await services.GetRequiredService().RecordHostOpened(cancellationToken); + } + } + + public Task StartingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs b/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs new file mode 100644 index 0000000000..9ca9388246 --- /dev/null +++ b/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using ServiceBus.Management.Infrastructure.Settings; + +/// +/// Runs the required copy as the host starts, before any hosted service of its own does. +/// A refusal throws, which fails the start and stops the host, as the copy requires. +/// +sealed class RequiredCopyBeforeTheHostOpens(IServiceProvider services, Settings settings) : IHostedLifecycleService +{ + // Inside the host rather than before it, because a Windows service reports itself started only once the host + // starts, and the Service Control Manager kills a process that has said nothing for 30 seconds. Every + // StartingAsync runs before any hosted service, so nothing has bound a port or begun ingesting. + public Task StartingAsync(CancellationToken cancellationToken = default) => + MigrationStartup.RunRequiredCopy(services, settings, cancellationToken); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index c431680696..5fd045058d 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -10,6 +10,12 @@ namespace ServiceControl.Persistence static class PersistenceFactory { + /// + /// The manifest names of the two persisters built on EF Core. Anything that is true of both of them and + /// of neither RavenDB nor a future persister is decided by this list. + /// + public static readonly string[] SqlPersistenceNames = ["SQLServer", "PostgreSQL"]; + public static IPersistence Create(Settings settings, bool maintenanceMode = false) { var persistenceConfiguration = CreatePersistenceConfiguration(settings.PersistenceType, settings); @@ -43,7 +49,20 @@ public static IMigrationSource CreateMigrationSource(Settings settings) public static async Task OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) { var source = CreateMigrationSource(settings); - await source.Open(cancellationToken); + var opened = false; + + try + { + await source.Open(cancellationToken); + opened = true; + } + finally + { + if (!opened) + { + await source.DisposeAsync(); + } + } return source; } diff --git a/src/ServiceControl/Program.cs b/src/ServiceControl/Program.cs index 7234479cd3..96fb2359df 100644 --- a/src/ServiceControl/Program.cs +++ b/src/ServiceControl/Program.cs @@ -39,7 +39,7 @@ await new CommandRunner(arguments.Command).Execute(arguments, settings); - return 0; + return Environment.ExitCode; } catch (Exception ex) { @@ -52,7 +52,13 @@ LoggingConfigurator.ConfigureNLog("bootstrap.txt", "./", NLog.LogLevel.Fatal); NLog.LogManager.GetCurrentClassLogger().Fatal(ex, "Unrecoverable error"); } - throw; + + // The message goes to the console on its own, because the log above already holds the whole exception. + // Rethrowing instead would print the stack trace a second time and fire the unhandled-exception handler + // for a third, which buries a configuration mistake in what reads like a crash. + await Console.Error.WriteLineAsync(ex.Message); + + return 1; } finally { From 09021eea8a5459b4a77a9627ef11638989109987 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 12 Nov 2026 16:12:51 +0800 Subject: [PATCH 17/28] Migration engine: Copy unresolved failed messages and their bodies to SQL (#6005) * Add support for failed message migration with bodys * Enhance migration process and tests: improve dry run estimates, add handling for failure details without timestamps, and optimize SQL Server and PostgreSQL dialects for better performance. * Add details on handling unresolved and retry-issued failed messages in migration documentation * Fix migration process for failed messages: add retry claims for retry-issued messages, update tests for accurate staging behavior, and refine documentation on message handling. * Clarify message skip reasons in migration documentation: differentiate between fault and harmless skips, and specify conditions for retries. * Remove unnecessary phrasing from migration overview documentation * Refactor FailedMessagesWriter to use FailedMessageDocumentTranslator * Add test for handling null content type in FailedMessageDocumentTranslator * Refactor indexing wait methods to allow error states in tests and update exception handling in migration tests * Add error handling for RavenDB index queries in FailedMessagesReader --- .../ravendb-to-sql-migration-instructions.md | 8 +- .../ravendb-to-sql-migration-overview.md | 74 +- .../ravendb-to-sql-migration-system-design.md | 16 +- .../FailedMessageBodyMigrationTests.cs | 185 ++++ .../MigrationAcceptanceTest.cs | 39 + .../When_a_batch_exceeds_one_statement.cs | 32 + ...unresolved_failed_messages_are_migrated.cs | 384 ++++++++ .../PostgreSqlMigrationSqlDialect.cs | 7 + .../SqlServerMigrationSqlDialect.cs | 21 +- .../BodyStorageIsWritableCheck.cs | 6 +- .../DataMigration/EFCoreMigrationTarget.cs | 11 +- .../FailedMessageDocumentTranslation.cs | 24 + .../FailedMessageDocumentTranslator.cs | 202 ++++ .../DataMigration/IMigrationCategoryWriter.cs | 7 +- .../DataMigration/MigrationCategoryWriter.cs | 17 +- .../Writers/EndpointSettingsWriter.cs | 24 +- .../Writers/FailedMessagesWriter.cs | 166 ++++ .../Writers/KnownEndpointsWriter.cs | 15 +- .../UnitOfWork/MessageBodyClassifier.cs | 4 +- .../Infrastructure/IMigrationSqlDialect.cs | 11 + .../Infrastructure/MigrationInsert.cs | 79 +- .../MigrationSqlDialectExtensions.cs | 12 + .../DataMigration/FailedMessageStream.cs | 105 ++ .../DataMigration/MigrationCategoryReader.cs | 14 +- .../MigrationEndpointDetailsParser.cs | 95 ++ .../DataMigration/RavenDocumentStream.cs | 71 +- .../DataMigration/RavenMigrationSource.cs | 31 +- .../RavenReadOnlySourceLifecycle.cs | 29 +- .../Readers/FailedMessagesReader.cs | 196 ++++ .../MigrationValueLengthTests.cs | 81 ++ .../FailedMessageBodySourceTests.cs | 94 ++ .../FailedMessageStatusChangedAtTests.cs | 114 +++ ...grationEndpointDetailsParserParityTests.cs | 106 +++ .../RavenMigrationSourceTests.cs | 5 +- .../UnresolvedFailedMessageSourceTests.cs | 901 ++++++++++++++++++ .../Infrastructure/RavenIndexAwaiter.cs | 14 +- .../MigrationGroupInsertSeekTests.cs | 69 ++ .../MigrationPlanReuseTests.cs | 78 ++ .../MigrationValueLengthTests.cs | 70 ++ .../EndpointSettingsMigrationTargetTests.cs | 28 + .../FailedMessageDocumentTranslatorTests.cs | 570 +++++++++++ .../FailedMessageMigrationCoverageTests.cs | 80 ++ .../FailedMessagesMigrationTargetTests.cs | 467 +++++++++ .../KnownEndpointsMigrationTargetTests.cs | 28 + .../Migration/MigrationTargetCoverageTests.cs | 20 +- .../EFCore/Migration/PreparedSkips.cs | 36 + .../DataMigration/IMigrationSource.cs | 4 +- .../DataMigration/IMigrationTarget.cs | 10 +- .../DataMigration/MigrationBodyLoader.cs | 129 +++ .../MigrationCategoryStateExtensions.cs | 10 +- .../DataMigration/MigrationEngine.cs | 86 +- .../DataMigration/MigrationMetadataKeys.cs | 19 + .../DataMigration/MigrationSkipReason.cs | 39 +- .../AllowUnreleasedMigrationSwitchTests.cs | 37 + .../CheckpointMigrationStateTests.cs | 2 +- .../Fakes/InMemoryMigrationSource.cs | 78 +- .../Fakes/InMemoryMigrationTarget.cs | 48 +- .../Migration/MigrationBodyLoaderTests.cs | 212 +++++ .../MigrationEngineBodyRetryTests.cs | 17 +- .../MigrationEngineCollisionTests.cs | 3 +- .../Migration/MigrationEngineHaltTests.cs | 3 +- .../MigrationEngineSkipReasonTests.cs | 30 +- .../Migration/RequiredCopyGateTests.cs | 53 +- .../Migration/StoppedCopyExplanationTests.cs | 136 ++- .../Hosting/Commands/RunCommand.cs | 26 + .../Checks/MigrationIsReleasedCheck.cs | 4 +- .../Migration/MigrationStartup.cs | 101 +- 67 files changed, 5427 insertions(+), 266 deletions(-) create mode 100644 src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs create mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs create mode 100644 src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index bf7a425abc..88be796b03 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -31,13 +31,13 @@ Make sure that your setup is supported: Get the data ready: -- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. +- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. The copy reads archived and resolved messages only after ServiceControl opens. The exception is a RavenDB index that is broken or leaves out some failed messages: then the copy reads every failed message during the outage, logs a warning, and a large archive makes the outage longer too. - Run `--import-failed-errors` while the instance is still on RavenDB. A failed error import is a message that ServiceControl took off the error queue but failed to store. The migration copies these either way, but imported ones arrive as ordinary failed messages. Keep these settings as they are: - Keep `ServiceControl/RetryHistoryDepth` above 0. If it is 0 or less, the migration refuses to start. At that value, the first completed retry after the move deletes all the copied retry history. -- Do not change `ServiceControl/ErrorRetentionPeriod` or `ServiceControl/EventRetentionPeriod` until the migration is finished. The migration uses them to decide which rows are too old to copy. If one changes after the copy starts, ServiceControl refuses to start. +- Do not change `ServiceControl/ErrorRetentionPeriod` or `ServiceControl/EventRetentionPeriod` until the migration is finished. The migration uses them to decide which rows are too old to copy. If one changes after an optional category has started copying with its default window, ServiceControl refuses to start and names the category. ## Keep the RavenDB settings @@ -110,9 +110,9 @@ By default, the migration copies all the optional data that RavenDB still holds ## Step 4: Run the dry run -The dry run reads RavenDB and tells you what the migration will do, including a range for how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. +The dry run reads RavenDB and tells you what the migration will do, including an estimate of how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. -Book the outage from the range in the report. The range is the shortest time the outage can take, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. +Book the outage from the estimate in the report, and add a margin. It is an estimate, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. ## Step 5: Back up RavenDB diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 613dca1680..b6a81fa023 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -20,7 +20,8 @@ This page describes the migration as designed, not what is built today. [The ins - **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. - **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. - **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. -- **The main thing that makes the outage long is a big backlog of unresolved failures.** They never age out, they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They almost never age out (see the [6.18 exception](#skipped-rows)), they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. +- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds the unresolved failures through a RavenDB index, so while ServiceControl is closed it reads only them. If the index is still catching up, the copy waits for it first. If the index is broken, or leaves out some failed messages, the copy reads every failed message instead, archived and resolved ones included, and logs a warning saying so. Then a big archive does make the outage longer. ```mermaid flowchart LR @@ -167,13 +168,17 @@ There are eighteen *categories*: sixteen required and two optional. **Never copied, because nothing needs them:** - RavenDB index definitions. -- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations`. +- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations` and `UnarchiveOperations`. - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. - The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. - Broker and audit service version details, which refill on the next throughput collection. - Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if it fails again: on RavenDB it can then never be edited again, and after the move it can be edited once more. - Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. +**Written new by the copy rather than copied:** + +- Retry claims in `FailedMessageRetries` are not copied. A claim ties a message to the retry that sent it, and while it exists, retrying the message from the failed messages list sends nothing. The copy writes a new claim for each retry-issued message it inserts, even one whose claim RavenDB had already released. As on RavenDB, the claim goes when the retried message succeeds, when it fails again, or when it is resent from Pending Retries. + ### Optional data windows - **Each optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. @@ -184,13 +189,14 @@ There are eighteen *categories*: sixteen required and two optional. ### Data that changes shape -- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so a message that failed five times arrives showing one attempt. This applies to every failed message, whatever its status. +- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so it keeps the newest attempt and the number of attempts. A message that failed five times still shows five attempts and still shows as a repeated failure, but only the newest attempt keeps its headers and exception. This applies to every failed message, whatever its status. - **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. - **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. - The copy logs a warning naming both endpoints and the one it kept. - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. - **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. The dry run counts both kinds of merge before anything moves. @@ -202,27 +208,50 @@ Every skipped row is counted under a reason, and its id is written to the log. > [!IMPORTANT] > The log is the only record of each skipped row. Look them up in RavenDB while you still have it. -- *Fault* skips mean something went wrong. -- *Harmless* skips are rows SQL would have removed anyway. +A skip is one of three kinds: -| Reason | Kind | Can a retry fix it? | -| --- | --- | --- | -| A message body that cannot be read after three tries, or that RavenDB does not hold at all. The whole message is skipped, because a message with no body is worse than none | Fault | Yes, once the cause is fixed | -| An error from the SQL target | Fault | Yes, once the cause is fixed | -| A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | Fault | No | -| A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | Fault | No | -| A subscription whose message type or address is over 200 characters, the SQL key limit | Fault | No | -| A row missing a value SQL requires, such as a known endpoint with no name | Fault | No | -| A blank group comment, which SQL never stores | Harmless | Not needed | -| An archived or resolved message, or event log item, already past its retention period | Harmless | Not needed | +- **Fault, a retry can fix it.** Something outside the data went wrong. Fix the cause and retry. +- **Fault, no retry can fix it.** The RavenDB document itself cannot become a SQL row. A retry reads the same document and skips it again, so only [abandoning](#retry-and-abandon) gets past it. It is a fault because the row is lost, not because anything is broken. +- **Harmless.** A row SQL would have removed anyway. + +A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Data that changes shape](#data-that-changes-shape). + +**Fault skips a retry can fix:** + +| Reason | +| --- | +| A message body that cannot be read after three tries. The whole message is skipped, because a message with no body is worse than none | +| An error from the SQL target | + +**Fault skips no retry can fix:** + +| Reason | +| --- | +| A message body that RavenDB does not hold at all, counted as `BodyMissing`. The whole message is skipped, for the same reason. The one exception is a message that expired during the copy (see below), which a retry does get past | +| A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | +| A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | +| A subscription whose message type or address is over 200 characters, the SQL key limit | +| A row missing a value SQL requires, such as a known endpoint with no name, or a failed message with no failing endpoint address to retry to | +| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters. Cutting it would change a key or send retries to the wrong address, so the row is skipped. The skip names the column and the length, counted the way the database counts (PostgreSQL by characters, SQL Server by UTF-16 units) | +| A value of the wrong type, such as a `TimeSent` that is not a date. On PostgreSQL, also a text value holding the character U+0000, which PostgreSQL cannot store | + +**Harmless skips:** + +| Reason | +| --- | +| A blank group comment, which SQL never stores | +| An archived or resolved message, or event log item, already past its retention period | - **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. +- **A malformed document is skipped and counted, not a reason to stop the copy on its own.** The rest of the category carries on, so abandoning a category that read to the end and ended Failed this way gives up only those rows. But if the skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then also gives up every row it had not yet read. In that case raise the threshold or fix the cause, and retry. - **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them shortly after it opens, by the same rule it always applies, so the migration does not repeat that rule. - **Rows outside a window are not skips.** They are simply not part of the copy. - **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. - The exception: a message that failed again after being archived or resolved on version 6.18 or earlier kept its old expiry, and upgrading does not remove it. So a few unresolved messages can still expire during the copy. + - A failed message that expires after the copy reads it, but before the copy reads its body, is a skip. It counts as a body RavenDB does not hold, so it is a fault skip and the category ends Failed. For unresolved messages this happens only with a source last written by version 6.18 or earlier. A retry gets past it, because the expired message is no longer there to read. + - If the copy stops, and the message it stopped at expires before the next start, that start cannot carry on. The category stops with an error saying RavenDB holds no document with that id. If RavenDB is the right database, `--migration-retry` gets past it, because a retry reads the category from the start. - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. ### Startup checks @@ -269,13 +298,14 @@ flowchart LR ### Writing to SQL -- **A failed message is written whole**, with its status unchanged. +- **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, as described in [Data that changes shape](#data-that-changes-shape). +- **A failed message keeps its failure groups and its last-modified time.** `LastModified` is RavenDB's own `@last-modified`, copied as it is, so failure groups, and lists filtered by when a message last changed, show the same messages as before. - **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. - **`StatusChangedAt` is rebuilt.** - - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. + - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. `@expires` is the status-change time plus `ServiceControl/ErrorRetentionPeriod`, so the copy subtracts the current value. If you changed that setting since RavenDB stamped a message, its rebuilt time is off by the difference. It still ages out of SQL when RavenDB would have deleted it, because SQL adds the same current value back when it checks the row against the retention period. - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. - **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. -- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier applies the inline threshold itself, because that threshold lives on the ingestion path. It defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. +- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier decides inline or external storage with the same rule ingestion uses, because `IBodyStoragePersistence` does not apply it. The size limit for inline storage defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. - **Throughput rows are written directly, not through the collector, and each day's count is set, not added.** - Throughput is required, so it copies while ServiceControl is closed, before any collector has written to SQL. - Setting makes the category safe to resume after a crash, where adding would double-count. The collector's own path does add (`LicensingDataStore.cs:198`). @@ -372,7 +402,7 @@ Stop ServiceControl, run one of these, then start it again. In a container, run - **A required category can be abandoned once it is Failed or has started copying.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone before it starts, point back or start over: ServiceControl has not opened, so nothing being served is lost. - **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. - **Abandoning cascades to categories whose rows hang off it**, and the command names each: throughput history (it points at licensing endpoint records) and endpoint settings (it belongs to known endpoints). Group comments only waits for unresolved failed messages, so it is not abandoned with them. -- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a missing required value. The refusal and the dry run say which, so abandon after the first failure rather than retrying. +- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a value too long for its column, a value of the wrong type, a missing required value, a body RavenDB does not hold. The refusal and the dry run say which, so abandon after the first failure rather than retrying. The exception is a category that stopped early at the threshold: abandoning it also gives up every row it had not yet read, so raise the threshold and retry first. > [!IMPORTANT] > Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. @@ -422,11 +452,11 @@ The dry run never writes to RavenDB, and can run again later against whatever is - Whether the source is embedded or external, and which server. - Both RavenDB database names, and the setting each came from. - Rows and body volume per category, counting only what is inside each window. -- **A range for how long ServiceControl will be closed. The range is a floor, not a promise:** it times counting the rows and sizing the bodies, not reading every row and body or writing to SQL, so the required copy can take longer. +- An estimate of how long ServiceControl will be closed. - **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. - The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. - How many failed error imports there are, with advice to run `--import-failed-errors` on RavenDB first. They are copied either way, but ones imported first arrive as ordinary failed messages. -- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure, unless that failure stopped it early at the threshold. It says nothing about load on the source. @@ -493,4 +523,4 @@ It says nothing about load on the source. ## Further reading - [The instructions](ravendb-to-sql-migration-instructions.md): how an operator runs the migration. -- [How the migration is put together](ravendb-to-sql-migration-system-design.md): which class does what, and what is built so far. +- [How the migration is put together](ravendb-to-sql-migration-system-design.md): which class does what. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index 2e8d087297..2aa0b511c3 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -79,6 +79,14 @@ The ingestion gate sees categories the copy never reached because `MigrationStar ## The required copy +Unresolved and retry-issued failed messages copy while the host is closed because the copy cannot merge with live writes. `FailedMessagesWriterBase` inserts only messages the target lacks and never updates one it holds. From the moment the host opens, ingestion writes to the same rows: + +- **A retry confirmation finds no row.** `ResolveRetriedMessages` updates only rows already in SQL, and `FailedMessageBatchWriter.ResolveRetried` deletes the retry claim either way. The copy then inserts the message as still failing, though its retry succeeded. +- **A message that fails again loses its history.** Ingestion upserts a row holding only the new attempt. The copy then finds the id present and drops the RavenDB row, with its earlier attempts, first failure time and retry-issued status. +- **Group actions see part of the list.** Retrying or archiving a group in ServicePulse acts only on the messages copied so far. + +Archived and resolved messages are safe to copy after the host opens. If one of them fails again, the live unresolved row should win, and an insert-only copy gives exactly that. Copying unresolved messages in the background would need a writer that merges by attempt time, and retry confirmations held until their row arrives. Both change live ingestion. + ```mermaid sequenceDiagram participant Run as RunCommand @@ -164,15 +172,15 @@ Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader | Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | | Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | | Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | -| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; single-document readers load by id; licensing settings readers use `LicensingSettingsSource` | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen | -| Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyUnreadable` after one attempt, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; single-document readers load by id; licensing settings readers use `LicensingSettingsSource` | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | +| Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyMissing` after one attempt, a permanent fault no retry fixes, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | | Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold, so a crash leaves orphan bodies, never dangling references | | Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `SetLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips and merges are logged after commit, and `Extend` adds the batch's `PreparedBatch.Merges` to `MergedCount`. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | | Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | | Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | | Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | -Body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageRowMapper.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. +Body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. Invariants enforced across class boundaries: @@ -270,7 +278,7 @@ Every reader derives from `MigrationCategoryReader` and every writer | 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. `SetLicensingEndpointThroughput` sets each day's count rather than incrementing, so re-reads are idempotent | | 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | | 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | -| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageRowMapper` | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator` | | 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | | 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | | | 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | diff --git a/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs new file mode 100644 index 0000000000..b0ed4e2e08 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs @@ -0,0 +1,185 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class FailedMessageBodyMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_small_text_body_stays_inline_and_is_never_written_externally() + { + var failure = new IngestedFailure { ContentType = "text/xml" }; + var content = Encoding.UTF8.GetBytes("1"); + await SeedSourceWithBody(failure.ToFailedMessage(), content, "text/xml"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.EqualTo("1")); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + Assert.That(row.BodyContentType, Is.EqualTo("text/xml")); + Assert.That(RecordedBodies.Written, Is.Empty); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("text/xml")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + [Test] + public async Task A_large_text_body_leaves_a_search_prefix_inline_and_writes_the_whole_body_externally() + { + var failure = new IngestedFailure { ContentType = "application/json" }; + var text = new string('a', EFSettings.BodyStorage.MaxBodySizeToStore * 2); + var content = Encoding.UTF8.GetBytes(text); + await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/json"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Not.Null, "an over-cap text body still keeps a prefix inline so that message search can find it"); + Assert.That(text, Does.StartWith(row.BodyText)); + Assert.That(row.BodyText.Length, Is.EqualTo(EFSettings.BodyStorage.MaxBodySizeToStore)); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + } + + var written = RecordedBodies.Written.Single(); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(written.BodyId, Is.EqualTo(failure.UniqueMessageId.ToString())); + Assert.That(written.Body, Is.EqualTo(content)); + Assert.That(written.ContentType, Is.EqualTo("application/json")); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("application/json")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + [Test] + public async Task A_binary_body_goes_external_with_nothing_left_inline() + { + var failure = new IngestedFailure { ContentType = "application/x-protobuf" }; + var content = new byte[] { 0x08, 0x00, 0x96, 0x01, 0xFF }; + await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/x-protobuf"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + var written = RecordedBodies.Written.Single(); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null, "there is nothing in a binary payload for message search to index"); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + Assert.That(row.BodyContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(written.BodyId, Is.EqualTo(failure.UniqueMessageId.ToString())); + Assert.That(written.Body, Is.EqualTo(content)); + Assert.That(written.ContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + // Covers the missing-attachment path (BodyMissing, one read) only; the retried BodyUnreadable path is covered by the loader unit tests. + [Test] + public async Task A_message_with_no_body_attachment_is_left_out_entirely() + { + var readable = new IngestedFailure(); + var missing = new IngestedFailure(); + + await SeedSourceWithBody(readable.ToFailedMessage(), Encoding.UTF8.GetBytes("1"), "text/xml"); + await SeedSource(($"FailedMessages/{missing.UniqueMessageId}", missing.ToFailedMessage())); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var checkpoint = await ReadCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await FindFailedMessage(readable.UniqueMessageId), Is.Not.Null); + Assert.That(await FindFailedMessage(missing.UniqueMessageId), Is.Null, "a row with no body would dispatch an empty message if anyone retried it"); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Contains.Key(MigrationSkipReason.BodyMissing), "the skip is counted under the reason that tells the operator no retry can fix it"); + Assert.That(checkpoint.SkipReasons, Does.Not.ContainKey(MigrationSkipReason.BodyUnreadable)); + } + } + + [Test] + public async Task A_zero_length_body_is_copied_rather_than_treated_as_missing() + { + var failure = new IngestedFailure(); + await SeedSourceWithBody(failure.ToFailedMessage(), [], "text/plain"); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Skipped, Is.Zero); + Assert.That(row.BodySize, Is.Zero); + Assert.That(row.BodyText, Is.Null, "an empty body is stored nowhere, which is not the same as being stored externally"); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(RecordedBodies.Written, Is.Empty); + } + } + + [Test] + public async Task The_copied_body_reads_back_through_the_products_own_body_storage() + { + var inline = new IngestedFailure { ContentType = "text/xml" }; + var external = new IngestedFailure { ContentType = "application/x-protobuf" }; + var inlineContent = Encoding.UTF8.GetBytes("1"); + var externalContent = new byte[] { 0x08, 0x00, 0x96, 0x01, 0xFF }; + + await SeedSourceWithBodies( + (inline.ToFailedMessage(), inlineContent, "text/xml"), + (external.ToFailedMessage(), externalContent, "application/x-protobuf")); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(await Fetch(inline.UniqueMessageId), Is.EqualTo(inlineContent)); + Assert.That(await Fetch(external.UniqueMessageId), Is.EqualTo(externalContent), "the body was stored under an id TryFetch does not resolve"); + } + + async Task Fetch(Guid uniqueMessageId) + { + var result = await BodyStorage.TryFetch(uniqueMessageId.ToString()); + + Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); + + return await ReadAll(result.Content.Stream); + } + + static async Task ReadAll(Stream stream) + { + await using (stream) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + return buffer.ToArray(); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index adf724538b..6268bbc41a 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -3,6 +3,7 @@ namespace ServiceControl.Migration.AcceptanceTests; using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Net.Http; using System.Net.Http.Json; @@ -22,8 +23,10 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.AcceptanceTesting.InfrastructureConfig; using ServiceControl.Hosting.Commands; using ServiceControl.Infrastructure.WebApi; +using ServiceControl.MessageFailures; using ServiceControl.Migration.Checks; using ServiceControl.Operations; +using ServiceControl.Operations.BodyStorage; using ServiceControl.Persistence; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.Abstractions; @@ -70,6 +73,10 @@ public async Task SetUp() SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", Source.ThroughputDatabase); SetSourceVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); +#if DEBUG + // Cleared because a developer who set it to smoke test the migration would let every test past the release check, which only the test host's own AllowUnreleasedMigration registration may do. + SetSourceVariable(RunCommand.AllowUnreleasedMigrationVariable, null); +#endif // Settings.Port has no public setter, so the port has to come in as the environment variable a customer would set. SetSourceVariable("SERVICECONTROL_PORT", PortUtility.GetAssignedOrAvailablePort(33500).ToString(CultureInfo.InvariantCulture)); @@ -149,6 +156,33 @@ protected async Task SeedSource(string database, params (string Id, object Docum protected Task SeedSource(params (string Id, object Document)[] documents) => SeedSource(Source.PrimaryDatabase, documents); + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => + SeedSourceWithBodies((document, body, contentType)); + + protected async Task SeedSourceWithBodies(params (FailedMessage Document, byte[] Body, string ContentType)[] messages) + { + using (var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase)) + { + foreach (var (document, _, _) in messages) + { + await session.StoreAsync(document, $"FailedMessages/{document.UniqueMessageId}"); + } + + await session.SaveChangesAsync(); + } + + using (var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase)) + { + foreach (var (document, body, contentType) in messages) + { + // "body" is the attachment name the RavenDB persister reads, retyped because this project cannot reference that persister. + session.Advanced.Attachments.Store($"FailedMessages/{document.UniqueMessageId}", "body", new MemoryStream(body), contentType); + } + + await session.SaveChangesAsync(); + } + } + protected Task SeedSourceEndpointSettings(params (string Name, bool TrackInstances)[] settings) => SeedSource([.. settings.Select(setting => ($"EndpointSettings/{DeterministicGuid.MakeId(setting.Name)}", (object)new EndpointSettings { Name = setting.Name, TrackInstances = setting.TrackInstances }))]); @@ -335,6 +369,9 @@ protected static async Task> CollectBatches(IMigrationSourc protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService(); protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService(); + protected IRetryBatchStore RetryBatchStore => targetServices.GetRequiredService(); + protected IRetryStagingStore RetryStagingStore => targetServices.GetRequiredService(); + protected IFailedMessageRetryDataStore FailedMessageRetryDataStore => targetServices.GetRequiredService(); protected async Task CopyCategory(string categoryId, CancellationToken cancellationToken = default) { @@ -402,6 +439,8 @@ protected Task FindFailedMessage(Guid uniqueMessageId) => protected EFPersisterSettings EFSettings => targetServices.GetRequiredService(); + protected IBodyStorage BodyStorage => targetServices.GetRequiredService(); + protected Task Ingest(params IngestedFailure[] failures) => InBatch(async unitOfWork => { diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs new file mode 100644 index 0000000000..c3d218938d --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +// Proves a copy larger than one batch, where one batch is as many rows as one insert statement takes. +[TestFixture] +[NonParallelizable] +class When_a_batch_exceeds_one_statement : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_more_messages_than_fit_in_one_statement() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var batchSize = await Target.BatchSizeFor(category); + var failures = Enumerable.Range(0, (batchSize * 2) + 1).Select(_ => new IngestedFailure()).ToArray(); + + await SeedSourceWithBodies([.. failures.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(result.Copied, Is.EqualTo(failures.Length)); + + foreach (var failure in failures) + { + Assert.That(await FindFailedMessage(failure.UniqueMessageId), Is.Not.Null); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs new file mode 100644 index 0000000000..f86c9ab931 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs @@ -0,0 +1,384 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Raven.Client.Documents.Operations; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure.WebApi; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class When_unresolved_failed_messages_are_migrated : MigrationAcceptanceTest +{ + [Test] + public async Task Fills_every_column_the_newest_attempt_supplies() + { + var failure = new IngestedFailure { FailingEndpointAddress = "Sales@MACHINE" }; + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.MessageId, Is.EqualTo(failure.MessageId)); + Assert.That(row.MessageType, Is.EqualTo(failure.MessageType)); + Assert.That(row.TimeSent, Is.EqualTo(failure.TimeSent)); + Assert.That(row.ConversationId, Is.EqualTo(failure.ConversationId)); + Assert.That(row.IsSystemMessage, Is.False); + Assert.That(row.SendingEndpointName, Is.EqualTo(failure.SendingEndpoint.Name)); + Assert.That(row.SendingEndpointHostId, Is.EqualTo(failure.SendingEndpoint.HostId)); + Assert.That(row.SendingEndpointHost, Is.EqualTo(failure.SendingEndpoint.Host)); + Assert.That(row.ReceivingEndpointName, Is.EqualTo(failure.ReceivingEndpoint.Name)); + Assert.That(row.ReceivingEndpointHostId, Is.EqualTo(failure.ReceivingEndpoint.HostId)); + Assert.That(row.ReceivingEndpointHost, Is.EqualTo(failure.ReceivingEndpoint.Host)); + Assert.That(row.ExceptionType, Is.EqualTo(failure.ExceptionType)); + Assert.That(row.ExceptionMessage, Is.EqualTo(failure.ExceptionMessage)); + Assert.That(row.FailingEndpointAddress, Is.EqualTo("Sales@MACHINE")); + Assert.That(row.HeadersJson, Does.Contain(failure.MessageId)); + } + } + + [Test] + public async Task Collapses_attempt_history_to_the_newest_attempt() + { + var failure = new IngestedFailure(); + await SeedFailure(failure, numberOfAttempts: 5); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(5), "the count survives even though the attempts do not"); + Assert.That(row.LastAttemptedAt, Is.EqualTo(failure.AttemptedAt)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(failure.AttemptedAt.AddMinutes(-4))); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(failure.TimeOfFailure)); + } + } + + [Test] + public async Task A_resumed_copy_leaves_the_rows_the_first_run_wrote() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + var first = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + await QueryTarget(async db => + { + var existing = await db.FailedMessages.SingleAsync(row => row.UniqueMessageId == failure.UniqueMessageId); + existing.ExceptionMessage = "Edited after the first copy, the way a live instance would have"; + await db.SaveChangesAsync(); + return existing; + }); + + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(1)); + Assert.That(row.ExceptionMessage, Is.EqualTo("Edited after the first copy, the way a live instance would have"), "the copier overwrote a row that was already in the target"); + Assert.That(second.Copied, Is.Zero); + Assert.That(second.AlreadyPresent, Is.EqualTo(1), "a row the target already holds is a third outcome"); + Assert.That(second.Skipped, Is.Zero, "counting it as a skip would push a healthy category toward the halt threshold"); + Assert.That(second.SkippedIds, Is.Empty); + } + } + + [Test] + public async Task Counts_only_the_statuses_this_category_owns() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + await SeedFailure(unresolved); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + // Written straight into the target, because this build has no category that copies archived messages. + await QueryTarget(async db => + { + db.FailedMessages.Add(new FailedMessageEntity + { + UniqueMessageId = Guid.NewGuid(), + Status = FailedMessageStatus.Archived, + HeadersJson = "{}", + FailingEndpointAddress = "error" + }); + + return await db.SaveChangesAsync(); + }); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await Target.Count(category), Is.EqualTo(2), "the category's count takes in the archive's rows unless it honours the category's statuses"); + Assert.That(await QueryTarget(db => db.FailedMessages.CountAsync()), Is.EqualTo(3), "the filter is doing the work, not an empty table"); + } + } + + [Test] + public async Task A_copied_message_keeps_its_failure_groups() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var groups = await QueryTarget(db => db.FailedMessageGroups + .AsNoTracking() + .Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId) + .ToListAsync()); + var expected = failure.Groups.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(groups, Has.Count.EqualTo(1), "a message copied without its group rows is missing from every group view"); + Assert.That(groups.Single().GroupId, Is.EqualTo(expected.Id)); + Assert.That(groups.Single().Title, Is.EqualTo(expected.Title)); + Assert.That(groups.Single().Type, Is.EqualTo(expected.Type)); + } + } + + [Test] + public async Task A_retry_by_id_after_the_copy_stages_the_unresolved_message_and_not_the_retry_issued_one() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + await SeedFailure(unresolved); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + + var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(copy.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + + string[] ids = [unresolved.UniqueMessageIdString, retryIssued.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-by-id", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EquivalentTo(new[] { unresolved.UniqueMessageIdString }), + "RavenDB ignores a retry of a message whose first retry is still outstanding, so after the move it must not be sent again"); + } + + [Test] + public async Task A_retry_issued_message_resent_from_pending_retries_after_the_copy_is_staged() + { + var retryIssued = new IngestedFailure(); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + + var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(copy.Copied, Is.EqualTo(1), "the test proves nothing unless the message reached the target"); + + // Pending Retries releases the claim, then sends a retry by id. + await FailedMessageRetryDataStore.RemoveFailedMessageRetry(retryIssued.UniqueMessageIdString); + string[] ids = [retryIssued.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-pending", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EqualTo(ids), + "a migrated claim that the release by message id cannot remove leaves the message unretryable for good"); + } + + [Test] + public async Task A_message_already_in_the_target_gets_no_group_rows_from_the_copy() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + await QueryTarget(db => db.FailedMessageGroups.Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId).ExecuteDeleteAsync()); + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var groups = await QueryTarget(db => db.FailedMessageGroups.AsNoTracking().CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.AlreadyPresent, Is.EqualTo(1)); + Assert.That(groups, Is.Zero, "the copy wrote RavenDB's groups onto a message it did not insert"); + } + } + + [Test] + public async Task Answers_for_a_copied_message_on_the_URL_it_had_on_RavenDB() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await RunHostUntilTheApiAnswers(); + + var response = await HttpClient.GetAsync($"api/errors/last/{failure.UniqueMessageIdString}"); + + Assert.That(response.IsSuccessStatusCode, Is.True, "the bookmark a customer had on RavenDB stopped resolving"); + + var fromApi = await response.Content.ReadFromJsonAsync(SerializerOptions.Default); + + using (Assert.EnterMultipleScope()) + { + Assert.That(fromApi.Id, Is.EqualTo(failure.UniqueMessageIdString)); + Assert.That(fromApi.Status, Is.EqualTo(FailedMessageStatus.Unresolved)); + Assert.That(fromApi.MessageType, Is.EqualTo(failure.MessageType)); + Assert.That(fromApi.ReceivingEndpoint.Name, Is.EqualTo(failure.ReceivingEndpoint.Name)); + Assert.That(fromApi.QueueAddress, Is.EqualTo(failure.QueueAddress)); + Assert.That(fromApi.NumberOfProcessingAttempts, Is.EqualTo(1)); + } + } + + [Test] + public async Task The_required_copy_completes_the_category_before_the_host_opens() + { + var failures = Enumerable.Range(0, 3).Select(_ => new IngestedFailure()).ToArray(); + + foreach (var failure in failures) + { + await SeedFailure(failure); + } + + await RunHostUntilTheApiAnswers(); + + var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SourceTotal, Is.Null, "the engine no longer counts the source before copying"); + } + } + + [Test] + public async Task Records_each_skip_under_the_reason_that_caused_it() + { + var good = new IngestedFailure(); + await SeedFailure(good); + + var unparseable = new IngestedFailure(); + var unparseableDocument = unparseable.ToFailedMessage(); + unparseableDocument.UniqueMessageId = "not-a-guid"; + await SeedSourceWithBody(unparseableDocument, unparseable.Body, unparseable.ContentType); + + var attemptless = new IngestedFailure(); + var attemptlessDocument = attemptless.ToFailedMessage(); + attemptlessDocument.ProcessingAttempts.Clear(); + await SeedSourceWithBody(attemptlessDocument, attemptless.Body, attemptless.ContentType); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var refused = Assert.ThrowsAsync(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + + var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refused.Message, Does.Contain("UnresolvedAndRetryIssuedFailedMessages").And.Contain("--migration-retry").And.Contain("--migration-abandon"), "a fault skip leaves a required category Failed, so the host stays closed and names both commands"); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary + { + [MigrationSkipReason.UnparseableUniqueMessageId] = 1, + [MigrationSkipReason.NoProcessingAttempts] = 1 + })); + } + } + + [Test] + public async Task A_document_whose_TimeSent_cannot_be_read_is_skipped_as_malformed_and_one_whose_host_id_is_not_a_GUID_copies_with_the_empty_host_id() + { + var good = new IngestedFailure(); + await SeedFailure(good); + + var badTimeSent = new IngestedFailure(); + var badTimeSentDocument = badTimeSent.ToFailedMessage(); + badTimeSentDocument.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = "garbage"; + + var badHostId = new IngestedFailure(); + var badHostIdDocument = badHostId.ToFailedMessage(); + var attempt = badHostIdDocument.ProcessingAttempts[^1]; + attempt.Headers[NServiceBus.Headers.HostId] = "not-a-guid"; + attempt.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + await SeedSourceWithBodies( + (badTimeSentDocument, badTimeSent.Body, badTimeSent.ContentType), + (badHostIdDocument, badHostId.Body, badHostId.ContentType)); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the copy reaches the end of the source instead of halting at the first bad document"); + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{badTimeSent.UniqueMessageIdString}" })); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(await QueryTarget(db => db.FailedMessages.Select(row => row.UniqueMessageId).ToListAsync()), Is.EquivalentTo(new[] { good.UniqueMessageId, badHostId.UniqueMessageId })); + var copied = await QueryTarget(db => db.FailedMessages.Where(row => row.UniqueMessageId == badHostId.UniqueMessageId).SingleAsync()); + Assert.That(copied.ReceivingEndpointHostId, Is.EqualTo(Guid.Empty), "ingestion stores an unparseable host id as the empty GUID, so the message copies with it"); + Assert.That(copied.ReceivingEndpointName, Is.EqualTo("Sales"), "the endpoint is rebuilt from the headers, not taken from the stored dictionary"); + Assert.That(copied.ReceivingEndpointHost, Is.EqualTo("MACHINE")); + } + } + + [Test] + public async Task A_failure_stored_without_its_time_is_skipped_when_newest_and_left_out_of_the_failure_times_when_older() + { + var newestHasNoTime = new IngestedFailure(); + var olderHasNoTime = new IngestedFailure(); + await SeedFailure(newestHasNoTime); + await SeedFailure(olderHasNoTime, numberOfAttempts: 2); + await DeleteStoredTimeOfFailure(newestHasNoTime, attempt: 0); + await DeleteStoredTimeOfFailure(olderHasNoTime, attempt: 0); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var copied = await GetFailedMessage(olderHasNoTime.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{newestHasNoTime.UniqueMessageIdString}" }), "a missing time read back as the copy's own clock, so the message copied with a made-up failure time"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 1 })); + Assert.That(copied.FirstTimeOfFailure, Is.EqualTo(olderHasNoTime.TimeOfFailure), "the older attempt has no time, so only the newest one counts"); + Assert.That(copied.LastTimeOfFailure, Is.EqualTo(olderHasNoTime.TimeOfFailure), "the copy's own clock would become the last failure time"); + } + } + + // Removes the key itself, because a stored default date is a different case that the source already reads correctly. + Task DeleteStoredTimeOfFailure(IngestedFailure failure, int attempt) => + SourceStore.Operations.ForDatabase(Source.PrimaryDatabase).SendAsync(new PatchOperation( + $"FailedMessages/{failure.UniqueMessageIdString}", + null, + new PatchRequest { Script = $"delete this.ProcessingAttempts[{attempt}].FailureDetails.TimeOfFailure;" })); + + // Seeded with its body, because the copy leaves out a message whose body the source does not hold. + Task SeedFailure(IngestedFailure failure, FailedMessageStatus status = FailedMessageStatus.Unresolved, int numberOfAttempts = 1) => + SeedSourceWithBody(failure.ToFailedMessage(status, numberOfAttempts), failure.Body, failure.ContentType); +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs index fdac0674c0..20c1542231 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -17,6 +17,12 @@ class PostgreSqlMigrationSqlDialect : PostgreSqlDialect, IMigrationSqlDialect // A fixed chunk, because PostgreSQL's parameter limit is far away and the row width does not bring it closer. public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement; + // A varchar(n) column counts characters, which are code points, so a pair of UTF-16 surrogates is one. + public int TextLength(string value) => value.EnumerateRunes().Count(); + + // PostgreSQL text types cannot hold U+0000 in any encoding, and refuse the statement rather than drop it. + public char? RejectedCharacterIn(string value) => value.Contains('\0') ? '\0' : null; + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class { var insert = MigrationInsert.For(dbContext); @@ -35,6 +41,7 @@ ON CONFLICT ({keyColumns}) DO NOTHING RETURNING {keyColumns} """, chunk, + configureParameter: null, cancellationToken)); } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs index 45d8ea8454..5de111af7b 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; +using System.Data.Common; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -67,6 +68,12 @@ public IEqualityComparer KeyComparer(Type entityType, string propertyNam public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement(parametersPerRow); + // An nvarchar(n) column counts byte-pairs, which are UTF-16 units, so a surrogate pair takes two. + public int TextLength(string value) => value.Length; + + // An nvarchar column stores any UTF-16 unit, U+0000 included. + public char? RejectedCharacterIn(string value) => null; + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class { var insert = MigrationInsert.For(dbContext); @@ -79,10 +86,11 @@ public async Task> InsertMissing(ServiceControlD foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) { // HOLDLOCK, or two writers can both find a key missing and both insert it. + // FORCESEEK, because on a small table SQL Server scans a whole index instead of seeking the primary key, and HOLDLOCK then locks every row the scan reads. inserted.AddRange(await insert.Execute( dbContext, $""" - MERGE {insert.Table} WITH (HOLDLOCK) AS t + MERGE {insert.Table} WITH (HOLDLOCK, FORCESEEK) AS t USING ( SELECT {columns} FROM ( @@ -98,12 +106,23 @@ WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Colum OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"inserted.{column}"))}; """, chunk, + DeclareUnboundedTextAsMax, cancellationToken)); } return inserted; } + // EF declares unbounded text as nvarchar(4000) or nvarchar(max) by each value's length, and each new mix compiles its own plan. + // Size -1 declares every such value as the nvarchar(max) its column already is. + static void DeclareUnboundedTextAsMax(IProperty property, DbParameter parameter) + { + if (property.ClrType == typeof(string) && property.GetMaxLength() is null) + { + parameter.Size = -1; + } + } + OpenedKeys Keys => OpenedState ?? throw new InvalidOperationException($"The SQL Server migration dialect is not open. Call {nameof(Open)} first."); sealed record OpenedKeys( diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs index 5a5c116bdf..025cb1fe22 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs @@ -5,9 +5,9 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; using ServiceControl.Persistence.DataMigration; /// -/// Writes, reads and deletes one probe body, so a body store that is misconfigured stops the copy before it -/// starts. Bodies can live on a file share, in Azure Blob Storage or in S3, and without this the first category -/// that carries bodies would find out part way through and skip every message it could not write. +/// Writes, reads and deletes one probe body, so a misconfigured body store stops the copy before it starts. +/// Bodies can live on a file share, in Azure Blob Storage or in S3, and without this check the first category +/// that carries bodies would only find out part way through, when a body it cannot write fails the batch and stops the category. /// sealed class BodyStorageIsWritableCheck(IBodyStoragePersistence bodyStorage) : IMigrationStartupCheck { diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index 09510292ff..85c4bea2be 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; using System.Collections.Frozen; +using Abstractions; using Implementation; using Infrastructure; using Microsoft.EntityFrameworkCore; @@ -14,12 +15,18 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; /// any of them beyond that, so a category this build cannot write is simply absent from /// and never reaches the engine. /// -sealed class EFCoreMigrationTarget(IServiceScopeFactory scopeFactory, IMigrationSqlDialect migrationDialect, ILogger logger) : DataStoreBase(scopeFactory), IMigrationTarget +sealed class EFCoreMigrationTarget( + IServiceScopeFactory scopeFactory, + IMigrationSqlDialect migrationDialect, + IBodyStoragePersistence storagePersistence, + EFPersisterSettings settings, + ILogger logger) : DataStoreBase(scopeFactory), IMigrationTarget { readonly FrozenDictionary writers = new IMigrationCategoryWriter[] { new KnownEndpointsWriter(migrationDialect), - new EndpointSettingsWriter(migrationDialect) + new EndpointSettingsWriter(migrationDialect), + new FailedMessagesWriter(migrationDialect, storagePersistence, settings) }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs new file mode 100644 index 0000000000..b7360c1725 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +/// +/// What one failed message document became: either a row with its failure group rows, or a skip with the reason. +/// +abstract record FailedMessageDocumentTranslation +{ + FailedMessageDocumentTranslation() + { + } + + /// + /// The document translates to a failed message row and the group rows keyed by it. + /// + public sealed record Translated(FailedMessageEntity Row, IReadOnlyList Groups) : FailedMessageDocumentTranslation; + + /// + /// The document cannot become a row, for the reason given, and says which document and why. + /// + public sealed record Skipped(MigrationSkipReason Reason, string Detail) : FailedMessageDocumentTranslation; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs new file mode 100644 index 0000000000..1e81a7fff2 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs @@ -0,0 +1,202 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text.Json; +using ServiceControl.MessageFailures; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.EntityConfigurations; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// Translates one RavenDB failed message document into a failed message row and its group rows, or a skip with the reason +/// when the document can't satisfy the target columns. +/// +static class FailedMessageDocumentTranslator +{ + public static FailedMessageDocumentTranslation Translate(MigrationRow source) + { + var document = (FailedMessage)source.Document; + + if (!Guid.TryParse(document.UniqueMessageId, out var uniqueMessageId)) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.UnparseableUniqueMessageId, + $"UniqueMessageId '{document.UniqueMessageId}' on document '{source.SourceId}' is not a GUID and the target column is a uniqueidentifier"); + } + + if (document.ProcessingAttempts is not { Count: > 0 }) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.NoProcessingAttempts, + $"Document '{source.SourceId}' has no processing attempts, so no attempt supplies the required columns"); + } + + if (document.ProcessingAttempts[^1] is not { } newest) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' is null, so no attempt supplies the required columns"); + } + + if (MissingFrom(newest) is { } missing) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' has no {missing}, which the row's columns are built from"); + } + + if (MalformedIn(newest.MessageMetadata) is { } malformed) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.MalformedValue, + $"The newest attempt on document '{source.SourceId}' holds a {malformed.Found.FullName} as its {malformed.Key} metadata, where the row needs a {malformed.Expected.FullName}: '{Shown(malformed.Value)}'"); + } + + // An attempt stored without FailureDetails reads back as the default object, whose address is empty. + if (string.IsNullOrEmpty(newest.FailureDetails.AddressOfFailingEndpoint)) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"FailedMessages.{nameof(FailedMessageEntity.FailingEndpointAddress)} is NOT NULL and the newest attempt on document '{source.SourceId}' has no AddressOfFailingEndpoint"); + } + + if (newest.FailureDetails.TimeOfFailure == default || newest.AttemptedAt == default) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' has no {(newest.FailureDetails.TimeOfFailure == default ? nameof(newest.FailureDetails.TimeOfFailure) : nameof(newest.AttemptedAt))}, which the row's failure times are built from"); + } + + var metadata = newest.MessageMetadata; + // The newest attempt passed the checks above, so the list always holds at least one time for Min and Max. + var timesOfFailure = document.ProcessingAttempts + .Where(HasRecordedFailure) + .Select(attempt => attempt.FailureDetails.TimeOfFailure) + .ToList(); + var sendingEndpoint = (EndpointDetails?)metadata.GetValueOrDefault("SendingEndpoint"); + var receivingEndpoint = (EndpointDetails?)metadata.GetValueOrDefault("ReceivingEndpoint"); + + var row = new FailedMessageEntity + { + UniqueMessageId = uniqueMessageId, + Status = document.Status, + StatusChangedAt = ReadResolvedDate(source, MigrationMetadataKeys.StatusChangedAt), + LastModified = ReadResolvedDate(source, MigrationMetadataKeys.LastModified), + NumberOfProcessingAttempts = document.ProcessingAttempts.Count, + FirstTimeOfFailure = AsUtc(timesOfFailure.Min()), + LastTimeOfFailure = AsUtc(timesOfFailure.Max()), + LastAttemptedAt = AsUtc(newest.AttemptedAt), + MessageId = newest.MessageId, + MessageType = LastCharactersThatFit(metadata.GetValueOrDefault("MessageType") as string), + TimeSent = metadata.GetValueOrDefault("TimeSent") is DateTime timeSent ? AsUtc(timeSent) : null, + ConversationId = metadata.GetValueOrDefault("ConversationId") as string, + SendingEndpointName = sendingEndpoint?.Name, + SendingEndpointHostId = sendingEndpoint?.HostId, + SendingEndpointHost = sendingEndpoint?.Host, + ReceivingEndpointName = receivingEndpoint?.Name, + ReceivingEndpointHostId = receivingEndpoint?.HostId, + ReceivingEndpointHost = receivingEndpoint?.Host, + ExceptionType = newest.FailureDetails.Exception?.ExceptionType, + ExceptionMessage = newest.FailureDetails.Exception?.Message, + IsSystemMessage = metadata.GetValueOrDefault("IsSystemMessage") is true, + HeadersJson = MessageHeaders.Write(newest.Headers), + FailingEndpointAddress = newest.FailureDetails.AddressOfFailingEndpoint + }; + + // Null groups and groups with no id are dropped, because GroupId is part of a NOT NULL key. + IReadOnlyList groups = + [ + .. (document.FailureGroups ?? []) + .Where(failureGroup => failureGroup is { Id: not null }) + .Select(failureGroup => new FailedMessageGroupEntity + { + FailedMessageUniqueId = uniqueMessageId, + GroupId = failureGroup.Id, + Title = failureGroup.Title ?? string.Empty, + Type = failureGroup.Type ?? string.Empty + }) + ]; + + return new FailedMessageDocumentTranslation.Translated(row, groups); + } + + /// + /// Fills the four body columns of a mapped row the way ingestion does, letting + /// decide whether the body stays inline or goes to external storage. Returns true when the body goes to external + /// storage. It changes nothing else on the row and writes no body itself. + /// + public static bool SetBody(FailedMessageEntity row, IReadOnlyDictionary headers, MigrationBody body, int maxBodySizeToStore) + { + var (bodyText, storeExternally) = MessageBodyClassifier.Classify(headers, body.Content, maxBodySizeToStore); + + row.BodyText = bodyText; + row.BodyStoredExternally = storeExternally; + row.BodySize = body.Content.Length; + row.BodyContentType = body.ContentType; + + return storeExternally; + } + + static string? MissingFrom(FailedMessage.ProcessingAttempt attempt) => + attempt.FailureDetails is null ? nameof(attempt.FailureDetails) + : attempt.MessageMetadata is null ? nameof(attempt.MessageMetadata) + : attempt.Headers is null ? nameof(attempt.Headers) + : null; + + // An attempt stored without FailureDetails reads back as a default object with an empty address and a time of failure set to when it was read. + static bool HasRecordedFailure(FailedMessage.ProcessingAttempt? attempt) => + attempt is { FailureDetails: { AddressOfFailingEndpoint: { Length: > 0 } } details } && details.TimeOfFailure != default; + + // A missing value becomes null, just as RavenDB's own read path shows it, so only a value of the wrong type needs a skip. + static (string Key, Type Found, Type Expected, object Value)? MalformedIn(Dictionary metadata) + { + foreach (var (key, expected) in TypedMetadata) + { + if (metadata.GetValueOrDefault(key) is { } value && !expected.IsInstanceOfType(value)) + { + return (key, value.GetType(), expected, value); + } + } + + return null; + } + + // Every metadata value the row reads by type, so a value of another type is skipped rather than copied as null or false. + static readonly (string Key, Type Expected)[] TypedMetadata = + [ + ("TimeSent", typeof(DateTime)), + ("MessageType", typeof(string)), + ("ConversationId", typeof(string)), + ("IsSystemMessage", typeof(bool)) + ]; + + const int MaxShownValueLength = 100; + + // Cut short because a stored value can be any length and the skip detail is one log line. + static string Shown(object value) + { + var text = value as string ?? JsonSerializer.Serialize(value); + + return text.Length > MaxShownValueLength ? $"{text[..MaxShownValueLength]}..." : text; + } + + // Throws rather than make up a date, because a made-up StatusChangedAt or LastModified would give the row a retention or modification time it never had. + static DateTime ReadResolvedDate(MigrationRow source, string key) => + source.Metadata.GetValueOrDefault(key) switch + { + DateTime value => AsUtc(value), + null => throw new InvalidOperationException($"Row '{source.SourceId}' arrived without a {key} value, which the source resolves before a failed message crosses the seam"), + var other => throw new InvalidOperationException($"Row '{source.SourceId}' carries a {other.GetType().FullName} as its {key} value, where the source resolves a {typeof(DateTime).FullName} before a failed message crosses the seam") + }; + + // Npgsql only writes UTC to a timestamp with time zone column, and a value whose kind is Unspecified is already UTC, so it must not be shifted. + // Kept apart from the RavenDB source's copy because this project does not reference it. + static DateTime AsUtc(DateTime value) => + value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); + + // Uses the same length limit that ingestion applies to the MessageType column. + static string? LastCharactersThatFit(string? value) => + value is { Length: > ColumnLengths.ShortTextLength } ? value[^ColumnLengths.ShortTextLength..] : value; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs index 3349a45a38..82f6d827c1 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs @@ -9,7 +9,9 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; /// interface IMigrationCategoryWriter { - /// The category this writer handles, named as in . + /// + /// The category this writer handles, named as in . + /// string CategoryId { get; } /// @@ -25,7 +27,8 @@ interface IMigrationCategoryWriter int BatchSize(ServiceControlDbContext dbContext); /// - /// Turns one batch of source documents into the rows to insert and the rows to skip, writing nothing: the insert it returns runs later, inside the target's transaction. + /// Turns one batch of source documents into the rows to insert and the rows to skip, writing nothing to the database: the insert it returns runs later, inside the target's transaction. + /// A writer may store data its rows point at outside the database, such as message bodies, because that data has to exist before the insert commits. /// Every batch row produces exactly one row or exactly one skip, never both and never neither, because the target works out how many were already there by subtracting both from the batch size. /// Task Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs index 4550acc0bb..959c40ff35 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; using DbContexts; +using Infrastructure; using ServiceControl.Persistence.DataMigration; /// @@ -23,12 +24,20 @@ public Task Prepare(ServiceControlDbContext dbContext, MigrationB /// /// Does what promises, for rows already cast. /// - /// The context to read the target through. Nothing may be written through it here. - /// Every row in the batch, in source order, each with its document. - /// Cancels any read of the target. - /// One prepared row or one skip for every entry in . protected abstract Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default); + /// + /// Returns a skip for a row whose entity holds a text value the database would refuse, because one such value fails the whole statement the row goes into, + /// or null when every value can be stored. The reason is for a value longer than its column allows, which is checked first, + /// or for a value holding a character the provider cannot store. + /// + protected static (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableValueSkip(MigrationInsert insert, IMigrationSqlDialect dialect, MigrationRow row, TEntity entity) where TEntity : class => + insert.ValueTooLongIn(entity, dialect) is { } tooLong + ? (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tooLong}, on document '{row.SourceId}'") + : insert.RejectedCharacterIn(entity, dialect) is { } rejected + ? (row.SourceId, MigrationSkipReason.MalformedValue, $"{rejected}, on document '{row.SourceId}'") + : null; + TDocument DocumentOf(MigrationRow row) => row.Document as TDocument ?? throw new InvalidCastException($"The {CategoryId} writer takes {typeof(TDocument).FullName} documents, but row {row.SourceId} holds {row.Document?.GetType().FullName ?? "no document"}. The reader and writer for one category must agree on the document type."); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs index 485518ca67..72db9db495 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -16,21 +16,37 @@ sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : Mig { public override string CategoryId => MigrationCategoryIds.EndpointSettings; - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => dbContext.EndpointSettings.LongCountAsync(cancellationToken); protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) { - List rows = [.. documents.Select(document => new EndpointSettingsEntity { Name = document.Document.Name, TrackInstances = document.Document.TrackInstances })]; - List rowSourceIds = [.. documents.Select(document => document.Row.SourceId)]; + var insert = MigrationInsert.For(dbContext); + List rows = []; + List rowSourceIds = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, settings) in documents) + { + var entity = new EndpointSettingsEntity { Name = settings.Name, TrackInstances = settings.TrackInstances }; + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); + rowSourceIds.Add(row.SourceId); + } // The source yields document-id order, and the statement keeps the first of any keys the database treats as one. return new PreparedBatch( async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, - [], + skips, await MergesIn(dbContext, rows, rowSourceIds, cancellationToken)); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs new file mode 100644 index 0000000000..9fe40c125e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs @@ -0,0 +1,166 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Abstractions; +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes failed messages, their failure group rows and a retry claim for each retry-issued message, inserting only +/// messages the target does not already hold and never updating one it does. Each category that copies failed +/// messages derives from this and names its category id and the statuses it copies. +/// +abstract class FailedMessagesWriterBase(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter +{ + // Like the claim a finished retry batch leaves, this points at no batch, so nothing stages it until it is released by message id. + public static readonly Guid MigratedRetryClaimBatchId = new("5c1b7a0e-2f4d-4e8a-9b36-7d0c4f1e8a52"); + + /// + /// The statuses this category copies. counts only rows with one of these statuses, because + /// both failed message categories share one table. + /// + protected abstract IReadOnlyCollection Statuses { get; } + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.FailedMessages + .Where(message => Statuses.Contains(message.Status)) + .LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, FailedMessage Document)> documents, CancellationToken cancellationToken = default) + { + var messageInsert = MigrationInsert.For(dbContext); + var groupInsert = MigrationInsert.For(dbContext); + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + List translated = []; + HashSet translatedIds = []; + List<(string SourceId, FailedMessageEntity Row, MigrationBody Body)> externalBodies = []; + + foreach (var (row, document) in documents) + { + switch (NamingTheDocument(row, () => FailedMessageDocumentTranslator.Translate(row))) + { + case FailedMessageDocumentTranslation.Translated translation: + var body = row.Body ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} reached the writer without a body. The engine attaches a body to every row of a category that carries bodies, or skips the row."); + // Set before the length check, because the stored content type is a text column with a limit. + var storeExternally = NamingTheDocument(row, () => FailedMessageDocumentTranslator.SetBody(translation.Row, document.ProcessingAttempts[^1].Headers, body, persisterSettings.BodyStorage.MaxBodySizeToStore)); + + // A message copied without one of its groups would be missing from that group, so a group that cannot be stored skips the message. + var unstorable = UnstorableValueSkip(messageInsert, migrationDialect, row, translation.Row) + ?? translation.Groups.Select(group => UnstorableValueSkip(groupInsert, migrationDialect, row, group)).FirstOrDefault(groupSkip => groupSkip is not null); + + if (unstorable is { } unstorableSkip) + { + skips.Add(unstorableSkip); + } + else + { + translated.Add(translation); + + // The insert keeps only the first document with an id, so only its body may be stored under that id. + if (translatedIds.Add(translation.Row.UniqueMessageId) && storeExternally) + { + externalBodies.Add((row.SourceId, translation.Row, body)); + } + } + + break; + case FailedMessageDocumentTranslation.Skipped skip: + skips.Add((row.SourceId, skip.Reason, skip.Detail)); + break; + default: + throw new InvalidOperationException($"The translator returned a translation the writer does not know for document '{row.SourceId}'."); + } + } + + await WriteBodiesTheTargetLacks(dbContext, externalBodies, cancellationToken); + + return new PreparedBatch((context, token) => InsertFailedMessages(context, translated, token), translated.Count, skips, Merges: []); + } + + // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. + async Task WriteBodiesTheTargetLacks(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, FailedMessageEntity Row, MigrationBody Body)> externalBodies, CancellationToken cancellationToken) + { + if (externalBodies.Count == 0) + { + return; + } + + var ids = externalBodies.Select(external => external.Row.UniqueMessageId).ToArray(); + + // This only skips writing bodies for messages the target already holds; InsertMissing still decides which rows go in. + var present = (await dbContext.FailedMessages + .AsNoTracking() + .Where(message => ids.Contains(message.UniqueMessageId)) + .Select(message => message.UniqueMessageId) + .ToListAsync(cancellationToken)).ToHashSet(); + + await Task.WhenAll(externalBodies + .Where(external => !present.Contains(external.Row.UniqueMessageId)) + .Select(external => WriteBody(external, cancellationToken))); + } + + async Task WriteBody((string SourceId, FailedMessageEntity Row, MigrationBody Body) external, CancellationToken cancellationToken) + { + var bodyId = external.Row.UniqueMessageId.ToString(); + + try + { + await storagePersistence.WriteBody(bodyId, external.Body.Content, external.Body.ContentType, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // The storage SDK errors do not name the body, so without this the error that stops the copy would not say which message failed. + catch (Exception ex) + { + throw new InvalidOperationException($"Writing the body of document '{external.SourceId}' in category {CategoryId} to external body storage as '{bodyId}' ({external.Body.Content.Length} bytes) failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } + + // Adds the document id to the error, because the bare error names no document and the source can hold millions. + T NamingTheDocument(MigrationRow row, Func step) + { + try + { + return step(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Preparing document '{row.SourceId}' in category {CategoryId} failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } + + async Task InsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList translated, CancellationToken cancellationToken) + { + var inserted = (await migrationDialect.InsertMissing(dbContext, [.. translated.Select(translation => translation.Row)], cancellationToken)) + .ToHashSet(ReferenceEqualityComparer.Instance); + + // Matched by reference, because InsertMissing returns the row objects it kept, and a document it turned away can share a kept row's id. + // A group goes in only when the message its foreign key points at went in, so a message already in the target keeps the groups it had. + await migrationDialect.InsertMissing(dbContext, [.. translated.Where(translation => inserted.Contains(translation.Row)).SelectMany(translation => translation.Groups)], cancellationToken); + + // A retry-issued message needs a claim, or a retry by id sends it again while its first retry is still outstanding. + await migrationDialect.InsertMissing(dbContext, [.. inserted + .Where(row => row.Status == FailedMessageStatus.RetryIssued) + .Select(row => new FailedMessageRetryEntity { UniqueMessageId = row.UniqueMessageId, RetryBatchId = MigratedRetryClaimBatchId })], cancellationToken); + + return inserted.Count; + } +} + +/// +/// Writes the unresolved and retry-issued failed messages. +/// +sealed class FailedMessagesWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) + : FailedMessagesWriterBase(migrationDialect, storagePersistence, persisterSettings) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; + + protected override IReadOnlyCollection Statuses => MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs index a0ec9ff796..fb7e845a67 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs @@ -14,13 +14,14 @@ sealed class KnownEndpointsWriter(IMigrationSqlDialect migrationDialect) : Migra { public override string CategoryId => MigrationCategoryIds.KnownEndpoints; - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => dbContext.KnownEndpoints.LongCountAsync(cancellationToken); protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, KnownEndpoint Document)> documents, CancellationToken cancellationToken = default) { + var insert = MigrationInsert.For(dbContext); var rows = new List(documents.Count); var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); @@ -33,14 +34,22 @@ protected override Task PrepareDocuments(ServiceControlDbContext continue; } - rows.Add(new KnownEndpointEntity + var entity = new KnownEndpointEntity { Id = endpoint.EndpointDetails.GetDeterministicId(), Name = endpoint.EndpointDetails.Name, HostId = endpoint.EndpointDetails.HostId, Host = endpoint.EndpointDetails.Host, Monitored = endpoint.Monitored - }); + }; + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); } return Task.FromResult(new PreparedBatch( diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs index 22f620a47a..6b3785277d 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using System.Text; +using NServiceBus; using ServiceControl.Persistence.Infrastructure; /// @@ -19,7 +20,8 @@ public static (string? BodyText, bool StoreExternally) Classify(IReadOnlyDiction return (null, false); } - if (headers.IsBinary()) + // A null content type says no more than a missing one, which IsBinary treats as binary, but IsBinary throws on null. + if (headers.GetValueOrDefault(Headers.ContentType) is null || headers.IsBinary()) { return (null, true); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs index 864562c443..67f875e037 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -24,6 +24,17 @@ public interface IMigrationSqlDialect /// int RowsPerStatement(int parametersPerRow); + /// + /// The length of a text value as the database counts it when checking a column's maximum length. + /// + int TextLength(string value); + + /// + /// Finds a character the database refuses to store in any text column, which would fail the whole statement the value goes into. + /// Returns that character, or null when the value has none. + /// + char? RejectedCharacterIn(string value); + /// /// Inserts the rows whose key the table does not hold, merges any whose keys the database treats as one, and returns the rows it inserted. /// diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs index 37ac476786..947e20afec 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; +using System.Data.Common; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -15,29 +16,43 @@ public sealed class MigrationInsert where TEntity : class readonly IReadOnlyList properties; readonly IReadOnlyList keyProperties; + readonly IReadOnlyList<(IProperty Property, string TableAndColumn, int MaxLength)> boundedText; + readonly IReadOnlyList<(IProperty Property, string TableAndColumn)> text; - MigrationInsert(string table, IReadOnlyList properties, IReadOnlyList keyProperties, Func columnName) + MigrationInsert(string tableName, string quotedTable, IReadOnlyList properties, IReadOnlyList keyProperties, Func columnName, Func quote) { - Table = table; + Table = quotedTable; this.properties = properties; this.keyProperties = keyProperties; - Columns = [.. properties.Select(columnName)]; - KeyColumns = [.. keyProperties.Select(columnName)]; + Columns = [.. properties.Select(property => quote(columnName(property)))]; + KeyColumns = [.. keyProperties.Select(property => quote(columnName(property)))]; + text = [.. properties.Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{tableName}.{columnName(property)}"))]; + boundedText = + [ + .. text + .Where(column => column.Property.GetMaxLength() is not null) + .Select(column => (column.Property, column.TableAndColumn, column.Property.GetMaxLength()!.Value)) + ]; } - /// The table name, quoted for this provider and carrying its schema. + /// + /// The table name, quoted for this provider and carrying its schema. + /// public string Table { get; } - /// Every column the insert writes, quoted, in the order the parameters of one row follow. + /// + /// Every column the insert writes, quoted, in the order the parameters of one row follow. + /// public IReadOnlyList Columns { get; } - /// The primary key columns, quoted, in the order the statement has to return them. + /// + /// The primary key columns, quoted, in the order the statement has to return them. + /// public IReadOnlyList KeyColumns { get; } /// /// Reads the table, columns and key out of the EF Core model for this entity. /// - /// The entity is not mapped, has no primary key, or has a column this insert cannot write, such as one the database generates. Rows like that go through EF in the caller's context instead. public static MigrationInsert For(ServiceControlDbContext dbContext) { var entityType = dbContext.Model.FindEntityType(typeof(TEntity)) @@ -59,19 +74,57 @@ public static MigrationInsert For(ServiceControlDbContext dbContext) var sql = dbContext.GetService(); return new MigrationInsert( + tableName, sql.DelimitIdentifier(tableName, entityType.GetSchema()), properties, key.Properties, - property => sql.DelimitIdentifier(property.GetColumnName(storeObject) - ?? throw new InvalidOperationException($"{typeof(TEntity).Name}.{property.Name} has no column in {tableName}."))); + property => property.GetColumnName(storeObject) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name}.{property.Name} has no column in {tableName}."), + sql.DelimitIdentifier); + } + + /// + /// Finds the first text value in a row that is longer than its column allows, which would fail the whole statement the row goes into. + /// Returns a message naming the column and both lengths, or null when every value fits. + /// + public string? ValueTooLongIn(TEntity row, IMigrationSqlDialect dialect) + { + foreach (var (property, tableAndColumn, maxLength) in boundedText) + { + // No provider counts a string as longer than its UTF-16 length, so a value within that length fits and skips the dialect's own count. + if (property.GetGetter().GetClrValue(row) is string { Length: var units } value && units > maxLength && dialect.TextLength(value) is var length && length > maxLength) + { + return $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}"; + } + } + + return null; + } + + /// + /// Finds the first text value in a row holding a character the provider refuses to store, which would fail the whole statement the row goes into. + /// Returns a message naming the column and the character, or null when there is none. + /// + public string? RejectedCharacterIn(TEntity row, IMigrationSqlDialect dialect) + { + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(row) is string value && dialect.RejectedCharacterIn(value) is { } rejected) + { + return $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains"; + } + } + + return null; } /// /// Runs one insert statement over a chunk of rows and returns the rows of that chunk it inserted. /// The statement reads parameters named @p0 upward, row after row in order, and must return the of each row it inserted, in that order. /// It runs on the caller's open transaction, and throws when there is none or when a returned key matches no row of the chunk. + /// Each parameter is built from EF Core's type mapping and then passed, with its property, to when one is given, so a provider can change how it is declared. /// - public async Task> Execute(ServiceControlDbContext dbContext, string sql, TEntity[] chunk, CancellationToken cancellationToken = default) + public async Task> Execute(ServiceControlDbContext dbContext, string sql, TEntity[] chunk, Action? configureParameter, CancellationToken cancellationToken = default) { await using var command = dbContext.Database.GetDbConnection().CreateCommand(); command.Transaction = (dbContext.Database.CurrentTransaction @@ -86,7 +139,9 @@ public async Task> Execute(ServiceControlDbContext dbCont foreach (var property in properties) { // The mapping applies the value converter and the provider type, as EF's own inserts do. - command.Parameters.Add(property.GetRelationalTypeMapping().CreateParameter(command, $"@p{index++}", property.GetGetter().GetClrValue(row), property.IsNullable)); + var parameter = property.GetRelationalTypeMapping().CreateParameter(command, $"@p{index++}", property.GetGetter().GetClrValue(row), property.IsNullable); + configureParameter?.Invoke(property, parameter); + command.Parameters.Add(parameter); } } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs new file mode 100644 index 0000000000..5ebc22df43 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; + +static class MigrationSqlDialectExtensions +{ + /// + /// How many rows fit in one of 's statements, sized from the same column list the statement chunks by. + /// + public static int RowsPerStatementFor(this IMigrationSqlDialect dialect, ServiceControlDbContext dbContext) where TEntity : class => + dialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs new file mode 100644 index 0000000000..67def01ba1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs @@ -0,0 +1,105 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Raven.Client; +using Raven.Client.Documents.Commands; +using ServiceControl.MessageFailures; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// +/// Turns the failed message documents RavenDB streams into migration rows. +/// +class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) +{ + /// + /// Builds the function that turns each streamed failed message into a row. The function returns null for a message whose + /// status is not in , and throws for a status that neither + /// failed message category copies. + /// + public Func, MigrationRow?> Project(IReadOnlyCollection statuses) => + result => + { + var status = result.Document.Status; + + if (!MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Contains(status) && !MigrationCategoryRegistry.ArchivedAndResolvedStatuses.Contains(status)) + { + throw new InvalidOperationException($"The failed message document '{result.Id}' has the status value {(int)status}, which neither failed message category copies, so it would be left behind without a skip to show for it."); + } + + if (!statuses.Contains(status)) + { + return null; + } + + RestoreMetadataTypes(result.Document); + + return new MigrationRow( + result.Id, + result.Document, + new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = StatusChangedAt(result.Document, ReadMetadataDate(result, Constants.Documents.Metadata.Expires), lifecycle.Settings.ErrorRetentionPeriod), + [MigrationMetadataKeys.LastModified] = ReadMetadataDate(result, Constants.Documents.Metadata.LastModified) + }); + }; + + // The translator checks TimeSent, SendingEndpoint and ReceivingEndpoint by exact type, but RavenDB can hand them back as text or untyped objects. + // Only the newest attempt is fixed because the translator reads no other, and a TimeSent that is not a date is left as it is for the translator to skip. + internal static void RestoreMetadataTypes(FailedMessage document) + { + if (document.ProcessingAttempts is not { Count: > 0 } || document.ProcessingAttempts[^1] is not { MessageMetadata: { } metadata } newest) + { + return; + } + + if (metadata.GetValueOrDefault("TimeSent") is string timeSent + && DateTime.TryParse(timeSent, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + metadata["TimeSent"] = parsed; + } + + if (newest.Headers is not { } headers) + { + return; + } + + RestoreEndpoint(metadata, "SendingEndpoint", () => MigrationEndpointDetailsParser.SendingEndpoint(headers)); + RestoreEndpoint(metadata, "ReceivingEndpoint", () => MigrationEndpointDetailsParser.ReceivingEndpoint(headers)); + } + + static void RestoreEndpoint(Dictionary metadata, string key, Func parseHeaders) + { + if (metadata.GetValueOrDefault(key) is null or EndpointDetails) + { + return; + } + + // Storing null is deliberate, because the translator reads it as no endpoint, the same as when ingestion stored none. + metadata[key] = parseHeaders()!; + } + + // For an archived or resolved message RavenDB stores only when it expires, so the time its status + // changed is that expiry less the ErrorRetentionPeriod the source set it with. + internal static DateTime? StatusChangedAt(FailedMessage document, DateTime? expires, TimeSpan sourceErrorRetentionPeriod) + { + ArgumentOutOfRangeException.ThrowIfLessThan(sourceErrorRetentionPeriod, TimeSpan.Zero, "ErrorRetentionPeriod"); + + return document.Status is FailedMessageStatus.Resolved or FailedMessageStatus.Archived && expires is { } deadline + ? AsUtc(deadline) - sourceErrorRetentionPeriod + : document.ProcessingAttempts?.Where(attempt => attempt is not null).MaxBy(attempt => attempt.AttemptedAt) is { } newest ? AsUtc(newest.AttemptedAt) : null; + } + + // RavenDB stores times in UTC, so a time read back with no kind is already UTC and must not be shifted by the local time zone. + static DateTime AsUtc(DateTime value) => + value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); + + static DateTime? ReadMetadataDate(StreamResult result, string key) => + result.Metadata.TryGetValue(key, out var value) && value != null + ? AsUtc(DateTime.Parse(value.ToString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)) + : null; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs index 590e534e18..bd26b24b74 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System; using System.Collections.Generic; using System.Threading; +using Raven.Client.Documents.Commands; using ServiceControl.Persistence.DataMigration; /// @@ -28,13 +29,22 @@ abstract class MigrationCategoryReader(RavenReadOnlySourceLifecycle l /// Streams the primary database's documents whose id starts with , each one whole as a row. /// protected IAsyncEnumerable WholeDocuments(string prefix, string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - RavenDocumentStream.ByPrefix( + Documents(prefix, resumeAfter, batchSize, RavenDocumentStream.WholeDocument, cancellationToken); + + /// + /// Streams the primary database's documents whose id starts with , in id order, through + /// with this category's id and the walk limit every reader shares, + /// . + /// + protected IAsyncEnumerable Documents(string prefix, string? resumeAfter, int batchSize, Func, MigrationRow?> project, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix( Lifecycle, CategoryId, Lifecycle.Settings.DatabaseName, prefix, resumeAfter, batchSize, - RavenDocumentStream.WholeDocument, + RavenDocumentStream.MaxDocumentsWalkedPerBatch, + project, cancellationToken); } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs new file mode 100644 index 0000000000..37a67848cb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs @@ -0,0 +1,95 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using NServiceBus; +using NServiceBus.Faults; +using ServiceControl.Operations; +using ServiceControl.Persistence.Infrastructure; + +/// +/// Rebuilds the sending and receiving endpoint of a failed message from its headers exactly as ingestion would have stored them, +/// so a message ingested with an odd header copies with the same endpoint details it would have had. +/// +static class MigrationEndpointDetailsParser +{ + public static EndpointDetails? SendingEndpoint(IReadOnlyDictionary headers) + { + var endpoint = new EndpointDetails + { + Name = ValueOf(headers, Headers.OriginatingEndpoint) ?? string.Empty, + Host = ValueOf(headers, Headers.OriginatingMachine) ?? string.Empty, + HostId = Guid.TryParse(ValueOf(headers, Headers.OriginatingHostId), out var hostId) ? hostId : Guid.Empty + }; + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + if (ValueOf(headers, Headers.OriginatingAddress) is not { } address) + { + return null; + } + + var (queue, machine) = ExtractQueueAndMachineName(address); + endpoint.Name = queue; + endpoint.Host = machine!; + return endpoint; + } + + public static EndpointDetails? ReceivingEndpoint(IReadOnlyDictionary headers) + { + var endpoint = new EndpointDetails + { + Name = ValueOf(headers, Headers.ProcessingEndpoint) ?? string.Empty, + Host = ValueOf(headers, Headers.HostDisplayName) ?? ValueOf(headers, Headers.ProcessingMachine) ?? string.Empty, + HostId = Guid.TryParse(ValueOf(headers, Headers.HostId), out var hostId) ? hostId : Guid.Empty + }; + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + if (ValueOf(headers, FaultsHeaderKeys.FailedQ) is not { } address) + { + return null; + } + + var (queue, machine) = ExtractQueueAndMachineName(address); + + if (string.IsNullOrEmpty(endpoint.Name)) + { + endpoint.Name = queue; + } + + if (string.IsNullOrEmpty(endpoint.Host)) + { + endpoint.Host = machine ?? "unknown"; + } + + if (string.IsNullOrEmpty(endpoint.Name)) + { + return null; + } + + if (endpoint.HostId == Guid.Empty) + { + endpoint.HostId = DeterministicGuid.MakeId(endpoint.Name, endpoint.Host); + } + + return endpoint; + } + + static string? ValueOf(IReadOnlyDictionary headers, string key) => + headers.TryGetValue(key, out var value) ? value : null; + + static (string Queue, string? Machine) ExtractQueueAndMachineName(string address) + { + var atIndex = address.IndexOf('@', StringComparison.InvariantCulture); + + return atIndex > -1 ? (address[..atIndex], address[(atIndex + 1)..]) : (address, null); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs index 892641c1f5..cfa5726b66 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs @@ -7,23 +7,28 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System.Collections.ObjectModel; using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Tasks; using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Session; using ServiceControl.Persistence.DataMigration; /// -/// The one way every category reads RavenDB: stream the documents whose id starts with a prefix, in id order, -/// and hand them back in batches. Id order is what makes the cursor work, because a resume asks RavenDB to -/// start after an id rather than to skip a count. +/// Reads RavenDB documents for a migration category in id order and hands them back in batches, most often every +/// document whose id starts with a prefix. Id order is what lets a copy resume, because a resume asks RavenDB to +/// start after the last id it copied rather than to skip a count. /// static class RavenDocumentStream { /// - /// Streams one collection and yields it in batches of at most . + /// The most documents one batch reads before it is handed back, even if it holds few rows or none. Every batch is + /// one commit, so a long run of documents that make no row costs one commit per this many documents rather than one + /// per batch size, while the cursor is still saved every few seconds and a resume reads at most this many documents again. + /// + public const int MaxDocumentsWalkedPerBatch = 10_000; + + /// + /// Streams the documents whose id starts with , after , and yields a batch as soon as it holds rows, or once it has read documents, even if it holds few rows or none. A batch never holds more than rows, but when returns null for some documents, one batch can cover more documents than that. If any documents were read after the last such batch, it yields one more batch for them, even an empty one, so the cursor past them is saved. When never returns null and is no more than , every batch but the last holds exactly rows. Throws when the source holds no document with the id in . /// - /// The document id prefix that selects the collection, such as "KnownEndpoints/". - /// The cursor a previous run saved, or null to start at the beginning. - /// Turns one document into a row, or into null for a document this category does not copy. The cursor still moves past a document that becomes null. - /// The source holds no document with the id in , which means the cursor and the database no longer belong together. public static async IAsyncEnumerable ByPrefix( RavenReadOnlySourceLifecycle lifecycle, string categoryId, @@ -31,40 +36,80 @@ public static async IAsyncEnumerable ByPrefix( string prefix, string? resumeAfter, int batchSize, + int maxDocumentsWalkedPerBatch, Func, MigrationRow?> project, [EnumeratorCancellation] CancellationToken cancellationToken = default) where TDocument : class { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxDocumentsWalkedPerBatch); + using var session = lifecycle.OpenSession(databaseName); // RavenDB starts a stream after whatever id it is given, even one that does not exist, so a stale cursor would skip rows nothing has copied. + await RefuseMissingCursor(session, categoryId, databaseName, resumeAfter, cancellationToken); + + await using var enumerator = await session.Advanced.StreamAsync( + prefix, startAfter: resumeAfter, token: cancellationToken); + + await foreach (var batch in Batches(enumerator, resumeAfter, batchSize, maxDocumentsWalkedPerBatch, project)) + { + yield return batch; + } + } + + /// + /// Checks that the cursor names a document the source still holds. The cursor comes from the target's checkpoint, so + /// if the source has no document with that id, either the copy is now reading a different database or the document was + /// deleted since, and resuming after it could skip rows that were never copied. Throws + /// when is not null and the source holds no document with that id. + /// + public static async Task RefuseMissingCursor(IAsyncDocumentSession session, string categoryId, string databaseName, string? resumeAfter, CancellationToken cancellationToken = default) + { if (resumeAfter is not null && !await session.Advanced.ExistsAsync(resumeAfter, cancellationToken)) { throw new InvalidOperationException( - $"The migration cannot resume the '{categoryId}' category after '{resumeAfter}', because the RavenDB database '{databaseName}' holds no document with that id. The cursor is saved in the target database and names a source document, so restoring RavenDB from a backup or changing which database it reads separates the two. Point the migration back at the RavenDB database this copy started from and restart."); + $"The migration cannot resume the '{categoryId}' category after '{resumeAfter}', because the RavenDB database '{databaseName}' no longer holds a document with that id. The likeliest cause is that the document expired and RavenDB deleted it while the copy was interrupted, which can happen to any archived or resolved message. Run --migration-retry {categoryId} to copy the category again from the start. If RavenDB was instead restored from a backup or the migration now reads a different database, point it back at the database this copy started from and restart."); } + } - await using var enumerator = await session.Advanced.StreamAsync( - prefix, startAfter: resumeAfter, token: cancellationToken); - + /// + /// Groups the documents from a RavenDB stream into batches, by the rules describes: + /// a batch goes out once it holds rows or has read + /// documents, and one last batch goes out for any documents read after that, even if it holds no rows. The stream must return + /// documents in id order, or a resume would start in the wrong place. is the id the stream + /// started after, or null. turns a document into a row, or returns null for a document the category + /// does not copy, and the cursor still moves past that document. Each batch's cursor is the id of the last document it read. + /// An empty stream yields nothing. + /// + public static async IAsyncEnumerable Batches( + IAsyncEnumerator> enumerator, + string? resumeAfter, + int batchSize, + int maxDocumentsWalkedPerBatch, + Func, MigrationRow?> project) + { var rows = new List(batchSize); var cursor = resumeAfter; var lastYielded = resumeAfter; + var walkedSinceYield = 0; while (await enumerator.MoveNextAsync()) { // The cursor moves on every document the stream hands back, even one that becomes no row, so a resume never walks it again. cursor = enumerator.Current.Id; + walkedSinceYield++; if (project(enumerator.Current) is { } row) { rows.Add(row); } - if (rows.Count == batchSize) + // Handing back a batch at the document cap saves progress through a long run of documents that make no row. + if (rows.Count == batchSize || walkedSinceYield == maxDocumentsWalkedPerBatch) { yield return new MigrationBatch(rows, cursor); rows = new List(batchSize); lastYielded = cursor; + walkedSinceYield = 0; } } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index 8cb44a7cf7..22eefa3bbf 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -5,12 +5,14 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; +using ServiceControl.Operations.BodyStorage.RavenAttachments; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB.DataMigration.Readers; @@ -24,7 +26,8 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig readonly FrozenDictionary readers = new IMigrationCategoryReader[] { new KnownEndpointsReader(lifecycle), - new EndpointSettingsReader(lifecycle) + new EndpointSettingsReader(lifecycle), + new FailedMessagesReader(lifecycle) }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); @@ -84,8 +87,30 @@ public IAsyncEnumerable Read( ? reader.Read(resumeAfter, batchSize, cancellationToken) : throw new NotSupportedException($"The migration source cannot yet read the '{category.Id}' category."); - public Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) => - throw new NotSupportedException($"The RavenDB migration source cannot read bodies for category {category.Id} yet"); + public async Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + // Another category may keep its body inside the document, where a body attachment read would find nothing and skip every row as BodyMissing. + if (category.Id != MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) + { + throw new NotSupportedException($"The migration source does not read bodies as attachments for the '{category.Id}' category."); + } + + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + var attachment = await session.Advanced.Attachments.GetAsync(sourceId, RavenAttachmentsBodyStorage.AttachmentName, cancellationToken); + + if (attachment == null) + { + return null; + } + + await using var stream = attachment.Stream; + + var content = new MemoryStream((int)attachment.Details.Size); + await stream.CopyToAsync(content, cancellationToken); + + return new MigrationBody(content.ToArray(), attachment.Details.ContentType); + } public IReadOnlyCollection SupportedCategoryIds => readers.Keys; diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs index 7adf8dc8c8..2500437e43 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -9,6 +9,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; +using Newtonsoft.Json; using Particular.LicensingComponent.Contracts; using Raven.Client.Documents; using Raven.Client.Documents.Conventions; @@ -16,7 +17,9 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using Raven.Client.Documents.Session; using Raven.Client.Exceptions.Database; using Raven.Client.Exceptions.Security; +using Raven.Client.Json.Serialization.NewtonsoftJson; using ServiceControl.Configuration; +using ServiceControl.Contracts.Operations; using ServiceControl.RavenDB; /// @@ -142,7 +145,11 @@ IDocumentStore Connect(string serverUrl) { Database = settings.DatabaseName, Urls = [serverUrl], - Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true, + Serialization = new NewtonsoftJsonSerializationConventions { CustomizeJsonDeserializer = serializer => serializer.Converters.Add(new FailureDetailsConverter()) } + } }; if (!settings.UseEmbeddedServer) @@ -237,4 +244,24 @@ public void StopApplication() { } } + + // The FailureDetails constructor sets TimeOfFailure to now. Starting from the default instead lets the translator see a stored attempt that has no time. + sealed class FailureDetailsConverter : JsonConverter + { + public override FailureDetails? ReadJson(JsonReader reader, Type objectType, FailureDetails? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var details = new FailureDetails { AddressOfFailingEndpoint = string.Empty, TimeOfFailure = default }; + serializer.Populate(reader, details); + + return details; + } + + public override void WriteJson(JsonWriter writer, FailureDetails? value, JsonSerializer serializer) => + throw new NotSupportedException("The migration source only reads, so it never writes a FailureDetails."); + } } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs new file mode 100644 index 0000000000..4cc1393ac9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs @@ -0,0 +1,196 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions; +using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the failed messages that are still waiting on somebody: unresolved, or with a retry issued. This category is +/// copied while ServiceControl is closed, and the archived and resolved messages in the same collection can outnumber it +/// many times, so it queries for just these statuses instead of reading them all. +/// +sealed class FailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; + + /// + /// Waits for to catch up, then streams the unresolved and retry issued messages in id + /// order, starting after . When the index is missing, errored or disabled, or does not hold + /// every failed message with a status some category copies, it logs a warning and reads the whole collection instead. + /// Both ways use the document id as the cursor, so a copy can switch from one to the other between runs. + /// Throws when the source holds no document with the id in , + /// or when a failed message has a status no category copies. + /// + public override async IAsyncEnumerable Read(string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var databaseName = Lifecycle.Settings.DatabaseName; + var project = new FailedMessageStream(Lifecycle).Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses); + + var batches = await IndexHoldsEveryCopiedFailedMessage(databaseName, cancellationToken) + ? ThroughIndex(databaseName, resumeAfter, batchSize, project, cancellationToken) + : Documents($"{FailedMessageIdGenerator.CollectionName}/", resumeAfter, batchSize, project, cancellationToken); + + await foreach (var batch in batches) + { + yield return batch; + } + } + + async IAsyncEnumerable ThroughIndex( + string databaseName, + string? resumeAfter, + int batchSize, + Func, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + await RavenDocumentStream.RefuseMissingCursor(session, CategoryId, databaseName, resumeAfter, cancellationToken); + + var query = Query(session).WhereIn(nameof(FailedMessage.Status), StatusValues(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses)); + + if (resumeAfter is not null) + { + query = query.AndAlso().WhereGreaterThan(DocumentId, resumeAfter); + } + + await using var enumerator = await session.Advanced.StreamAsync(query.OrderBy(DocumentId).OfType(), cancellationToken); + + await foreach (var batch in RavenDocumentStream.Batches(enumerator, resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, project)) + { + yield return batch; + } + } + + // A full read passes a document RavenDB could not index to the target to skip, and refuses a status no category copies, + // but a query on the index sees neither, so either kind of document sends the copy back to a full read. + async Task IndexHoldsEveryCopiedFailedMessage(string databaseName, CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + while (true) + { + var index = await WaitForIndex(databaseName, cancellationToken); + + if (index is null || index.State is IndexState.Error or IndexState.Disabled) + { + logger.LogWarning("The RavenDB index {IndexName} in {DatabaseName} is {IndexState}, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer", IndexName, databaseName, index?.State.ToString() ?? "missing", CategoryId); + return false; + } + + // Expiration keeps deleting archived and resolved messages during the copy, so the collection is counted before and + // after the index count, because a deletion in between could hide a document the index left out. A stale index count + // means something changed after the wait, so it waits and counts again. + var storedBefore = await StoredFailedMessages(databaseName, cancellationToken); + int indexed; + QueryStatistics statistics; + try + { + indexed = await Query(session).Statistics(out statistics).WhereIn(nameof(FailedMessage.Status), StatusValues(CopiedStatuses)).CountAsync(cancellationToken); + } + catch (RavenException) + { + // RavenDB can mark the index errored after the wait saw it working, and then refuses the query. The next pass falls back to a full read. + if ((await WaitForIndex(databaseName, cancellationToken))?.State is IndexState.Error or IndexState.Disabled) + { + continue; + } + + throw; + } + + var storedAfter = await StoredFailedMessages(databaseName, cancellationToken); + + if (statistics.IsStale) + { + continue; + } + + if (indexed == storedBefore && indexed == storedAfter) + { + return true; + } + + logger.LogWarning("The RavenDB database {DatabaseName} held {StoredBefore} and then {StoredAfter} failed messages while its index {IndexName} held {Indexed} with a status a category copies, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer. The counts differ when RavenDB could not index a document, which the index errors in RavenDB Studio name, when a document has a status no category copies, or when expiration deleted documents during the count", databaseName, storedBefore, storedAfter, IndexName, indexed, CategoryId); + return false; + } + } + + async Task StoredFailedMessages(string databaseName, CancellationToken cancellationToken) => + (await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken)) + .Collections.GetValueOrDefault(FailedMessageIdGenerator.CollectionName); + + // A query on an index that is behind leaves out what it has not indexed yet, which here means unresolved messages nothing + // would copy. It waits rather than halts, because halting would need an operator to retry for a wait that is normally seconds. + async Task WaitForIndex(string databaseName, CancellationToken cancellationToken) + { + var waiting = Stopwatch.StartNew(); + var nextLog = TimeSpan.Zero; + + while (true) + { + var statistics = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + var index = statistics.Indexes.SingleOrDefault(candidate => candidate.Name == IndexName); + + // A missing, errored or disabled index never catches up, so waiting for it would keep ServiceControl closed for good. + if (index is null || !index.IsStale || index.State is IndexState.Error or IndexState.Disabled) + { + return index; + } + + if (waiting.Elapsed >= nextLog) + { + await LogWait(databaseName, waiting.Elapsed, cancellationToken); + nextLog += LogEvery; + } + + await Task.Delay(PollEvery, cancellationToken); + } + } + + // A paused index stays stale with its state still Normal, so without this the log reads exactly like an index that is merely slow. + async Task LogWait(string databaseName, TimeSpan waited, CancellationToken cancellationToken) + { + var indexing = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexingStatusOperation(), cancellationToken); + var paused = indexing.Status == IndexRunningStatus.Paused + || indexing.Indexes.Any(index => index.Name == IndexName && index.Status == IndexRunningStatus.Paused); + + if (paused) + { + logger.LogWarning("Copying {CategoryId} is waiting for the RavenDB index {IndexName} in {DatabaseName}, but indexing is paused, so ServiceControl stays closed until it is resumed in RavenDB Studio. Waited {Waited} so far", CategoryId, IndexName, databaseName, waited); + return; + } + + logger.LogInformation("Waiting for the RavenDB index {IndexName} in {DatabaseName} to catch up before copying {CategoryId}, waited {Waited} so far", IndexName, databaseName, CategoryId, waited); + } + + static IAsyncDocumentQuery Query(IAsyncDocumentSession session) => + session.Advanced.AsyncDocumentQuery(); + + static object[] StatusValues(IEnumerable statuses) => statuses.Select(status => (object)(int)status).ToArray(); + + static readonly FailedMessageStatus[] CopiedStatuses = [.. MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses, .. MigrationCategoryRegistry.ArchivedAndResolvedStatuses]; + static readonly string IndexName = new FailedMessageViewIndex().IndexName; + static readonly TimeSpan PollEvery = TimeSpan.FromSeconds(1); + static readonly TimeSpan LogEvery = TimeSpan.FromSeconds(30); + static readonly ILogger logger = LoggerUtil.CreateStaticLogger(); + + const string DocumentId = "id()"; +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs new file mode 100644 index 0000000000..07c39bd01b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs @@ -0,0 +1,81 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationValueLengthTests : PersistenceTestBase +{ + static readonly string OutsideTheBasicPlane = char.ConvertFromUtf32(0x1F600); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task A_value_is_measured_in_code_points_as_varchar_counts_them() + { + var fits = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(450), HostId = Guid.NewGuid(), Host = "SALES01" } }; + var tooLong = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(451), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", fits, new Dictionary()), + new MigrationRow("KnownEndpoints/2", tooLong, new Dictionary()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "450 code points fit a varchar(450) column though they are 900 UTF-16 units, and a skip here would be permanent"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2" })); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { Repeat(450) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 451, on document 'KnownEndpoints/2'" + }), "the length reported is the one PostgreSQL counts"); + Assert.That(ServiceProvider.GetRequiredService().TextLength(Repeat(451)), Is.LessThanOrEqualTo(Repeat(451).Length), "the check skips measuring a value whose UTF-16 length fits, which is safe only while no count exceeds it"); + } + } + + [Test] + public async Task A_value_holding_a_NUL_character_is_skipped_as_malformed_naming_the_column() + { + var clean = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SALES01" } }; + var withNul = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SAL\u0000ES" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", clean, new Dictionary()), + new MigrationRow("KnownEndpoints/2", withNul, new Dictionary()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(KnownEndpointEntity.Host))} cannot hold the character U+0000, which the value contains, on document 'KnownEndpoints/2'" + })); + } + } + + static string Repeat(int codePoints) => string.Concat(Enumerable.Repeat(OutsideTheBasicPlane, codePoints)); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs new file mode 100644 index 0000000000..beaee964fe --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class FailedMessageBodySourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory FailedMessages = + MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + [Test] + public async Task Reads_the_attachment_and_its_own_content_type() + { + var content = Encoding.UTF8.GetBytes("1"); + var documentId = await SeedWithAttachment(content, "application/json"); + + await using var source = await OpenMigrationSource(); + var body = await source.ReadBody(FailedMessages, documentId); + + Assert.That(body, Is.Not.Null); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Content.ToArray(), Is.EqualTo(content)); + Assert.That(body.ContentType, Is.EqualTo("application/json"), "the message header says text/xml, the attachment's own type wins"); + } + } + + [Test] + public async Task Reads_a_zero_length_attachment_as_an_empty_body() + { + var documentId = await SeedWithAttachment([], "text/plain"); + + await using var source = await OpenMigrationSource(); + var body = await source.ReadBody(FailedMessages, documentId); + + Assert.That(body, Is.Not.Null, "an empty payload is a body, and is not the same as a missing one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Content.Length, Is.Zero); + Assert.That(body.ContentType, Is.EqualTo("text/plain")); + } + } + + [Test] + public async Task Returns_null_when_the_document_has_no_attachment() + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + + await using var source = await OpenMigrationSource(); + + Assert.That(await source.ReadBody(FailedMessages, message.Id), Is.Null); + } + + [Test] + public async Task Returns_null_for_a_document_that_does_not_exist() + { + await using var source = await OpenMigrationSource(); + + Assert.That(await source.ReadBody(FailedMessages, "FailedMessages/no-such-message"), Is.Null, + "a message RavenDB expired during the copy has to be a BodyMissing skip, not three failed reads and a BodyUnreadable one"); + } + + [Test] + public async Task Refuses_a_category_it_does_not_read_body_attachments_for() + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + var failedErrorImports = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.FailedErrorImports); + + await using var source = await OpenMigrationSource(); + + var exception = Assert.ThrowsAsync(() => source.ReadBody(failedErrorImports, message.Id), + "a failed error import keeps its body inside the document, so reading an attachment would skip every one of them as BodyMissing"); + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.FailedErrorImports)); + } + + async Task SeedWithAttachment(byte[] content, string contentType) + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + + using var session = await SessionProvider.OpenSession(); + session.Advanced.Attachments.Store(message.Id, "body", new MemoryStream(content), contentType); + await session.SaveChangesAsync(); + + return message.Id; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs new file mode 100644 index 0000000000..4aee73b24b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs @@ -0,0 +1,114 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration +{ + using System; + using NUnit.Framework; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.RavenDB.DataMigration; + + [TestFixture] + class FailedMessageStatusChangedAtTests + { + static readonly TimeSpan ErrorRetentionPeriod = TimeSpan.FromDays(10); + + [TestCase(FailedMessageStatus.Resolved)] + [TestCase(FailedMessageStatus.Archived)] + public void Reconstructs_the_status_change_instant_from_the_expiry_stamp(FailedMessageStatus status) + { + var statusChangedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(status), statusChangedAt + ErrorRetentionPeriod, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(statusChangedAt)); + } + + [TestCase(FailedMessageStatus.Unresolved)] + [TestCase(FailedMessageStatus.RetryIssued)] + public void Records_the_newest_attempt_for_a_status_that_never_expires(FailedMessageStatus status) + { + var failure = new IngestedFailure(); + var stamp = new DateTime(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(failure.ToFailedMessage(status), stamp, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(failure.AttemptedAt), "a stamp on a status that never expires is not its status change"); + } + + [Test] + public void Falls_back_to_the_newest_attempt_for_an_archived_message_with_no_stamp() + { + var failure = new IngestedFailure(); + + var resolved = FailedMessageStream.StatusChangedAt(failure.ToFailedMessage(FailedMessageStatus.Archived), null, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(failure.AttemptedAt)); + } + + [Test] + public void Returns_null_for_a_document_with_no_attempts_and_no_stamp() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + Assert.That(FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod), Is.Null); + } + + [Test] + public void Returns_null_for_a_document_whose_attempt_list_is_null_and_has_no_stamp() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + + Assert.That(FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod), Is.Null, "the translator skips a message with no attempts, so this must not throw and halt the category"); + } + + [Test] + public void Reconstructs_an_instant_already_past_the_retention_cutoff_faithfully() + { + var expires = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived), expires, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(expires - ErrorRetentionPeriod), "the value the source implies, not one adjusted to survive the sweep"); + } + + [TestCase(DateTimeKind.Unspecified)] + [TestCase(DateTimeKind.Utc)] + public void Returns_a_UTC_instant_whatever_kind_the_attempt_was_read_with(DateTimeKind kind) + { + var document = new IngestedFailure().ToFailedMessage(); + var attemptedAt = new DateTime(2026, 9, 1, 8, 0, 0, kind); + document.ProcessingAttempts[^1].AttemptedAt = attemptedAt; + + var resolved = FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod); + + using (Assert.EnterMultipleScope()) + { + Assert.That(resolved.Value.Kind, Is.EqualTo(DateTimeKind.Utc), "MigrationMetadataKeys promises a UTC DateTime and DateTime equality ignores Kind"); + Assert.That(resolved.Value.Ticks, Is.EqualTo(attemptedAt.Ticks)); + } + } + + [Test] + public void Reads_an_expiry_stamp_with_no_kind_as_UTC() + { + var statusChangedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Unspecified); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived), statusChangedAt + ErrorRetentionPeriod, ErrorRetentionPeriod); + + using (Assert.EnterMultipleScope()) + { + Assert.That(resolved.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(resolved.Value.Ticks, Is.EqualTo(statusChangedAt.Ticks), "an unspecified instant is already UTC and must not be shifted by the local offset"); + } + } + + [Test] + public void Refuses_a_negative_source_retention_period() + { + var ex = Assert.Throws(() => + FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(), null, TimeSpan.FromDays(-1))); + + Assert.That(ex.Message, Does.Contain("ErrorRetentionPeriod")); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs new file mode 100644 index 0000000000..0cbf6a8e4d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs @@ -0,0 +1,106 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using NServiceBus; +using NServiceBus.Faults; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.RavenDB.DataMigration; +// Aliased because the RavenDB persister's own EndpointDetailsParser would otherwise win from this namespace. +using IngestionParser = Contracts.Operations.EndpointDetailsParser; + +[TestFixture] +class MigrationEndpointDetailsParserParityTests +{ + [TestCaseSource(nameof(HeaderSets))] + public void Rebuilds_both_endpoints_exactly_as_ingestion_does(Dictionary headers) + { + using (Assert.EnterMultipleScope()) + { + Assert.That(Shape(MigrationEndpointDetailsParser.SendingEndpoint(headers)), Is.EqualTo(Shape(IngestionParser.SendingEndpoint(headers))), + "a migrated message whose sending endpoint differs from a newly ingested one is grouped apart from it in ServicePulse"); + Assert.That(Shape(MigrationEndpointDetailsParser.ReceivingEndpoint(headers)), Is.EqualTo(Shape(IngestionParser.ReceivingEndpoint(headers))), + "a migrated message whose receiving endpoint differs from a newly ingested one is grouped apart from it in ServicePulse"); + } + } + + static (string Name, string Host, Guid HostId)? Shape(EndpointDetails endpoint) => + endpoint is null ? null : (endpoint.Name, endpoint.Host, endpoint.HostId); + + static IEnumerable HeaderSets() + { + var sendingHostId = Guid.Parse("6a3a9b1c-2d4e-4f50-8a61-7b8c9d0e1f20").ToString(); + var receivingHostId = Guid.Parse("0f1e2d3c-4b5a-4968-8776-655443322110").ToString(); + + yield return Case("full headers", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = sendingHostId, + [Headers.ProcessingEndpoint] = "Billing", + [Headers.HostDisplayName] = "Billing host", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = receivingHostId + }); + + yield return Case("non-GUID host ids", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = "not-a-guid", + [Headers.ProcessingEndpoint] = "Billing", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = "not-a-guid" + }); + + yield return Case("null-valued host ids", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = null, + [Headers.ProcessingEndpoint] = "Billing", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = null + }); + + yield return Case("null-valued host display name", new() + { + [Headers.ProcessingEndpoint] = "Billing", + [Headers.HostDisplayName] = null, + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = receivingHostId + }); + + yield return Case("addresses with a machine", new() + { + [Headers.OriginatingAddress] = "Sales@SALES-01", + [FaultsHeaderKeys.FailedQ] = "Billing@BILLING-01" + }); + + yield return Case("addresses without a machine", new() + { + [Headers.OriginatingAddress] = "Sales", + [FaultsHeaderKeys.FailedQ] = "Billing" + }); + + yield return Case("failed queue with a non-GUID host id", new() + { + [FaultsHeaderKeys.FailedQ] = "Billing@BILLING-01", + [Headers.HostId] = "not-a-guid" + }); + + yield return Case("null-valued addresses", new() + { + [Headers.OriginatingAddress] = null, + [FaultsHeaderKeys.FailedQ] = null + }); + + yield return Case("no endpoint headers", new() + { + [Headers.MessageId] = "message-1" + }); + } + + static TestCaseData Case(string name, Dictionary headers) => new TestCaseData(headers).SetArgDisplayNames(name); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index c34641d535..e97f52a466 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -70,6 +70,8 @@ public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_inste Assert.That(refusal.Message, Does.Contain(EndpointSettingsCategory.Id)); Assert.That(refusal.Message, Does.Contain("EndpointSettings/zzz-from-another-database")); Assert.That(refusal.Message, Does.Contain(DatabaseName), "the operator has to be told which database the cursor was looked for in"); + Assert.That(refusal.Message, Does.Contain("expired"), "an archived or resolved document can expire during an interrupted copy, on any source"); + Assert.That(refusal.Message, Does.Contain($"--migration-retry {EndpointSettingsCategory.Id}"), "a retry re-reads the category from the start, so it is the way out"); }); } @@ -123,7 +125,8 @@ public async Task Reading_every_category_creates_no_index() static readonly Dictionary> Seeds = new() { [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), - [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }) + [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }), + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()) }; string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs new file mode 100644 index 0000000000..f2b5f8bc11 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs @@ -0,0 +1,901 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Configuration; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration; + +[TestFixture] +class UnresolvedFailedMessageSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_both_unresolved_statuses_and_neither_of_the_others() + { + var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + var retryIssued = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); + await CompleteDatabaseOperation(); + + var rows = await ReadAll(); + + Assert.That( + rows.Select(row => ((FailedMessage)row.Document).UniqueMessageId), + Is.EquivalentTo(new[] { unresolved.UniqueMessageId, retryIssued.UniqueMessageId })); + } + + [Test] + public async Task Counts_only_the_documents_this_category_copies_not_the_whole_collection() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(Category), Is.EqualTo(2), "the archived message shares the collection but belongs to another category"); + } + + [Test] + public async Task Refuses_a_document_whose_status_neither_failed_message_category_copies() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage((FailedMessageStatus)99)); + await CompleteDatabaseOperation(); + + var refusal = Assert.ThrowsAsync(ReadAll); + + Assert.That(refusal.Message, Does.Contain(stored.Id).And.Contain("99"), "a status no category copies would otherwise be left behind with nothing to show for it"); + } + + [Test] + public async Task Hands_over_attempt_times_as_UTC() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(numberOfAttempts: 2)); + await CompleteDatabaseOperation(); + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + Assert.That(attempts.SelectMany(attempt => new[] { attempt.AttemptedAt.Kind, attempt.FailureDetails.TimeOfFailure.Kind }), Is.All.EqualTo(DateTimeKind.Utc)); + } + + [Test] + public async Task Resolves_StatusChangedAt_to_the_newest_attempt_when_it_is_neither_first_nor_last() + { + var failure = new IngestedFailure(); + var message = failure.ToFailedMessage(FailedMessageStatus.Unresolved, numberOfAttempts: 3); + var attempts = message.ProcessingAttempts; + message.ProcessingAttempts = [attempts[1], attempts[2], attempts[0]]; + await SeedFailedMessage(message); + await CompleteDatabaseOperation(); + + var row = (await ReadAll()).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.EqualTo(failure.AttemptedAt)); + Assert.That(((DateTime)row.Metadata[MigrationMetadataKeys.StatusChangedAt]).Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public async Task Sends_only_the_migration_metadata_keys_across_the_seam() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata.Keys, Is.EquivalentTo(new[] { MigrationMetadataKeys.StatusChangedAt, MigrationMetadataKeys.LastModified }), "a store's own metadata, such as @expires, never travels"); + } + + [Test] + public async Task Carries_the_documents_own_last_modified() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + DateTime lastModified; + + using (var session = await SessionProvider.OpenSession()) + { + lastModified = session.Advanced.GetLastModifiedFor(await session.LoadAsync(stored.Id)).Value; + } + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.LastModified], Is.EqualTo(lastModified.ToUniversalTime()), "the value RavenDB shows as the message's LastModified, not the copy time"); + } + + [Test] + public async Task A_run_of_dropped_documents_yields_an_empty_batch_only_once_the_walk_limit_is_reached() + { + var ids = await SeedInIdOrder(Enumerable.Repeat(FailedMessageStatus.Archived, 7).ToArray()); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 3); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[2], ids[5], ids[6] }), "an empty stretch commits once per walk limit, plus the tail so the cursor past it is saved"); + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + } + } + + [Test] + public async Task The_category_reads_no_archived_or_resolved_document() + { + await SeedInIdOrder(FailedMessageStatus.Archived, FailedMessageStatus.Resolved, FailedMessageStatus.Archived); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category, batchSize: 2); + + Assert.That(batches, Is.Empty, "the copy runs while ServiceControl is closed, so reading the archive only to drop it makes the outage grow with the archive"); + } + + [Test] + public async Task Resuming_after_a_cursor_reads_only_the_unresolved_messages_after_it() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.RetryIssued, + FailedMessageStatus.Unresolved, FailedMessageStatus.Resolved, FailedMessageStatus.Unresolved); + + await using var source = await OpenMigrationSource(); + var first = (await CollectBatches(source, Category, batchSize: 2))[0]; + var resumed = await CollectBatches(source, Category, resumeAfter: first.Cursor, batchSize: 2); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { ids[0], ids[2] })); + Assert.That(first.Cursor, Is.EqualTo(ids[2])); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[3], ids[5] }), "a resume neither repeats a row nor skips one"); + } + } + + [Test] + public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_instead_of_starting_past_it() + { + await SeedInIdOrder(FailedMessageStatus.Unresolved, FailedMessageStatus.Unresolved); + const string cursor = "FailedMessages/zzz-from-another-database"; + + await using var source = await OpenMigrationSource(); + var refusal = Assert.ThrowsAsync(() => CollectBatches(source, Category, resumeAfter: cursor)); + + Assert.That(refusal.Message, Does.Contain(Category.Id).And.Contain(cursor), "a cursor from a restored backup or another database would otherwise resume part way through and skip rows"); + } + + [Test] + public async Task Waits_for_the_index_to_catch_up_so_no_unresolved_message_is_missed() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await CompleteDatabaseOperation(); + await DocumentStore.Maintenance.SendAsync(new StopIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + + // The stopped index still counts the message as archived, so its count matches the collection, and only the wait for the index keeps the read from finding nothing. + using (var session = DocumentStore.OpenAsyncSession()) + { + (await session.LoadAsync(stored.Id, TestContext.CurrentContext.CancellationToken)).Status = FailedMessageStatus.Unresolved; + await session.SaveChangesAsync(TestContext.CurrentContext.CancellationToken); + } + + var reading = ReadAll(); + var finishedWhileStale = await Task.WhenAny(reading, Task.Delay(TimeSpan.FromSeconds(3), TestContext.CurrentContext.CancellationToken)) == reading; + + await DocumentStore.Maintenance.SendAsync(new StartIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var rows = await reading.WaitAsync(TimeSpan.FromSeconds(60), TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finishedWhileStale, Is.False, "a query on an index that has not caught up leaves out the messages it has not indexed yet"); + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id })); + } + } + + [Test] + public async Task Resumes_after_a_cursor_that_names_an_archived_message() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved); + + await using var source = await OpenMigrationSource(); + var resumed = await CollectBatches(source, Category, resumeAfter: ids[1]); + + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[2], ids[4] }), "a run that read the whole collection can save an archived message as its cursor, and the next run may query the index"); + } + + [Test] + public async Task Reads_through_a_Corax_index_as_through_a_Lucene_one() + { + await UseCorax(); + var ids = await SeedInIdOrder(FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.RetryIssued, FailedMessageStatus.Resolved); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category); + + Assert.That(batches.Select(batch => (batch.Cursor, batch.Rows.Select(row => row.SourceId))), Is.EqualTo(new[] { (ids[2], new[] { ids[0], ids[2] }) }), "a full read would walk on to the resolved message and save its cursor past it"); + } + + [Test] + public async Task Resumes_through_a_Corax_index_with_no_row_repeated_or_skipped() + { + await UseCorax(); + var statuses = Enumerable.Range(0, 60).Select(position => (FailedMessageStatus)((position % 4) + 1)).ToArray(); + var ids = await SeedInIdOrder(statuses); + var expected = ids.Where((_, position) => statuses[position] is FailedMessageStatus.Unresolved or FailedMessageStatus.RetryIssued).ToList(); + + await using var source = await OpenMigrationSource(); + var read = new List(); + string cursor = null; + + // Every batch is read by a fresh call, as a restart after each commit would. + while ((await CollectBatches(source, Category, resumeAfter: cursor, batchSize: 7)).FirstOrDefault() is { } batch) + { + Assert.That(batch.Cursor, Is.Not.EqualTo(cursor), "a resume that hands back the batch it resumed after never ends"); + read.AddRange(batch.Rows.Select(row => row.SourceId)); + cursor = batch.Cursor; + } + + var afterArchived = await CollectBatches(source, Category, resumeAfter: ids[Array.IndexOf(statuses, FailedMessageStatus.Archived)]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(read, Is.EqualTo(expected), "Corax has to order and compare id() as the cursor expects"); + Assert.That(afterArchived.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(expected.SkipWhile(id => string.CompareOrdinal(id, ids[Array.IndexOf(statuses, FailedMessageStatus.Archived)]) < 0))); + } + } + + [Test] + public async Task Reads_the_whole_collection_when_the_index_is_missing() + { + await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); + } + + [Test] + public async Task Reads_the_whole_collection_instead_of_waiting_for_a_disabled_index() + { + await DocumentStore.Maintenance.SendAsync(new DisableIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "a disabled index never catches up, so waiting for it would keep ServiceControl closed for good"); + } + + [Test] + public async Task Reads_the_whole_collection_instead_of_waiting_for_an_errored_index() + { + var unindexable = await SeedFailedMessage(FailedMessageWithATimeSentThatIsNotADate()); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + Assert.That(await IndexStateNow(), Is.EqualTo(IndexState.Error), "one document that fails out of one attempt errors the index"); + + // An errored index stops part way, so this one stays unindexed and the index stays stale. + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { unindexable.Id, stored.Id }), "an errored index never catches up, so waiting for it would keep ServiceControl closed for good"); + } + + [Test] + public async Task Reads_the_whole_collection_when_the_index_leaves_out_a_document_it_could_not_index() + { + await using (var bulkInsert = DocumentStore.BulkInsert(token: TestContext.CurrentContext.CancellationToken)) + { + foreach (var message in Enumerable.Range(0, 200).Select(_ => new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived))) + { + await bulkInsert.StoreAsync(message, FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId)); + } + } + + var unindexable = await SeedFailedMessage(FailedMessageWithATimeSentThatIsNotADate()); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + Assert.That(await IndexStateNow(), Is.EqualTo(IndexState.Normal), "RavenDB errors the whole index only above a 15 percent failure rate, so one bad document among many leaves it working and silently short"); + + var rows = await ReadAll(); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { unindexable.Id }), "a query would leave out the document the index could not map, where the target should record it as a skip"); + } + + [Test] + public async Task A_batch_yields_as_soon_as_it_holds_batch_size_rows_however_far_below_the_walk_limit() + { + var ids = await SeedInIdOrder(Enumerable.Repeat(FailedMessageStatus.Unresolved, 5).ToArray()); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 3); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[1], ids[3], ids[4] })); + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 2, 1 }), "a batch never holds more than the target can write in one go"); + } + } + + [Test] + public async Task A_batch_short_of_batch_size_rows_yields_once_the_walk_limit_is_reached() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Archived, FailedMessageStatus.Archived, + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Archived); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 4); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[3], ids[6] }), "a sparse stretch still saves its cursor every walk limit, so a resume walks at most that far again"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => row.SourceId)), Is.EqualTo(new[] { new[] { ids[0] }, new[] { ids[4] } })); + } + } + + [Test] + public async Task A_full_batch_yields_on_rows_held_not_on_documents_walked() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 10); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[2], ids[4] }), "a batch is full when it holds batch size rows, however many documents it walked to find them"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => row.SourceId)), Is.EqualTo(new[] { new[] { ids[0], ids[2] }, new[] { ids[4] } })); + } + } + + [TestCase(0)] + [TestCase(-1)] + public void Refuses_a_walk_limit_that_is_not_positive(int maxDocumentsWalkedPerBatch) + { + var refusal = Assert.ThrowsAsync(() => StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch)); + + Assert.That(refusal.ParamName, Is.EqualTo("maxDocumentsWalkedPerBatch"), "a limit of zero or less is never reached, so every batch would wait for batch size rows however far it walked"); + } + + [Test] + public async Task The_category_saves_its_cursor_at_least_every_ten_thousand_documents_read() + { + var messages = Enumerable.Range(0, 10_001) + .Select(_ => new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)) + .OrderBy(message => FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId), StringComparer.Ordinal) + .ToList(); + + await using (var bulkInsert = DocumentStore.BulkInsert(token: TestContext.CurrentContext.CancellationToken)) + { + foreach (var message in messages) + { + await bulkInsert.StoreAsync(message, FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId)); + } + } + + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category, batchSize: 20_000); + + Assert.That( + batches.Select(batch => (batch.Cursor, batch.Rows.Count)), + Is.EqualTo(new[] { (FailedMessageIdGenerator.MakeDocumentId(messages[9_999].UniqueMessageId), 10_000), (FailedMessageIdGenerator.MakeDocumentId(messages[10_000].UniqueMessageId), 1) }), + "the reader commits every RavenDocumentStream.MaxDocumentsWalkedPerBatch documents, which is 10,000, however large the batch size"); + } + + [Test] + public async Task A_document_stored_without_FailureDetails_comes_back_with_an_empty_failing_address() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(numberOfAttempts: 2)); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "delete this.ProcessingAttempts[0].FailureDetails; delete this.ProcessingAttempts[1].FailureDetails;" })); + await CompleteDatabaseOperation(); + var readStartedAt = DateTime.UtcNow; + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempts.Select(attempt => attempt.FailureDetails?.AddressOfFailingEndpoint), Is.All.Empty, "a missing key reads back as the default object, not null, so the translator has to treat an empty address as missing"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.TimeOfFailure), Is.All.GreaterThanOrEqualTo(readStartedAt), "the default object stamps the time it was read, so its time of failure is invented"); + } + } + + [Test] + public async Task A_FailureDetails_stored_without_its_TimeOfFailure_comes_back_with_no_time_rather_than_the_time_it_was_read() + { + var failure = new IngestedFailure { FailingEndpointAddress = "Sales@MACHINE" }; + var stored = await SeedFailedMessage(failure.ToFailedMessage(numberOfAttempts: 2)); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "delete this.ProcessingAttempts[0].FailureDetails.TimeOfFailure; delete this.ProcessingAttempts[1].FailureDetails.TimeOfFailure;" })); + await CompleteDatabaseOperation(); + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempts[^1].FailureDetails.TimeOfFailure, Is.EqualTo(default(DateTime)), "the translator skips a newest attempt with no time of failure, and the read time would copy it with a made-up one"); + Assert.That(attempts[0].FailureDetails.TimeOfFailure, Is.EqualTo(default(DateTime)), "the translator leaves an older attempt with no time out of the first and last failure times"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.AddressOfFailingEndpoint), Is.All.EqualTo("Sales@MACHINE"), "the rest of FailureDetails is still read as stored"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.Exception?.Message), Is.All.EqualTo(failure.ExceptionMessage)); + } + } + + [Test] + public async Task A_FailureDetails_stored_as_null_comes_back_as_null() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "this.ProcessingAttempts[0].FailureDetails = null;" })); + await CompleteDatabaseOperation(); + + var attempt = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single(); + + Assert.That(attempt.FailureDetails, Is.Null, "the translator skips an attempt with no FailureDetails, so one bad document must not halt the category"); + } + + [TestCase("SendingEndpoint", NServiceBus.Headers.OriginatingHostId)] + [TestCase("ReceivingEndpoint", NServiceBus.Headers.HostId)] + public void RestoreMetadataTypes_gives_an_endpoint_the_empty_host_id_when_its_host_id_header_is_null(string key, string hostIdHeader) + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[hostIdHeader] = null; + attempt.MessageMetadata[key] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata[key], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty), "a header present with no value reads as no host id, as ingestion stores it, so the message copies"); + } + + [Test] + public void RestoreMetadataTypes_leaves_a_document_whose_attempt_list_is_null_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + } + + [Test] + public async Task Reads_a_document_whose_attempt_list_is_null_with_no_StatusChangedAt() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.Null, "the translator skips a message with no attempts, so the source passes it through rather than halting the category"); + } + + [Test] + public async Task Hands_over_the_metadata_ingestion_reads_as_the_types_it_reads() + { + var failure = new IngestedFailure { IsSystemMessage = true }; + await SeedFailedMessage(failure.ToFailedMessage()); + await CompleteDatabaseOperation(); + + var metadata = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single().MessageMetadata; + + using (Assert.EnterMultipleScope()) + { + Assert.That(metadata["TimeSent"], Is.TypeOf().And.EqualTo(failure.TimeSent)); + Assert.That(((DateTime)metadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(metadata["SendingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(failure.SendingEndpoint.HostId)); + Assert.That(metadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(failure.ReceivingEndpoint.HostId)); + Assert.That(metadata["MessageType"], Is.TypeOf().And.EqualTo(failure.MessageType)); + Assert.That(metadata["ConversationId"], Is.TypeOf().And.EqualTo(failure.ConversationId)); + Assert.That(metadata["IsSystemMessage"], Is.TypeOf().And.True, "the translator reads it with 'is true', so any other type reads as false"); + } + } + + [Test] + public async Task Restores_a_TimeSent_stored_as_text_and_an_endpoint_stored_without_its_type() + { + var document = new IngestedFailure().ToFailedMessage(); + var stored = document.ProcessingAttempts.Single(); + stored.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + stored.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00.0000000Z"; + stored.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + await SeedFailedMessage(document); + await CompleteDatabaseOperation(); + + var attempt = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempt.MessageMetadata["TimeSent"], Is.TypeOf().And.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc))); + Assert.That(((DateTime)attempt.MessageMetadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.Name)).EqualTo("Sales"), "rebuilt from the headers, not converted from the stored dictionary"); + Assert.That(((EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]).Host, Is.EqualTo("MACHINE")); + } + } + + [Test] + public async Task Passes_a_TimeSent_that_is_not_a_date_through_as_stored_and_gives_endpoints_whose_host_id_is_not_a_GUID_the_empty_host_id() + { + var document = SenderAndReceiverHeadersFailure(); + var stored = document.ProcessingAttempts.Single(); + stored.Headers[NServiceBus.Headers.HostId] = "not-a-guid"; + stored.Headers[NServiceBus.Headers.OriginatingHostId] = "not-a-guid-either"; + stored.MessageMetadata["TimeSent"] = "garbage"; + stored.MessageMetadata["SendingEndpoint"] = new Dictionary { ["Name"] = "Stored sender" }; + stored.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored receiver" }; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var metadata = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single().MessageMetadata; + + using (Assert.EnterMultipleScope()) + { + Assert.That(metadata["TimeSent"], Is.EqualTo("garbage"), "the target skips the document as malformed, so one bad document does not halt the category"); + Assert.That(metadata["SendingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty)); + Assert.That(metadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty)); + } + } + + [Test] + public void RestoreMetadataTypes_parses_text_as_UTC_and_rebuilds_untyped_endpoints_from_the_headers() + { + var document = new IngestedFailure().ToFailedMessage(numberOfAttempts: 2); + var sendingHostId = Guid.NewGuid(); + + foreach (var attempt in document.ProcessingAttempts) + { + attempt.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + attempt.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00.1234567"; + attempt.Headers[NServiceBus.Headers.OriginatingEndpoint] = "Ordering"; + attempt.Headers["NServiceBus.OriginatingMachine"] = "SENDER"; + attempt.Headers[NServiceBus.Headers.OriginatingHostId] = sendingHostId.ToString(); + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = "not an endpoint"; + } + + FailedMessageStream.RestoreMetadataTypes(document); + + var newest = document.ProcessingAttempts[^1]; + var older = document.ProcessingAttempts[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(newest.MessageMetadata["TimeSent"], Is.TypeOf().And.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc).AddTicks(1234567))); + Assert.That(((DateTime)newest.MessageMetadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(newest.MessageMetadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.Name)).EqualTo("Sales")); + var sending = (EndpointDetails)newest.MessageMetadata["SendingEndpoint"]; + Assert.That(sending, Is.Not.Null); + Assert.That(sending.Name, Is.EqualTo("Ordering")); + Assert.That(sending.Host, Is.EqualTo("SENDER")); + Assert.That(sending.HostId, Is.EqualTo(sendingHostId)); + Assert.That(older.MessageMetadata["TimeSent"], Is.EqualTo("2026-07-22T09:59:00.1234567"), "the translator reads only the newest attempt, so an older one is left as stored"); + } + } + + [Test] + public void RestoreMetadataTypes_leaves_a_TimeSent_text_that_is_not_a_date_as_stored() + { + var document = new IngestedFailure().ToFailedMessage(); + var metadata = document.ProcessingAttempts.Single().MessageMetadata; + metadata["TimeSent"] = "garbage"; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(metadata["TimeSent"], Is.EqualTo("garbage"), "the target skips the document as malformed, so one bad document does not halt the category"); + } + + [TestCase("SendingEndpoint", NServiceBus.Headers.OriginatingHostId)] + [TestCase("ReceivingEndpoint", NServiceBus.Headers.HostId)] + public void RestoreMetadataTypes_gives_an_endpoint_the_empty_host_id_when_its_host_id_header_is_not_a_GUID(string key, string hostIdHeader) + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[hostIdHeader] = "not-a-guid"; + attempt.MessageMetadata[key] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata[key]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.HostId, Is.EqualTo(Guid.Empty), "ingestion stores an unparseable host id as the empty GUID, so the message copies rather than being skipped"); + Assert.That(endpoint.Name, Is.EqualTo(key == "SendingEndpoint" ? "Ordering" : "Sales")); + Assert.That(endpoint.Host, Is.EqualTo(key == "SendingEndpoint" ? "SENDER" : "MACHINE")); + } + } + + [Test] + public void RestoreMetadataTypes_builds_a_receiving_endpoint_from_the_failed_queue_when_the_headers_name_none() + { + var document = new IngestedFailure { QueueAddress = "Sales@MACHINE" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.Name, Is.EqualTo("Sales")); + Assert.That(endpoint.Host, Is.EqualTo("MACHINE")); + Assert.That(endpoint.HostId, Is.EqualTo(new Guid("f0c10bfe-aba7-6ceb-d333-ab938515451c")), "ingestion derives the host id from the name and host when it has none"); + } + } + + [Test] + public void RestoreMetadataTypes_gives_a_receiving_endpoint_built_from_a_failed_queue_with_no_machine_the_unknown_host() + { + var document = new IngestedFailure { QueueAddress = "Sales" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.Name, Is.EqualTo("Sales")); + Assert.That(endpoint.Host, Is.EqualTo("unknown")); + Assert.That(endpoint.HostId, Is.EqualTo(new Guid("862257db-407e-ec11-92cc-05ab0c3f7325"))); + } + } + + [Test] + public void RestoreMetadataTypes_leaves_a_newest_attempt_with_no_metadata_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.MessageMetadata = null; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + Assert.That(attempt.MessageMetadata, Is.Null, "the translator skips an attempt with no metadata, so an empty stand-in would hide the fault"); + } + + [Test] + public void RestoreMetadataTypes_leaves_an_untyped_endpoint_on_an_attempt_with_no_headers_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers = null; + attempt.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00Z"; + var stored = new Dictionary { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = stored; + + FailedMessageStream.RestoreMetadataTypes(document); + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.SameAs(stored)); + Assert.That(attempt.MessageMetadata["TimeSent"], Is.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc)), "TimeSent does not come from the headers, so it is still restored"); + } + } + + [Test] + public void RestoreMetadataTypes_lets_an_unexpected_fault_through() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers = new HeadersThatCannotBeRead(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + Assert.Throws(() => FailedMessageStream.RestoreMetadataTypes(document), "a fault the data cannot explain is a defect that must halt the category, not be swallowed as one bad document"); + } + + [TestCase(0)] + [TestCase(1)] + public async Task Reads_a_document_with_a_null_attempt_and_resolves_StatusChangedAt_from_the_others(int nullAttempt) + { + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(numberOfAttempts: 2); + var expected = document.ProcessingAttempts[1 - nullAttempt].AttemptedAt; + document.ProcessingAttempts[nullAttempt] = null; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.EqualTo(expected), "a null attempt is a data fault for the translator to skip, not a reason to halt the category"); + } + + [Test] + public void RestoreMetadataTypes_ignores_a_TimeSent_that_is_not_a_date_on_an_attempt_the_mapper_never_reads() + { + var document = new IngestedFailure().ToFailedMessage(numberOfAttempts: 2); + document.ProcessingAttempts[0].MessageMetadata["TimeSent"] = "garbage"; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document), "a fault in data that is never copied must not halt the category"); + } + + [Test] + public void RestoreMetadataTypes_leaves_a_document_with_no_attempts_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + } + + [Test] + public void RestoreMetadataTypes_keeps_the_receiving_host_id_header_on_the_named_header_path() + { + var hostId = Guid.NewGuid(); + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.HostId] = hostId.ToString(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(hostId)); + } + + [Test] + public void RestoreMetadataTypes_keeps_the_receiving_host_id_header_on_the_failed_queue_path() + { + var hostId = Guid.NewGuid(); + var document = new IngestedFailure { QueueAddress = "Sales@MACHINE" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.Headers[NServiceBus.Headers.HostId] = hostId.ToString(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.HostId)).EqualTo(hostId), "a host id the headers carry is not replaced by one derived from the name and host"); + } + + [Test] + public void RestoreMetadataTypes_prefers_the_host_display_name_to_the_processing_machine() + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.HostDisplayName] = "DISPLAY"; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf().And.Property(nameof(EndpointDetails.Host)).EqualTo("DISPLAY")); + } + + [Test] + public void RestoreMetadataTypes_does_not_throw_when_every_header_it_reads_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + + foreach (var header in new[] + { + NServiceBus.Headers.OriginatingEndpoint, "NServiceBus.OriginatingMachine", NServiceBus.Headers.OriginatingHostId, NServiceBus.Headers.OriginatingAddress, + NServiceBus.Headers.ProcessingEndpoint, NServiceBus.Headers.HostDisplayName, "NServiceBus.ProcessingMachine", NServiceBus.Headers.HostId, "NServiceBus.FailedQ" + }) + { + attempt.Headers[header] = null; + } + + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document), "with the catch gone, a throw on any header value would halt the whole category"); + } + + [Test] + public void RestoreMetadataTypes_restores_no_sending_endpoint_when_its_address_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.OriginatingAddress] = null; + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["SendingEndpoint"], Is.Null, "ingestion stores no endpoint when the address it falls back to has no value"); + } + + [Test] + public void RestoreMetadataTypes_restores_no_receiving_endpoint_when_its_failed_queue_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers["NServiceBus.FailedQ"] = null; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.Null, "ingestion stores no endpoint when the failed queue it falls back to has no value"); + } + + static FailedMessage SenderAndReceiverHeadersFailure() + { + var document = new IngestedFailure().ToFailedMessage(); + var headers = document.ProcessingAttempts.Single().Headers; + headers[NServiceBus.Headers.OriginatingEndpoint] = "Ordering"; + headers["NServiceBus.OriginatingMachine"] = "SENDER"; + headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + return document; + } + + // The index cannot turn this TimeSent into a date, so RavenDB fails to index the document. + static FailedMessage FailedMessageWithATimeSentThatIsNotADate() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved); + document.ProcessingAttempts.Single().MessageMetadata["TimeSent"] = "garbage"; + return document; + } + + async Task UseCorax() + { + await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + await new FailedMessageViewIndex { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } }.ExecuteAsync(DocumentStore, token: TestContext.CurrentContext.CancellationToken); + + var engine = (await DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(IndexName), TestContext.CurrentContext.CancellationToken)).SearchEngineType; + Assert.That(engine, Is.EqualTo(SearchEngineType.Corax), "the test proves nothing about Corax if the index still runs on Lucene"); + } + + async Task IndexStateNow() => + (await DocumentStore.Maintenance.SendAsync(new GetStatisticsOperation(), TestContext.CurrentContext.CancellationToken)).Indexes.Single(index => index.Name == IndexName).State; + + async Task> ReadAll() + { + await using var source = await OpenMigrationSource(); + + return (await CollectBatches(source, Category)).SelectMany(batch => batch.Rows).ToList(); + } + + // RavenDB streams in id order, so statuses are handed out by sorted id rather than by seeding order. + async Task> SeedInIdOrder(params FailedMessageStatus[] statuses) + { + var messages = statuses.Select(_ => new IngestedFailure().ToFailedMessage()) + .OrderBy(message => FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId), StringComparer.Ordinal) + .ToList(); + var ids = new List(messages.Count); + + for (var position = 0; position < messages.Count; position++) + { + messages[position].Status = statuses[position]; + ids.Add((await SeedFailedMessage(messages[position])).Id); + } + + await CompleteDatabaseOperation(); + + return ids; + } + + // Takes the walk limit as a parameter so a test can reach it without seeding the 10,000 documents of RavenDocumentStream.MaxDocumentsWalkedPerBatch. + async Task> StreamWithWalkLimit(int batchSize, int maxDocumentsWalkedPerBatch) + { + var settings = (RavenPersisterSettings)PersistenceSettings; + await using var lifecycle = new RavenReadOnlySourceLifecycle(settings, new SettingsRootNamespace("ServiceControl")); + await lifecycle.Open(TestContext.CurrentContext.CancellationToken); + + var batches = new List(); + + await foreach (var batch in RavenDocumentStream.ByPrefix( + lifecycle, Category.Id, settings.DatabaseName, $"{FailedMessageIdGenerator.CollectionName}/", null, batchSize, maxDocumentsWalkedPerBatch, + new FailedMessageStream(lifecycle).Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses), TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + static readonly string IndexName = new FailedMessageViewIndex().IndexName; + + // The parsers read headers through IReadOnlyDictionary, so re-implementing its lookup makes every header read fail. + sealed class HeadersThatCannotBeRead : Dictionary, IReadOnlyDictionary + { + bool IReadOnlyDictionary.TryGetValue(string key, out string value) => + throw new InvalidOperationException("The headers cannot be read."); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs b/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs index 904bc59d15..5516e0d439 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs @@ -20,7 +20,15 @@ public static class RavenIndexAwaiter public static Task WaitForIndexingAsync(this IDocumentStore store, CancellationToken cancellationToken = default) => store.WaitForIndexingAsync(DefaultTimeout, cancellationToken); - public static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, CancellationToken cancellationToken = default) + public static Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, CancellationToken cancellationToken = default) => + store.WaitForIndexingAsync(timeout, allowIndexErrors: false, cancellationToken); + + // For a test that stores a document an index cannot map on purpose. An index in the error state counts as done, + // because it never catches up. + public static Task WaitForIndexingAllowingErrorsAsync(this IDocumentStore store, CancellationToken cancellationToken = default) => + store.WaitForIndexingAsync(DefaultTimeout, allowIndexErrors: true, cancellationToken); + + static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, bool allowIndexErrors, CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); IndexStats[] stats; @@ -31,14 +39,14 @@ public static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpa // An index in the error state never becomes up to date. Fail now with the errors instead of after the timeout. var erroredIndexes = stats.Where(i => i.State == IndexState.Error || i.ErrorsCount > 0).Select(i => i.Name).ToArray(); - if (erroredIndexes.Length > 0) + if (erroredIndexes.Length > 0 && !allowIndexErrors) { var errors = await store.Maintenance.SendAsync(new GetIndexErrorsOperation(erroredIndexes), cancellationToken); var details = errors.SelectMany(e => e.Errors.Select(x => $"{e.Name}: {x.Action} {x.Document} {x.Error}")); Assert.Fail($"Indexes have errors:{Environment.NewLine}{string.Join(Environment.NewLine, details)}"); } - if (stats.All(i => !i.IsStale)) + if (stats.All(i => !i.IsStale || (allowIndexErrors && i.State == IndexState.Error))) { return; } diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs new file mode 100644 index 0000000000..f1914f23b1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs @@ -0,0 +1,69 @@ +// ReSharper disable once CheckNamespace +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationGroupInsertSeekTests : PersistenceTestBase +{ + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task The_group_insert_seeks_existing_rows_by_key_while_the_group_table_is_small() + { + List sharedGroups = + [ + .. new[] { "Exception Type and Stack Trace", "Message Type", "Endpoint Address", "Endpoint Instance", "Endpoint Name" } + .Select(type => new FailedMessage.FailureGroup { Id = Guid.NewGuid().ToString(), Title = $"{type} title", Type = type }) + ]; + var rows = Enumerable.Range(0, 74) + .Select(_ => new IngestedFailure { Groups = sharedGroups }.ToFailedMessage()) + .Select(document => new MigrationRow($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }, new MigrationBody("1"u8.ToArray(), "text/xml"))) + .ToList(); + + var result = await Target.Write(FailedMessagesCategory, new MigrationBatch(rows, rows[^1].SourceId), new MigrationCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using var scope = ServiceProvider.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(74)); + Assert.That(await GroupTableReadsInCachedPlan(db), Is.Not.Empty.And.None.Contains("Scan"), "a scan reads every row of an index and HOLDLOCK locks each one, which FORCESEEK prevents"); + } + } + + // Seek and scan operators both hold an IndexScan element, so the RelOp above it says which one it is. + static Task> GroupTableReadsInCachedPlan(ServiceControlDbContext db) + { + var statement = $"MERGE {MigrationInsert.For(db).Table} "; + return db.Database.SqlQuery($""" + WITH XMLNAMESPACES (DEFAULT 'http://schemas.microsoft.com/sqlserver/2004/07/showplan') + SELECT o.n.value('../../@PhysicalOp', 'nvarchar(60)') + ' ' + o.n.value('@Index', 'nvarchar(260)') AS [Value] + FROM sys.dm_exec_cached_plans AS p + CROSS APPLY sys.dm_exec_sql_text(p.plan_handle) AS t + CROSS APPLY sys.dm_exec_query_plan(p.plan_handle) AS q + CROSS APPLY q.query_plan.nodes('//RelOp/IndexScan/Object[@Table="[FailedMessageGroups]"]') AS o(n) + WHERE t.dbid = DB_ID() AND CHARINDEX({statement}, t.text) > 0 + """).ToListAsync(); + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs new file mode 100644 index 0000000000..f6135cc924 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs @@ -0,0 +1,78 @@ +// ReSharper disable once CheckNamespace +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationPlanReuseTests : PersistenceTestBase +{ + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task Batches_whose_unbounded_text_sits_either_side_of_4000_characters_share_one_cached_plan() + { + var shortHeaders = new IngestedFailure(); + shortHeaders.Headers["Padding"] = new string('h', 3_000); + var longHeaders = new IngestedFailure { Groups = [new() { Id = Guid.NewGuid().ToString(), Title = new string('t', 5_000), Type = "Message Type" }] }; + longHeaders.Headers["Padding"] = new string('h', 5_000); + + await Target.Write(FailedMessagesCategory, BatchOf(shortHeaders.ToFailedMessage()), Checkpoint()); + var saved = await ServiceProvider.GetRequiredService().Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await Target.Write(FailedMessagesCategory, BatchOf(longHeaders.ToFailedMessage()), saved); + + using var scope = ServiceProvider.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var headersJsonLengths = await db.FailedMessages.AsNoTracking().Select(message => message.HeadersJson.Length).ToListAsync(); + var titleLengths = await db.FailedMessageGroups.AsNoTracking().Select(group => group.Title.Length).ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(headersJsonLengths, Has.Exactly(1).LessThan(4_000).And.Exactly(1).GreaterThan(4_000), "one HeadersJson must fit nvarchar(4000) and the other must not. If the long one is exactly 4000, the insert cut it short"); + Assert.That(titleLengths, Does.Contain(5_000), "the long group Title must be stored whole"); + Assert.That(await PlanUseCountsFor(db), Is.EqualTo(new[] { 2 }), "short and long text must reuse one plan, because each new parameter declaration list compiles its own"); + Assert.That(await PlanUseCountsFor(db), Is.EqualTo(new[] { 2 }), "the group Title column is unbounded too"); + } + } + + // Scoped to this test's schema, because the suite's other fixtures share the server and its plan cache. + static Task> PlanUseCountsFor(ServiceControlDbContext db) where TEntity : class + { + var statement = $"MERGE {MigrationInsert.For(db).Table} "; + return db.Database.SqlQuery($""" + SELECT p.usecounts AS [Value] + FROM sys.dm_exec_cached_plans AS p + CROSS APPLY sys.dm_exec_sql_text(p.plan_handle) AS t + WHERE t.dbid = DB_ID() AND CHARINDEX({statement}, t.text) > 0 + """).ToListAsync(); + } + + static MigrationBatch BatchOf(FailedMessage document) + { + var row = new MigrationRow($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }, new MigrationBody("1"u8.ToArray(), "text/xml")); + + return new MigrationBatch([row], row.SourceId); + } + + static MigrationCheckpoint Checkpoint() => + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs new file mode 100644 index 0000000000..7f024576bc --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs @@ -0,0 +1,70 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationValueLengthTests : PersistenceTestBase +{ + static readonly string OutsideTheBasicPlane = char.ConvertFromUtf32(0x1F600); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task A_value_is_measured_in_utf16_units_as_nvarchar_counts_them() + { + var fits = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(225), HostId = Guid.NewGuid(), Host = "SALES01" } }; + var tooLong = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(226), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", fits, new Dictionary()), + new MigrationRow("KnownEndpoints/2", tooLong, new Dictionary()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "225 code points are 450 UTF-16 units, which an nvarchar(450) column holds"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2" }), "226 code points are 452 UTF-16 units, which would fail the insert"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { Repeat(225) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 452, on document 'KnownEndpoints/2'" + }), "the length reported is the one SQL Server counts"); + Assert.That(ServiceProvider.GetRequiredService().TextLength(Repeat(226)), Is.LessThanOrEqualTo(Repeat(226).Length), "the check skips measuring a value whose UTF-16 length fits, which is safe only while no count exceeds it"); + } + } + + [Test] + public async Task A_value_holding_a_NUL_character_copies() + { + var withNul = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SAL\u0000ES" } }; + var batch = new MigrationBatch([new MigrationRow("KnownEndpoints/1", withNul, new Dictionary())], "KnownEndpoints/1"); + + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/1", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "nvarchar stores U+0000, so a skip here would give up a row SQL Server can hold"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Single().EndpointDetails.Host, Is.EqualTo("SAL\u0000ES")); + } + } + + static string Repeat(int codePoints) => string.Concat(Enumerable.Repeat(OutsideTheBasicPlane, codePoints)); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs index 2826d26011..a6e01f9fc4 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs @@ -7,7 +7,10 @@ namespace ServiceControl.Persistence.Tests; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; class EndpointSettingsMigrationTargetTests : PersistenceTestBase { @@ -78,6 +81,31 @@ public async Task A_setting_whose_endpoint_is_not_known_is_copied_and_left_to_th } } + [Test] + public async Task A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var batch = BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = new string('s', 450), TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = new string('n', 451), TrackInstances = true }), + ("EndpointSettings/3", new EndpointSettings { Name = "Billing", TrackInstances = false })); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new EndpointSettingsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(EndpointSettingsCategory, batch, CheckpointAfter("EndpointSettings/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "a name exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "EndpointSettings/2" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 1 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(await TrackInstancesFor(new string('s', 450)), Is.True); + Assert.That(await TrackInstancesFor("Billing"), Is.False); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(EndpointSettingsEntity.Name))} takes at most 450 characters and the value holds 451, on document 'EndpointSettings/2'" + })); + } + } + [Test] public async Task A_batch_that_copies_nothing_still_saves_the_cursor() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs new file mode 100644 index 0000000000..9686b9a4f1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs @@ -0,0 +1,570 @@ +#nullable enable +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[FixtureLifeCycle(LifeCycle.InstancePerTestCase)] +class FailedMessageDocumentTranslatorTests +{ + static readonly DateTime StatusChangedAtFromTheSource = new(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + static readonly DateTime LastModifiedFromTheSource = new(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc); + + [Test] + public void Carries_the_unique_message_id_across_unchanged() + { + var uniqueMessageId = Guid.NewGuid(); + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(); + document.UniqueMessageId = uniqueMessageId.ToString(); + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.UniqueMessageId, Is.EqualTo(uniqueMessageId)); + } + + [Test] + public void Skips_a_unique_message_id_that_is_not_a_guid() + { + var document = new IngestedFailure().ToFailedMessage(); + document.UniqueMessageId = "not-a-guid"; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.UnparseableUniqueMessageId)); + Assert.That(skip.Detail, Does.Contain("not-a-guid")); + } + } + + [Test] + public void Skips_a_document_with_no_processing_attempts() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts)); + Assert.That(skip.Detail, Does.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Skips_a_document_with_no_processing_attempts_that_the_source_sent_with_no_StatusChangedAt() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + var complete = Row(document); + var row = complete with { Metadata = new Dictionary(complete.Metadata) { [MigrationMetadataKeys.StatusChangedAt] = null } }; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts), "the source resolves StatusChangedAt from the newest attempt, so an attemptless document always arrives without one"); + } + + [Test] + public void Skips_a_document_whose_newest_attempt_has_no_failing_endpoint_address() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].FailureDetails.AddressOfFailingEndpoint = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "FailingEndpointAddress is NOT NULL, so the insert would fail and halt the category"); + Assert.That(skip.Detail, Does.Contain(nameof(FailedMessageEntity.FailingEndpointAddress)).And.Contain(document.UniqueMessageId), "the log has to name the column and the message an operator would look up"); + } + } + + [TestCase(nameof(FailedMessage.ProcessingAttempt.FailureDetails))] + [TestCase(nameof(FailedMessage.ProcessingAttempt.MessageMetadata))] + [TestCase(nameof(FailedMessage.ProcessingAttempt.Headers))] + public void Skips_a_newest_attempt_with_no_value_the_row_is_built_from(string missing) + { + var document = new IngestedFailure().ToFailedMessage(); + var newest = document.ProcessingAttempts[^1]; + switch (missing) + { + case nameof(newest.FailureDetails): + newest.FailureDetails = null!; + break; + case nameof(newest.MessageMetadata): + newest.MessageMetadata = null!; + break; + default: + newest.Headers = null!; + break; + } + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + Assert.That(skip.Detail, Does.Contain(missing).And.Contain(document.UniqueMessageId), "the log has to name what is missing and the message an operator would look up"); + } + } + + [TestCase("TimeSent", "garbage", "garbage")] + [TestCase("MessageType", 43L, "43")] + [TestCase("ConversationId", 44L, "44")] + [TestCase("IsSystemMessage", "yes", "yes")] + public void Skips_a_value_present_with_the_wrong_type_and_names_the_key_the_type_found_and_the_value(string key, object malformed, string shown) + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata[key] = malformed; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.MalformedValue), "copying it as null or false would lose a value the source holds without a skip to show for it"); + Assert.That(skip.Detail, Does.Contain(key).And.Contain(malformed.GetType().FullName).And.Contain(document.UniqueMessageId)); + Assert.That(skip.Detail, Does.Contain($"'{shown}'"), "the operator should not have to open RavenDB to see the value"); + } + } + + [Test] + public void Shows_only_the_first_100_characters_of_a_malformed_value() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = new string('a', 100) + new string('b', 400); + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Detail, Does.Contain($"'{new string('a', 100)}...'").And.Not.Contain(new string('a', 100) + "b"), "a stored value can be any length, and the detail is one log line"); + } + + [Test] + public void Skips_a_newest_attempt_whose_FailureDetails_key_was_missing_from_the_document() + { + var document = new IngestedFailure().ToFailedMessage(); + // What RavenDB hands back for an attempt stored without the key. + document.ProcessingAttempts[^1].FailureDetails = new FailedMessage.ProcessingAttempt().FailureDetails; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "copying it would store an empty failing address to retry to and a time of failure that is the time it was read"); + Assert.That(skip.Detail, Does.Contain(nameof(FailedMessageEntity.FailingEndpointAddress)).And.Contain(document.UniqueMessageId)); + } + } + + [TestCase("TimeOfFailure")] + [TestCase("AttemptedAt")] + public void Skips_a_newest_attempt_with_no_time_it_failed_or_was_attempted(string missing) + { + var document = new IngestedFailure().ToFailedMessage(); + var newest = document.ProcessingAttempts[^1]; + if (missing == "TimeOfFailure") + { + newest.FailureDetails.TimeOfFailure = default; + } + else + { + newest.AttemptedAt = default; + } + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "the year 1 is not a time the message failed"); + Assert.That(skip.Detail, Does.Contain(missing).And.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Leaves_an_older_attempt_whose_FailureDetails_key_was_missing_out_of_the_first_and_last_failure_times() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails = new FailedMessage.ProcessingAttempt().FailureDetails; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[1].FailureDetails.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure), "the default object's time of failure is the time it was read, which would become the last failure"); + } + } + + [Test] + public void Leaves_an_older_attempt_with_no_time_of_failure_out_of_the_first_failure_time() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails.TimeOfFailure = default; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[1].FailureDetails.TimeOfFailure), "the year 1 is not a time the message failed"); + } + + [Test] + public void Skips_a_document_whose_processing_attempts_list_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts)); + } + + [Test] + public void Translates_a_null_failure_groups_list_to_no_groups() + { + var document = new IngestedFailure().ToFailedMessage(); + document.FailureGroups = null!; + + var translation = FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That(((FailedMessageDocumentTranslation.Translated)translation).Groups, Is.Empty); + } + + [Test] + public void Drops_a_failure_group_entry_that_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + document.FailureGroups = [null!, new() { Id = "group-1", Title = "Title 1", Type = "Type 1" }]; + + var translation = FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That(((FailedMessageDocumentTranslation.Translated)translation).Groups.Select(group => group.GroupId), Is.EqualTo(new[] { "group-1" })); + } + + [TestCase("SendingEndpoint", "not an endpoint")] + [TestCase("ReceivingEndpoint", 42)] + public void Refuses_an_endpoint_that_is_not_an_EndpointDetails_rather_than_copying_it_as_null(string key, object wrongType) + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata[key] = wrongType; + + Assert.Throws(() => FailedMessageDocumentTranslator.Translate(Row(document)), "the source restores every endpoint before the translator sees it, so a wrong type is a defect that must halt the copy, not a row with empty endpoint columns"); + } + + [TestCase(true)] + [TestCase(false)] + public void Translates_a_TimeSent_or_endpoint_that_is_missing_or_null_to_null(bool keyPresent) + { + var document = new IngestedFailure().ToFailedMessage(); + var metadata = document.ProcessingAttempts[^1].MessageMetadata; + foreach (var key in new[] { "TimeSent", "SendingEndpoint", "ReceivingEndpoint" }) + { + if (keyPresent) + { + metadata[key] = null!; + } + else + { + metadata.Remove(key); + } + } + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.TimeSent, Is.Null); + Assert.That(new[] { row.SendingEndpointName, row.SendingEndpointHost, row.ReceivingEndpointName, row.ReceivingEndpointHost }, Is.All.Null, "RavenDB's own read path shows a missing endpoint as null"); + Assert.That(new[] { row.SendingEndpointHostId, row.ReceivingEndpointHostId }, Is.All.Null); + } + } + + [Test] + public void Skips_a_document_whose_newest_attempt_is_null() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[2] = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + Assert.That(skip.Detail, Does.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Takes_the_earliest_failure_from_the_attempts_that_have_failure_details() + { + var document = ThreeAttempts(); + var earliestWithDetails = document.ProcessingAttempts[1].FailureDetails.TimeOfFailure; + document.ProcessingAttempts[0].FailureDetails = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(earliestWithDetails), "an older attempt with no failure details has no time to offer, and must not halt the category"); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + } + } + + [Test] + public void Ignores_an_earlier_attempt_that_is_null() + { + var document = ThreeAttempts(); + var earliestRemaining = document.ProcessingAttempts[1].FailureDetails.TimeOfFailure; + document.ProcessingAttempts[0] = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(3), "RavenDB's read path counts every stored attempt, null ones included"); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(earliestRemaining)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + } + } + + [Test] + public void Translates_when_only_an_earlier_attempt_has_a_value_of_the_wrong_type() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].MessageMetadata["TimeSent"] = "garbage"; + document.ProcessingAttempts[0].MessageMetadata["ReceivingEndpoint"] = "not an endpoint"; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address"), "only the newest attempt's metadata becomes the row"); + } + + [Test] + public void Keeps_the_last_450_characters_of_a_message_type_longer_than_the_column() + { + var document = new IngestedFailure().ToFailedMessage(); + var longType = new string('a', 100) + new string('b', 450); + document.ProcessingAttempts[^1].MessageMetadata["MessageType"] = longType; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.MessageType, Is.EqualTo(new string('b', 450))); + } + + [Test] + public void Takes_the_newest_attempt_for_the_row_and_the_earliest_and_latest_failure_across_all() + { + var document = ThreeAttempts(); + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(3)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[0].FailureDetails.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(document.ProcessingAttempts[2].AttemptedAt)); + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address")); + Assert.That(row.ExceptionMessage, Is.EqualTo("newest-exception")); + } + } + + [Test] + public void Skips_when_only_the_newest_of_three_attempts_has_no_failing_endpoint_address() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[2].FailureDetails.AddressOfFailingEndpoint = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + } + + [Test] + public void Translates_when_only_an_earlier_attempt_has_no_failing_endpoint_address() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails.AddressOfFailingEndpoint = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address")); + } + + [Test] + public void Translates_each_group_with_an_id_to_a_row_keyed_by_the_message_and_drops_a_group_with_no_id() + { + var uniqueMessageId = Guid.NewGuid(); + var document = new IngestedFailure().ToFailedMessage(); + document.UniqueMessageId = uniqueMessageId.ToString(); + document.FailureGroups = + [ + new() { Id = "group-1", Title = "Title 1", Type = "Type 1" }, + new() { Id = "group-2", Title = null, Type = null }, + new() { Id = null!, Title = "No id", Type = "Type" } + ]; + + var translation = (FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That( + translation.Groups.Select(group => (group.FailedMessageUniqueId, group.GroupId, group.Title, group.Type)), + Is.EqualTo(new[] + { + (uniqueMessageId, "group-1", "Title 1", "Type 1"), + (uniqueMessageId, "group-2", string.Empty, string.Empty) + })); + } + + [TestCase(FailedMessageStatus.Unresolved)] + [TestCase(FailedMessageStatus.Resolved)] + [TestCase(FailedMessageStatus.RetryIssued)] + [TestCase(FailedMessageStatus.Archived)] + public void Preserves_the_stored_status(FailedMessageStatus status) + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(status)))); + + Assert.That(row.Status, Is.EqualTo(status)); + } + + [Test] + public void Stores_the_StatusChangedAt_the_source_resolved() + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)))); + + Assert.That(row.StatusChangedAt, Is.EqualTo(StatusChangedAtFromTheSource)); + } + + [Test] + public void Stores_the_LastModified_the_source_carried() + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)))); + + Assert.That(row.LastModified, Is.EqualTo(LastModifiedFromTheSource), "RavenDB's @last-modified, not the copy time"); + } + + [TestCase(MigrationMetadataKeys.StatusChangedAt)] + [TestCase(MigrationMetadataKeys.LastModified)] + public void Refuses_a_row_the_source_sent_without_a_resolved_value(string missingKey) + { + var complete = Row(new IngestedFailure().ToFailedMessage()); + var row = complete with { Metadata = complete.Metadata.Where(entry => entry.Key != missingKey).ToDictionary(entry => entry.Key, entry => entry.Value) }; + + var ex = Assert.Throws(() => FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(ex.Message, Does.Contain(missingKey).And.Contain(row.SourceId)); + } + + [TestCase(MigrationMetadataKeys.StatusChangedAt)] + [TestCase(MigrationMetadataKeys.LastModified)] + public void Refuses_a_resolved_value_of_the_wrong_type_and_names_the_type_it_found(string key) + { + var complete = Row(new IngestedFailure().ToFailedMessage()); + var row = complete with { Metadata = new Dictionary(complete.Metadata) { [key] = "2026-09-01T08:00:00Z" } }; + + var ex = Assert.Throws(() => FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(ex.Message, Does.Contain(typeof(string).FullName).And.Contain(key).And.Contain(row.SourceId), "a value that is present but mistyped must not read as missing"); + } + + [TestCase(DateTimeKind.Unspecified)] + [TestCase(DateTimeKind.Local)] + public void Writes_every_date_as_UTC_whatever_kind_it_arrived_with(DateTimeKind kind) + { + var document = ThreeAttempts(); + foreach (var attempt in document.ProcessingAttempts) + { + attempt.FailureDetails.TimeOfFailure = DateTime.SpecifyKind(attempt.FailureDetails.TimeOfFailure, kind); + attempt.AttemptedAt = DateTime.SpecifyKind(attempt.AttemptedAt, kind); + } + + var newest = document.ProcessingAttempts[^1]; + var timeSent = new DateTime(2026, 7, 22, 9, 59, 0, kind); + newest.MessageMetadata["TimeSent"] = timeSent; + var statusChangedAt = DateTime.SpecifyKind(StatusChangedAtFromTheSource, kind); + var lastModified = DateTime.SpecifyKind(LastModifiedFromTheSource, kind); + var complete = Row(document); + var source = complete with + { + Metadata = new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = statusChangedAt, + [MigrationMetadataKeys.LastModified] = lastModified + } + }; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(source)); + + DateTime[] written = [row.StatusChangedAt, row.LastModified, row.FirstTimeOfFailure, row.LastTimeOfFailure, row.LastAttemptedAt, row.TimeSent!.Value]; + DateTime[] expected = [statusChangedAt, lastModified, document.ProcessingAttempts[0].FailureDetails.TimeOfFailure, newest.FailureDetails.TimeOfFailure, newest.AttemptedAt, timeSent]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(written.Select(value => value.Kind), Is.All.EqualTo(DateTimeKind.Utc), "Npgsql refuses to write a DateTime of any other kind to a timestamp with time zone column"); + Assert.That(written, Is.EqualTo(expected.Select(value => kind == DateTimeKind.Local ? value.ToUniversalTime() : DateTime.SpecifyKind(value, DateTimeKind.Utc))), "a value with no kind is already UTC and must not be shifted by the local offset"); + } + } + + [Test] + public void Stores_a_body_externally_when_the_content_type_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var headers = document.ProcessingAttempts[^1].Headers; + headers[NServiceBus.Headers.ContentType] = null!; + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + var storeExternally = FailedMessageDocumentTranslator.SetBody(row, headers, new MigrationBody("hello"u8.ToArray(), "text/plain"), BodyStorageSettings.DefaultMaxBodySizeToStore); + + using (Assert.EnterMultipleScope()) + { + Assert.That(storeExternally, Is.True, "a null content type says nothing about the body, so it is treated like a missing one, which is binary"); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodyText, Is.Null); + } + } + + static FailedMessageEntity RowOf(FailedMessageDocumentTranslation translation) + { + Assert.That(translation, Is.TypeOf()); + return ((FailedMessageDocumentTranslation.Translated)translation).Row; + } + + static FailedMessageDocumentTranslation.Skipped SkipOf(FailedMessageDocumentTranslation translation) + { + Assert.That(translation, Is.TypeOf()); + return (FailedMessageDocumentTranslation.Skipped)translation; + } + + static FailedMessage ThreeAttempts() + { + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(numberOfAttempts: 3); + for (var i = 0; i < 3; i++) + { + var attempt = document.ProcessingAttempts[i]; + attempt.FailureDetails.TimeOfFailure = new DateTime(2026, 7, 22, 10 + i, 0, 0, DateTimeKind.Utc); + attempt.AttemptedAt = attempt.FailureDetails.TimeOfFailure; + } + + var newest = document.ProcessingAttempts[2]; + newest.FailureDetails.AddressOfFailingEndpoint = "newest-address"; + newest.FailureDetails.Exception!.Message = "newest-exception"; + return document; + } + + // Every row carries the two dates the source resolves, as a real source's rows do. + static MigrationRow Row(FailedMessage document) => + new( + $"FailedMessages/{document.UniqueMessageId}", + document, + new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = StatusChangedAtFromTheSource, + [MigrationMetadataKeys.LastModified] = LastModifiedFromTheSource + }); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs new file mode 100644 index 0000000000..59afd305ea --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs @@ -0,0 +1,80 @@ +#nullable enable +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class FailedMessageMigrationCoverageTests : PersistenceTestBase +{ + [Test] + public void Every_failed_message_column_is_set_from_a_fully_populated_document() + { + var source = FullyPopulatedRow(); + var row = ((FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(source)).Row; + // Over the cap, so it sets all four body columns: an inline prefix, the external flag, the size and the content type. + var overCapText = Encoding.UTF8.GetBytes(new string('a', BodyStorageSettings.DefaultMaxBodySizeToStore * 2)); + + FailedMessageDocumentTranslator.SetBody(row, ((FailedMessage)source.Document).ProcessingAttempts[^1].Headers, new MigrationBody(overCapText, "text/plain"), BodyStorageSettings.DefaultMaxBodySizeToStore); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), row); + } + + [Test] + public void Every_failure_group_column_is_set_from_a_fully_populated_document() + { + var translation = (FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(FullyPopulatedRow()); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), translation.Groups.Single()); + } + + [Test] + public async Task Every_retry_claim_column_is_set_for_a_copied_retry_issued_message() + { + var target = ServiceProvider.GetRequiredService(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var source = FullyPopulatedRow() with { Body = new MigrationBody(Encoding.UTF8.GetBytes("1"), "text/xml") }; + ((FailedMessage)source.Document).Status = FailedMessageStatus.RetryIssued; + await target.Open(); + + await target.Write(category, new MigrationBatch([source], source.SourceId), new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using var scope = ServiceProvider.CreateScope(); + var claim = await scope.ServiceProvider.GetRequiredService().FailedMessageRetries.AsNoTracking().SingleAsync(); + + // The migrated claim's batch is never staged, so its attempt count stays 0 and is never read. + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), claim, nameof(FailedMessageRetryEntity.StageAttempts)); + } + + static MigrationRow FullyPopulatedRow() + { + var failure = new IngestedFailure { IsSystemMessage = true, FailingEndpointAddress = "Sales@MACHINE" }; + + return new MigrationRow( + $"FailedMessages/{failure.UniqueMessageIdString}", + failure.ToFailedMessage(), + new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }); + } + + IModel Model() + { + using var scope = ServiceProvider.CreateScope(); + return scope.ServiceProvider.GetRequiredService().Model; + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs new file mode 100644 index 0000000000..576d0e0cdb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs @@ -0,0 +1,467 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class FailedMessagesMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + IMigrationCheckpointStore CheckpointStore => ServiceProvider.GetRequiredService(); + + IBodyStoragePersistence BodyStore => ServiceProvider.GetRequiredService(); + + static readonly MigrationBody SmallTextBody = new(Encoding.UTF8.GetBytes("1"), "text/xml"); + + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + [Test] + public async Task A_copied_message_arrives_with_its_failure_group_rows() + { + var failure = new IngestedFailure(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(failure.ToFailedMessage()), Checkpoint()); + + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(groups.Select(group => (group.GroupId, group.Title, group.Type)), Is.EqualTo(failure.Groups.Select(group => (group.Id, group.Title, group.Type)))); + } + } + + [Test] + public async Task A_message_already_in_the_target_keeps_its_row_and_gets_no_group_rows() + { + var failure = new IngestedFailure(); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = failure.UniqueMessageId, Status = FailedMessageStatus.Unresolved, ExceptionMessage = "Already in the target", HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(failure.ToFailedMessage()), Checkpoint()); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == failure.UniqueMessageId)); + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "nothing was lost, so a skip would push a healthy category toward the halt threshold"); + Assert.That(row.ExceptionMessage, Is.EqualTo("Already in the target"), "insert-if-absent never updates"); + Assert.That(groups, Is.Zero, "the copy wrote the source's groups onto a message it did not insert"); + } + } + + [Test] + public async Task A_copied_retry_issued_message_gets_a_retry_claim_and_an_unresolved_one_does_not() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(unresolved.ToFailedMessage(), retryIssued.ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => new { claim.UniqueMessageId, claim.RetryBatchId, claim.StageAttempts }).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + Assert.That(claims, Is.EqualTo(new[] { new { retryIssued.UniqueMessageId, RetryBatchId = FailedMessagesWriterBase.MigratedRetryClaimBatchId, StageAttempts = 0 } }), + "without a claim a retry by id sends a retry-issued message again, and with one an unresolved message could not be retried by id"); + } + } + + [Test] + public async Task A_retry_issued_message_already_in_the_target_gets_no_retry_claim() + { + var present = new IngestedFailure(); + var absent = new IngestedFailure(); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = present.UniqueMessageId, Status = FailedMessageStatus.RetryIssued, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage(FailedMessageStatus.RetryIssued), absent.ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + + var claimed = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(claimed, Is.EqualTo(new[] { absent.UniqueMessageId }), "a message the insert turned away keeps what the target had, so it gets no claim from the copy"); + } + } + + [Test] + public async Task Two_documents_sharing_a_unique_message_id_get_a_retry_claim_only_when_the_kept_one_is_retry_issued() + { + var unresolvedKept = new IngestedFailure(); + var retryIssuedKept = new IngestedFailure(); + var retryIssuedTurnedAway = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + retryIssuedTurnedAway.UniqueMessageId = unresolvedKept.UniqueMessageIdString; + var unresolvedTurnedAway = new IngestedFailure().ToFailedMessage(); + unresolvedTurnedAway.UniqueMessageId = retryIssuedKept.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + unresolvedKept.ToFailedMessage(), retryIssuedTurnedAway, + retryIssuedKept.ToFailedMessage(FailedMessageStatus.RetryIssued), unresolvedTurnedAway), Checkpoint()); + + var claimed = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(claimed, Is.EqualTo(new[] { retryIssuedKept.UniqueMessageId }), "the claim follows the status of the row the insert kept, not of the document it turned away"); + } + } + + [Test] + public async Task Copied_counts_the_messages_inserted_not_the_messages_handed_in() + { + var present = new IngestedFailure(); + var absent = new IngestedFailure(); + await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage()), Checkpoint()); + var saved = await CheckpointStore.Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage(), absent.ToFailedMessage()), saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task Two_documents_sharing_a_unique_message_id_store_only_the_first_documents_groups() + { + var first = new IngestedFailure { Groups = [new() { Id = "group-of-the-first", Title = "First", Type = "Message Type" }] }; + var second = new IngestedFailure { Groups = [new() { Id = "group-of-the-second", Title = "Second", Type = "Message Type" }] }; + var secondDocument = second.ToFailedMessage(); + secondDocument.UniqueMessageId = first.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf(first.ToFailedMessage(), secondDocument), Checkpoint()); + + var groupIds = await Query(db => db.FailedMessageGroups.AsNoTracking().Where(group => group.FailedMessageUniqueId == first.UniqueMessageId).Select(group => group.GroupId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(groupIds, Is.EqualTo(new[] { "group-of-the-first" }), "the declined document's groups landed on the message the insert kept"); + } + } + + [Test] + public async Task A_document_the_mapper_skips_is_counted_under_its_reason_and_never_written() + { + var good = new IngestedFailure(); + var unparseable = new IngestedFailure().ToFailedMessage(); + unparseable.UniqueMessageId = "not-a-guid"; + var attemptless = new IngestedFailure().ToFailedMessage(); + attemptless.ProcessingAttempts.Clear(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(good.ToFailedMessage(), unparseable, attemptless), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(2)); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary + { + [MigrationSkipReason.UnparseableUniqueMessageId] = 1, + [MigrationSkipReason.NoProcessingAttempts] = 1 + })); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(1)); + } + } + + [Test] + public async Task A_document_with_a_value_of_the_wrong_type_or_no_failure_details_is_skipped_and_the_rest_copy() + { + var good = new IngestedFailure(); + var badTimeSent = new IngestedFailure().ToFailedMessage(); + badTimeSent.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = "garbage"; + var noFailureDetails = new IngestedFailure().ToFailedMessage(); + noFailureDetails.ProcessingAttempts[^1].FailureDetails = null; + + var result = await Target.Write(FailedMessagesCategory, BatchOf(badTimeSent, good.ToFailedMessage(), noFailureDetails), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EquivalentTo(new[] { $"FailedMessages/{badTimeSent.UniqueMessageId}", $"FailedMessages/{noFailureDetails.UniqueMessageId}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary + { + [MigrationSkipReason.MalformedValue] = 1, + [MigrationSkipReason.RequiredValueMissing] = 1 + }), "one malformed document no longer halts the category"); + Assert.That(await Query(db => db.FailedMessages.AsNoTracking().Select(message => message.UniqueMessageId).ToListAsync()), Is.EqualTo(new[] { good.UniqueMessageId })); + } + } + + [Test] + public async Task A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy() + { + var fitsExactly = new IngestedFailure + { + ConversationId = new string('c', 450), + Groups = [new() { Id = new string('g', 64), Title = "OrderPlaced", Type = new string('t', 255) }] + }; + var longConversation = new IngestedFailure { ConversationId = new string('c', 451) }; + var longGroupType = new IngestedFailure { Groups = [new() { Id = Guid.NewGuid().ToString(), Title = "OrderPlaced", Type = new string('t', 256) }] }; + var longMessageType = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(fitsExactly.ToFailedMessage(), longConversation.ToFailedMessage(), longGroupType.ToFailedMessage(), longMessageType.ToFailedMessage()); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new FailedMessagesWriter(ServiceProvider.GetRequiredService(), BodyStore, ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + + var copiedIds = await Query(db => db.FailedMessages.AsNoTracking().Select(message => message.UniqueMessageId).ToListAsync()); + var groupOwners = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => group.FailedMessageUniqueId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "values exactly as long as their columns fit, and MessageType keeps its last 450 characters as ingestion does, so neither is skipped for length"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{longConversation.UniqueMessageIdString}", $"FailedMessages/{longGroupType.UniqueMessageIdString}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(copiedIds, Is.EquivalentTo(new[] { fitsExactly.UniqueMessageId, longMessageType.UniqueMessageId })); + Assert.That(groupOwners, Is.EquivalentTo(new[] { fitsExactly.UniqueMessageId, longMessageType.UniqueMessageId }), "a message whose group row cannot be stored is skipped whole, not copied without its group"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(FailedMessageEntity.ConversationId))} takes at most 450 characters and the value holds 451, on document 'FailedMessages/{longConversation.UniqueMessageIdString}'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(FailedMessageGroupEntity.Type))} takes at most 255 characters and the value holds 256, on document 'FailedMessages/{longGroupType.UniqueMessageIdString}'" + })); + } + } + + [Test] + public async Task Count_takes_in_only_unresolved_and_retry_issued_messages() + { + await Target.Write(FailedMessagesCategory, BatchOf(new IngestedFailure().ToFailedMessage(), new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + await Query(db => + { + db.FailedMessages.AddRange( + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.Archived, HeadersJson = "{}", FailingEndpointAddress = "error" }, + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.Resolved, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await Target.Count(FailedMessagesCategory), Is.EqualTo(2), "the archived and resolved rows belong to another category"); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(4), "the filter is doing the work, not an empty table"); + } + } + + // PostgreSQL takes a fixed number of rows per statement whatever the entity, so only SQL Server can catch the wrong entity here. + [Test] + public async Task A_batch_is_one_statement_of_failed_message_rows() + { + var dialect = ServiceProvider.GetRequiredService(); + var expected = await Query(db => Task.FromResult(dialect.RowsPerStatement(MigrationInsert.For(db).Columns.Count))); + + Assert.That(await Target.BatchSizeFor(FailedMessagesCategory), Is.EqualTo(expected)); + } + + [Test] + public async Task A_write_larger_than_one_statement_copies_every_message_and_its_groups() + { + var rowsPerStatement = await Target.BatchSizeFor(FailedMessagesCategory); + var documents = Enumerable.Range(0, (rowsPerStatement * 2) + 1).Select(_ => new IngestedFailure().ToFailedMessage()).ToArray(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(documents), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(documents.Length)); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(documents.Length)); + Assert.That(await Query(db => db.FailedMessageGroups.CountAsync()), Is.EqualTo(documents.Length)); + } + } + + [Test] + public async Task A_message_already_in_the_target_has_no_body_written_and_one_it_lacks_does() + { + var present = new IngestedFailure { ContentType = "application/octet-stream" }; + var absent = new IngestedFailure { ContentType = "application/octet-stream" }; + var binaryBody = new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = present.UniqueMessageId, Status = FailedMessageStatus.Unresolved, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(RowOf(present.ToFailedMessage(), binaryBody), RowOf(absent.ToFailedMessage(), binaryBody)), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await BodyStored(present.UniqueMessageIdString), Is.False, "the insert declines a message the target holds, so a body written for it is a wasted write"); + Assert.That(await BodyStored(absent.UniqueMessageIdString), Is.True, "the test proves nothing if no body is ever written"); + } + } + + [Test] + public async Task Only_the_first_document_sharing_a_unique_message_id_has_its_body_stored() + { + var first = new IngestedFailure(); + var second = new IngestedFailure { ContentType = "application/octet-stream" }; + var secondDocument = second.ToFailedMessage(); + secondDocument.UniqueMessageId = first.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + RowOf(first.ToFailedMessage(), SmallTextBody), + RowOf(secondDocument, new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"))), Checkpoint()); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == first.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(row.BodyStoredExternally, Is.False, "the insert keeps the first document, whose body is inline"); + Assert.That(await BodyStored(first.UniqueMessageIdString), Is.False, "the declined document's body was stored under the id of the row the insert kept"); + } + } + + [Test] + public async Task A_body_content_type_longer_than_its_column_skips_the_message_and_the_rest_copy() + { + var good = new IngestedFailure(); + var longContentType = new IngestedFailure().ToFailedMessage(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + RowOf(good.ToFailedMessage(), SmallTextBody), + RowOf(longContentType, SmallTextBody with { ContentType = $"text/{new string('x', 446)}" })), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{longContentType.UniqueMessageId}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 1 }), "an over-long content type would fail the whole insert and halt the category"); + } + } + + [Test] + public async Task A_body_that_cannot_be_written_fails_the_write_before_any_row_or_checkpoint_commits() + { + var target = new EFCoreMigrationTarget( + ServiceProvider.GetRequiredService(), + ServiceProvider.GetRequiredService(), + new UnwritableBodyStore(), + ServiceProvider.GetRequiredService(), + ServiceProvider.GetRequiredService>()); + var inline = new IngestedFailure(); + var binary = new IngestedFailure { ContentType = "application/octet-stream" }; + + var exception = Assert.ThrowsAsync(() => target.Write(FailedMessagesCategory, BatchOf( + RowOf(inline.ToFailedMessage(), SmallTextBody), + RowOf(binary.ToFailedMessage(), new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"))), Checkpoint())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{binary.UniqueMessageId}'").And.Contain($"'{binary.UniqueMessageId}'").And.Contain(nameof(IOException)), + "a storage SDK's own error does not name the body, so without the row and body id the operator cannot tell which message halted the category"); + Assert.That(exception.InnerException, Is.TypeOf().And.Message.EqualTo(UnwritableBodyStore.Failure)); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.Zero, "a row committed without its body would name a body that was never stored"); + Assert.That(await CheckpointStore.Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), Is.Null, "a checkpoint past the batch would stop a retry from reading it again"); + } + } + + [Test] + public void A_document_that_cannot_be_prepared_fails_the_write_naming_the_row_and_category() + { + // The source rebuilds SendingEndpoint as EndpointDetails, so a value it left as stored is an engine defect the translator's cast rejects. + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata["SendingEndpoint"] = new Dictionary { ["Name"] = "Stored" }; + + var exception = Assert.ThrowsAsync(() => Target.Write(FailedMessagesCategory, BatchOf(document), Checkpoint())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages).And.Contain(nameof(InvalidCastException)), + "the halt's LastError is built from this message, and without the row the operator cannot tell which document to look at"); + Assert.That(exception.InnerException, Is.TypeOf()); + } + } + + [Test] + public void A_row_that_reaches_the_writer_without_a_body_fails_the_write_naming_the_row_and_category() + { + var document = new IngestedFailure().ToFailedMessage(); + + var exception = Assert.ThrowsAsync(() => Target.Write(FailedMessagesCategory, BatchOf(RowOf(document, body: null)), Checkpoint())); + + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)); + } + + // Disposes the stream, because the file store holds the body file open until it is. + async Task BodyStored(string bodyId) + { + var stored = await BodyStore.ReadBody(bodyId); + if (stored is null) + { + return false; + } + + await stored.Stream.DisposeAsync(); + return true; + } + + async Task Query(Func> query) + { + using var scope = ServiceProvider.CreateScope(); + return await query(scope.ServiceProvider.GetRequiredService()); + } + + static MigrationBatch BatchOf(params FailedMessage[] documents) => BatchOf([.. documents.Select(document => RowOf(document, SmallTextBody))]); + + static MigrationBatch BatchOf(params MigrationRow[] rows) => new(rows, rows[^1].SourceId); + + // Every row carries the two dates the source resolves, as a real source's rows do. + static MigrationRow RowOf(FailedMessage document, MigrationBody body) => + new($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }, body); + + sealed class UnwritableBodyStore : IBodyStoragePersistence + { + public const string Failure = "The body store is unreachable"; + + public Task WriteBody(string bodyId, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) => Task.FromException(new IOException(Failure)); + + public Task ReadBody(string bodyId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task DeleteBodyIfExists(string bodyId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + static MigrationCheckpoint Checkpoint() => + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs index 41cf3f5bd5..f63dbdfdd0 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs @@ -10,7 +10,10 @@ namespace ServiceControl.Persistence.Tests; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; class KnownEndpointsMigrationTargetTests : PersistenceTestBase { @@ -81,6 +84,31 @@ public async Task A_known_endpoint_with_no_name_or_no_host_is_skipped_as_require } } + [Test] + public async Task A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fitsExactly = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = new string('n', 450), HostId = Guid.NewGuid(), Host = new string('h', 450) } }; + var longName = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = new string('n', 451), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var longHost = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = new string('h', 452) } }; + var batch = BatchOf(("KnownEndpoints/1", fitsExactly), ("KnownEndpoints/2", longName), ("KnownEndpoints/3", longHost)); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, CheckpointAfter("KnownEndpoints/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2", "KnownEndpoints/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { new string('n', 450) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 451, on document 'KnownEndpoints/2'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(KnownEndpointEntity.Host))} takes at most 450 characters and the value holds 452, on document 'KnownEndpoints/3'" + })); + } + } + [Test] public async Task A_batch_of_nothing_but_skips_saves_the_cursor_and_writes_no_rows() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index da87ec7dcd..eb7caebdcc 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.Tests; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -13,7 +14,7 @@ public async Task Every_category_the_target_supports_has_a_batch_size_and_a_coun { var target = ServiceProvider.GetRequiredService(); - Assert.That(target.SupportedCategoryIds, Is.SupersetOf(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }), "the test proves nothing if the target supports no category"); + Assert.That(target.SupportedCategoryIds, Is.SupersetOf(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages }), "the test proves nothing if the target supports no category"); foreach (var id in target.SupportedCategoryIds) { @@ -23,4 +24,21 @@ public async Task Every_category_the_target_supports_has_a_batch_size_and_a_coun Assert.That(await target.Count(category, CancellationToken.None), Is.Zero, id); } } + + [Test] + public void Every_category_the_target_supports_has_a_test_for_a_value_longer_than_its_column() + { + var target = ServiceProvider.GetRequiredService(); + + Assert.That(target.SupportedCategoryIds, Is.EquivalentTo(ValueTooLongTests.Keys), + "a writer that does not run the length check sends an over-long value to the insert, which fails the whole statement and halts the category; give the new category a ValueTooLong test like those listed here, then add it to this list"); + } + + // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. + static readonly Dictionary ValueTooLongTests = new() + { + [MigrationCategoryIds.KnownEndpoints] = nameof(KnownEndpointsMigrationTargetTests.A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy) + }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs new file mode 100644 index 0000000000..e04ebaee93 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +static class PreparedSkips +{ + /// + /// Prepares a batch through one writer without writing anything, so a test can read the detail line the target logs for each skip. + /// + public static async Task> DetailsFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService(), batch); + + return [.. prepared.Skips.Select(skip => skip.Detail)]; + } + + /// + /// The table and column a property is stored in, as this provider names them. + /// + public static string TableAndColumnOf(IServiceProvider services, string property) + { + using var scope = services.CreateScope(); + var entityType = scope.ServiceProvider.GetRequiredService().Model.FindEntityType(typeof(TEntity))!; + + return $"{entityType.GetTableName()}.{entityType.FindProperty(property)!.GetColumnName()}"; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs index 7e969ffa5f..d6c40216b4 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -40,7 +40,6 @@ public interface IMigrationSource : IAsyncDisposable /// /// Reads a category in batches, after the checkpoint cursor, or from the start when it is null. Throws on a cursor it never issued. /// - /// A ceiling, not a target: returning fewer costs nothing, returning more fails the target's write. IAsyncEnumerable Read( MigrationCategory category, string? resumeAfter, @@ -48,7 +47,8 @@ IAsyncEnumerable Read( CancellationToken cancellationToken = default); /// - /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + /// Reads one row's message body when Read did not attach it. The engine calls it only for a category that + /// carries bodies, and only for a row whose body Read left null. Returns null when the source has no body for the row. /// Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index 1e457a27a8..e2b93060ab 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -24,8 +24,10 @@ public interface IMigrationTarget /// /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. /// Add this batch's own outcome with and save that, so the stored counts are never a guess. + /// For a category that carries bodies, every row must have its body. + /// Bodies that go to external storage are written first, outside the transaction. So a failed write can leave a + /// stored body that no row points to, but never a row that points to a body that was not stored. /// - /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Nothing the target does is counted yet. Task Write( MigrationCategory category, MigrationBatch batch, @@ -55,10 +57,4 @@ Task Write( /// /// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. /// -/// The checkpoint as the target stored it, carrying the version that save landed on. The engine carries on from this one, never from the one it passed in. -/// Rows this batch wrote. The same number must show up as the rise in the saved copied count, because the halt threshold reads one and status and verify read the other. -/// Rows this batch could not write, benign ones included. -/// The source ids of those rows, so the engine can name each one in the log. -/// Rows the target already held. They were neither copied nor skipped, and they still count as accounted for. -/// How many rows each reason skipped. The counts have to add up to Skipped, or the checkpoint refuses the batch. public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs new file mode 100644 index 0000000000..e8af7f8ab2 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +/// +/// The rows returns for one batch, each with its body, and the rows it left out +/// with the reason for each. +/// +sealed record MigrationBodyLoadResult(IReadOnlyList Rows, IReadOnlyList<(string SourceId, MigrationSkipReason Reason)> Skipped); + +/// +/// Attaches a message body to every row of a batch that arrived without one, reading at most +/// bodies from the source at a time. +/// +sealed class MigrationBodyLoader(IMigrationSource source, TimeProvider timeProvider, TimeSpan retryBackoff, ILogger logger) +{ + /// + /// How many body reads are open against the source at once. Each body is its own round trip, so reading them one at + /// a time makes the copy as slow as the link, while opening a whole batch at once puts a load spike on a source that + /// may still be live. + /// + public const int MaxConcurrentFetches = 8; + + /// + /// Gives every row without a body the body the source holds for it. A row that already has one keeps it and + /// is never read. A read that fails is tried up to times, + /// and a row whose every attempt fails is left out as . A row + /// the source holds no body for is read once and left out as . + /// Each skip is logged as a warning. + /// + public async Task Load(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken = default) + { + using var inFlight = new SemaphoreSlim(MaxConcurrentFetches); + + var fetched = await Task.WhenAll(batch.Rows.Select(async row => + { + if (row.Body is not null) + { + return (row.Body, LastAttemptError: null); + } + + // The slot is held for the whole fetch, retries included, so each message uses one slot rather than one per attempt. + await inFlight.WaitAsync(cancellationToken); + + try + { + return await Fetch(category, row.SourceId, cancellationToken); + } + finally + { + inFlight.Release(); + } + })); + + var rows = new List(batch.Rows.Count); + var skipped = new List<(string SourceId, MigrationSkipReason Reason)>(); + + // Rebuilt in the batch's order, because that is the cursor's order and the fetches finish in any order. + for (var index = 0; index < batch.Rows.Count; index++) + { + var row = batch.Rows[index]; + var (body, lastAttemptError) = fetched[index]; + + if (body is not null) + { + rows.Add(row with { Body = body }); + } + else if (lastAttemptError is null) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId} as {Reason}: the source has no body for it, either because it holds the message without one, which a --migration-retry meets again, or because the message expired during the copy", + row.SourceId, category.Id, MigrationSkipReason.BodyMissing); + skipped.Add((row.SourceId, MigrationSkipReason.BodyMissing)); + } + else + { + logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId} as {Reason}: body unreadable after {MaxAttempts} attempts", row.SourceId, category.Id, MigrationSkipReason.BodyUnreadable, MigrationEngine.MaxBodyReadAttempts); + skipped.Add((row.SourceId, MigrationSkipReason.BodyUnreadable)); + } + } + + return new MigrationBodyLoadResult(rows, skipped); + } + + // A null body with no error means the source holds no body; with an error, every attempt failed. + async Task<(MigrationBody? Body, Exception? LastAttemptError)> Fetch(MigrationCategory category, string sourceId, CancellationToken cancellationToken) + { + Exception? lastAttemptError = null; + + for (var attempt = 1; attempt <= MigrationEngine.MaxBodyReadAttempts; attempt++) + { + try + { + return (await source.ReadBody(category, sourceId, cancellationToken), null); + } + // Only the run's own cancellation stops here; a client timeout raised as a cancellation is retried. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Logged here because the exception that halts the category does not say which message it was reading. + catch (Exception ex) when (IsDefect(ex)) + { + logger.LogError(ex, "Reading the body of {SourceId} in category {CategoryId} failed in a way no retry fixes, so the batch fails", sourceId, category.Id); + throw; + } + catch (Exception ex) + { + lastAttemptError = ex; + logger.LogWarning(ex, "Reading the body of {SourceId} failed on attempt {Attempt} of {MaxAttempts}", sourceId, attempt, MigrationEngine.MaxBodyReadAttempts); + + if (attempt < MigrationEngine.MaxBodyReadAttempts && retryBackoff > TimeSpan.Zero) + { + await Task.Delay(retryBackoff, timeProvider, cancellationToken); + } + } + } + + return (null, lastAttemptError); + } + + // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. + static bool IsDefect(Exception exception) => + exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs index 0a6ae5bb2c..8462c1e454 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs @@ -33,7 +33,13 @@ reason is MigrationSkipReason.PastRetention or MigrationSkipReason.BlankGroupComment; /// - /// Whether no retry can fix a row skipped for this reason, because the row itself can never be stored. + /// Whether no retry can fix a row skipped for this reason. /// - public static bool IsPermanent(this MigrationSkipReason reason) => reason is MigrationSkipReason.RequiredValueMissing; + public static bool IsPermanent(this MigrationSkipReason reason) => + reason is MigrationSkipReason.RequiredValueMissing + or MigrationSkipReason.UnparseableUniqueMessageId + or MigrationSkipReason.NoProcessingAttempts + or MigrationSkipReason.ValueTooLong + or MigrationSkipReason.MalformedValue + or MigrationSkipReason.BodyMissing; } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index bf975e1e21..e13538cdf7 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -24,9 +24,13 @@ public sealed class MigrationEngine( MigrationEngineOptions options, ILogger logger) { - /// How many times a message body is read before the row is skipped as unreadable. + /// + /// How many times a message body is read before the row is skipped as unreadable. + /// public const int MaxBodyReadAttempts = 3; + readonly MigrationBodyLoader bodyLoader = new(source, timeProvider, options.BodyRetryBackoff, logger); + /// /// The categories of one kind to copy, in the order to copy them. Optional categories the operator did not /// ask for are left out. @@ -43,9 +47,6 @@ public IReadOnlyList SelectCategories(MigrationCategoryKind k /// that halts does not stop the ones after it. Before copying anything it saves a not-started checkpoint for /// every category that has none, so a copy stopped between two categories still lists the ones it never reached. /// - /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. - /// The host is shutting down. - /// Saving the checkpoint a category stopped or settled on failed. Its stored row is the last one saved, and the categories after it were not run. // Runs in the order given without re-sorting: required and optional orders both start at 1, so // sorting a mixed list would put an optional category in front of a required one. public async Task> RunCategories( @@ -75,10 +76,6 @@ public async Task> RunCategories( /// Copies one category, carrying on from its saved cursor, and returns the checkpoint it ended on. A category /// already finished or Failed is returned untouched without reading the source. /// - /// The stored checkpoint, whose state says how it ended and whose LastError says why it stopped. - /// Another writer saved this category's checkpoint, which means a second instance is copying into the same database. The state is left as that writer set it. - /// The host is shutting down. The last committed batch saved its own counts, so the stored checkpoint is already correct and a restart carries on from it. - /// Saving the checkpoint the category stopped or settled on failed, such as a halt the store could not write. The stored row is the last one saved, so it still reads as copying from its last committed batch. public async Task RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) { var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); @@ -146,16 +143,13 @@ public async Task RunCategoryAsync(MigrationCategory catego // Kept out of checkpoint until the write commits: the catch below saves checkpoint, and a // restart re-reads the uncommitted batch and would count these skips twice. var bodySkips = 0; + Dictionary? bodySkipReasons = null; if (category.CarriesBodies) { - var (withBodies, failed) = await FetchBodiesWithRetry(category, batch, cancellationToken); - batchToWrite = withBodies; - bodySkips = failed.Count; - - foreach (var (id, lastAttemptError) in failed) - { - logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); - } + var loaded = await bodyLoader.Load(category, batch, cancellationToken); + batchToWrite = batch with { Rows = loaded.Rows }; + bodySkips = loaded.Skipped.Count; + bodySkipReasons = loaded.Skipped.CountBy(skip => skip.Reason).ToDictionary(count => count.Key, count => (long)count.Value); } // The target adds its own outcome inside the transaction that writes the rows, @@ -165,7 +159,7 @@ public async Task RunCategoryAsync(MigrationCategory catego Cursor = batch.Cursor, LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, SkippedCount = checkpoint.SkippedCount + bodySkips, - SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [MigrationSkipReason.BodyUnreadable] = bodySkips }) + SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkipReasons) }; var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); @@ -253,64 +247,6 @@ static long FaultSkips(IReadOnlyDictionary? skipReaso Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); - async Task<(MigrationBatch Batch, IReadOnlyList<(string SourceId, Exception LastAttemptError)> Failed)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) - { - var survivors = new List(batch.Rows.Count); - var failed = new List<(string SourceId, Exception LastAttemptError)>(); - - foreach (var row in batch.Rows) - { - if (row.Body is not null) - { - survivors.Add(row); - continue; - } - - MigrationBody? body = null; - Exception? lastAttemptError = null; - var succeeded = false; - - for (var attempt = 1; attempt <= MaxBodyReadAttempts && !succeeded; attempt++) - { - try - { - body = await source.ReadBody(category, row.SourceId, cancellationToken); - succeeded = true; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - // A shutdown is not a transient body failure. Retrying it to the attempt limit and then - // recording the message as permanently unreadable would lose a row to a restart. - throw; - } - catch (Exception ex) when (!IsDefect(ex)) - { - lastAttemptError = ex; - logger.LogWarning(ex, "Attempt {Attempt} to read the body for {SourceId} failed", attempt, row.SourceId); - if (attempt < MaxBodyReadAttempts) - { - await Pause(options.BodyRetryBackoff, cancellationToken); - } - } - } - - if (succeeded) - { - survivors.Add(row with { Body = body }); - } - else - { - failed.Add((row.SourceId, lastAttemptError!)); - } - } - - return (batch with { Rows = survivors }, failed); - } - - // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. - static bool IsDefect(Exception exception) => - exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; - // Zero means no throttling, and it must return without waiting: against a fake clock // nobody advances, even a zero-length wait never finishes. Task Pause(TimeSpan duration, CancellationToken cancellationToken) => diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs new file mode 100644 index 0000000000..808d5090f6 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// Keys for the extra values a source attaches to a . The source works each value out +/// itself, so the target never needs to know how the source stores it. +/// +public static class MigrationMetadataKeys +{ + /// + /// The instant a failed message moved to its current status, as a UTC . The retention sweep + /// ages a row out from this instant. + /// + public const string StatusChangedAt = "StatusChangedAt"; + + /// + /// The time the source last changed the row's document, as a UTC . + /// + public const string LastModified = "LastModified"; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs index 11d03ffa52..6c4d0fe806 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -7,13 +7,20 @@ namespace ServiceControl.Persistence.DataMigration; /// public enum MigrationSkipReason { - /// The message body could not be read from the source, after the engine had tried more than once. + /// + /// Every attempt to read the message body from the source failed, so the whole message was left out rather than + /// written without one. A retry reads it again once the cause is fixed. + /// BodyUnreadable, - /// The row is older than the retention period, so the instance would have deleted it soon anyway. + /// + /// The row is older than the retention period, so the instance would have deleted it soon anyway. + /// PastRetention, - /// The source row has no value for something the target column requires. + /// + /// The source row has no value for something the target column requires. + /// RequiredValueMissing, /// @@ -21,6 +28,32 @@ public enum MigrationSkipReason /// BlankGroupComment, + /// + /// The document's unique message id is not a GUID, and the row's key has to be one. + /// + UnparseableUniqueMessageId, + + /// + /// The document holds no processing attempt, and every column that describes the failure comes from one. + /// + NoProcessingAttempts, + + /// + /// A value is longer than its target column allows, so storing the row would fail the whole insert. + /// + ValueTooLong, + + /// + /// A value the target needs is present but holds the wrong type, so the row cannot be built from it. + /// + MalformedValue, + + /// + /// The source holds the message but no body for it, so the whole message was left out rather than written + /// without one. + /// + BodyMissing, + /// /// A reason a newer build wrote and this one cannot name, so an older host still starts instead of throwing. /// Nothing in this build writes it. It counts as a fault, because harmless would let this build settle a diff --git a/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs b/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs new file mode 100644 index 0000000000..464ba86b26 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs @@ -0,0 +1,37 @@ +#if DEBUG +namespace ServiceControl.UnitTests.Migration; + +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Migration.Checks; + +[TestFixture] +class AllowUnreleasedMigrationSwitchTests +{ + [TestCase("true")] + [TestCase("TRUE")] + public void The_switch_registers_the_marker_when_the_variable_is_true(string value) + { + var services = new ServiceCollection(); + + RunCommand.AllowUnreleasedMigrationWhenAsked(services, value); + + using var provider = services.BuildServiceProvider(); + Assert.That(provider.GetService(), Is.Not.Null); + } + + [TestCase(null)] + [TestCase("")] + [TestCase("false")] + public void The_switch_registers_nothing_unless_the_variable_is_true(string value) + { + var services = new ServiceCollection(); + + RunCommand.AllowUnreleasedMigrationWhenAsked(services, value); + + using var provider = services.BuildServiceProvider(); + Assert.That(provider.GetService(), Is.Null); + } +} +#endif diff --git a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs index 7a71eb1c2d..9947ca3ca3 100644 --- a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs +++ b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs @@ -104,7 +104,7 @@ public void Unknown_stays_the_last_skip_reason() => public void Exactly_the_reasons_no_retry_can_fix_are_permanent() => Assert.That( Enum.GetValues().Where(reason => reason.IsPermanent()), - Is.EquivalentTo(new[] { MigrationSkipReason.RequiredValueMissing })); + Is.EquivalentTo(new[] { MigrationSkipReason.RequiredValueMissing, MigrationSkipReason.UnparseableUniqueMessageId, MigrationSkipReason.NoProcessingAttempts, MigrationSkipReason.ValueTooLong, MigrationSkipReason.MalformedValue, MigrationSkipReason.BodyMissing })); static MigrationCheckpoint Checkpoint(string categoryId, MigrationCategoryState state) => new(categoryId, state, null, 0, 0, null, null, null, null, null, null); diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs index ebc2dd8253..0d3bbfceb6 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -16,6 +16,10 @@ public sealed class InMemoryMigrationSource : IMigrationSource readonly Dictionary bodyFailuresInTurn = []; readonly Dictionary bodyReadAttempts = []; readonly Dictionary bodies = []; + // The loader reads several bodies at once, so the attempt counts and the in-flight count are shared between threads. + readonly Lock bodyReadGate = new(); + TaskCompletionSource? heldBodyReads; + int bodyReadsInFlight; public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); @@ -35,7 +39,39 @@ public sealed class InMemoryMigrationSource : IMigrationSource /// public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } - public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); + public int BodyReadAttempts(string sourceId) + { + lock (bodyReadGate) + { + return bodyReadAttempts.GetValueOrDefault(sourceId); + } + } + + /// + /// The most body reads that were open at the same moment since the source was created. + /// + public int PeakBodyReadsInFlight { get; private set; } + + /// + /// Makes every body read wait until is called. A waiting read already counts as an attempt and as open. + /// + public void HoldBodyReads() => heldBodyReads = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Lets every held body read carry on, and every later one run without waiting. Call first. + /// + public void ReleaseBodyReads() => heldBodyReads!.SetResult(); + + /// + /// Completes once at least this many body reads are open at the same moment. + /// + public async Task BodyReadsInFlight(int count, CancellationToken cancellationToken = default) + { + while (Volatile.Read(ref bodyReadsInFlight) < count) + { + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken); + } + } public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; @@ -82,25 +118,43 @@ public async IAsyncEnumerable Read( { await Task.Yield(); - var attempt = bodyReadAttempts[sourceId] = BodyReadAttempts(sourceId) + 1; + int attempt; - if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + lock (bodyReadGate) { - await stop.Source.CancelAsync(); - throw new OperationCanceledException(stop.Source.Token); + attempt = bodyReadAttempts[sourceId] = bodyReadAttempts.GetValueOrDefault(sourceId) + 1; + PeakBodyReadsInFlight = Math.Max(PeakBodyReadsInFlight, Interlocked.Increment(ref bodyReadsInFlight)); } - if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) + try { - throw failures.Failure; - } + if (heldBodyReads is { } held) + { + await held.Task; + } + + if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } - if (bodyFailuresInTurn.TryGetValue(sourceId, out var failuresInTurn) && attempt <= failuresInTurn.Length) + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) + { + throw failures.Failure; + } + + if (bodyFailuresInTurn.TryGetValue(sourceId, out var failuresInTurn) && attempt <= failuresInTurn.Length) + { + throw failuresInTurn[attempt - 1]; + } + + return bodies.GetValueOrDefault(sourceId); + } + finally { - throw failuresInTurn[attempt - 1]; + Interlocked.Decrement(ref bodyReadsInFlight); } - - return bodies.GetValueOrDefault(sourceId); } // Every registered category, not just the seeded ones: Seed writes rowsByCategory after construction, diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index 84c171e912..b6ef99b0a1 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -14,6 +14,7 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint readonly Dictionary> rowsHandedToWriteByCategory = []; readonly HashSet preExistingKeys = []; readonly Dictionary rejectedKeys = []; + readonly TaskCompletionSource ignoredCallReleased = new(TaskCreationOptions.RunContinuationsAsynchronously); public int DefaultBatchSize { get; set; } = 3; public string NoBatchSizeFor { get; set; } @@ -46,8 +47,22 @@ public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpoint /// Waits on this call until the token is cancelled and then throws, as a write stuck on a database that stopped answering does. /// public int? HangOnCall { get; set; } + /// + /// Waits on this call until ReleaseIgnoredCall and never looks at the token, then writes without it, as a driver call stuck below the token does. + /// If the store then refuses the checkpoint save, this fake has already recorded the batch's rows, unlike the real target, which rolls both back. + /// + public int? IgnoreTokenOnCall { get; set; } + /// + /// The write that IgnoreTokenOnCall holds, or null until that call is made. It ends only after ReleaseIgnoredCall. + /// + public Task IgnoredCall { get; private set; } int callCount; + /// + /// Lets the write that IgnoreTokenOnCall holds carry on. Calling it again does nothing. + /// + public void ReleaseIgnoredCall() => ignoredCallReleased.TrySetResult(); + public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; @@ -66,32 +81,51 @@ public IReadOnlyList RowsHandedToWrite(string categoryId) => public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : Task.FromResult(DefaultBatchSize); - public async Task Write( + public Task Write( MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) { - callCount++; + var callNumber = ++callCount; BeforeWrite?.Invoke(checkpointToExtend); - if (HangOnCall == callCount) + return IgnoreTokenOnCall == callNumber + ? IgnoredCall = WriteOnceReleased(category, batch, checkpointToExtend, callNumber) + : WriteNow(category, batch, checkpointToExtend, callNumber, cancellationToken); + } + + // The commit ignores the token too, so a late commit meets the store's version check instead of the token. + async Task WriteOnceReleased(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, int callNumber) + { + await ignoredCallReleased.Task; + return await WriteNow(category, batch, checkpointToExtend, callNumber, CancellationToken.None); + } + + async Task WriteNow( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + int callNumber, + CancellationToken cancellationToken) + { + if (HangOnCall == callNumber) { await Task.Delay(Timeout.Infinite, cancellationToken); } - if (StopOnCall is { } stop && stop.CallNumber == callCount) + if (StopOnCall is { } stop && stop.CallNumber == callNumber) { await stop.Source.CancelAsync(); throw new OperationCanceledException(stop.Source.Token); } - if (FailOnCallNumber == callCount) + if (FailOnCallNumber == callNumber) { - throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callCount}"); + throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callNumber}"); } - if (CancelOnCall is { } cancel && cancel.CallNumber == callCount) + if (CancelOnCall is { } cancel && cancel.CallNumber == callNumber) { await cancel.Source.CancelAsync(); } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs new file mode 100644 index 0000000000..201bb885c8 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs @@ -0,0 +1,212 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationBodyLoaderTests +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + static readonly MigrationBody Body = new(Encoding.UTF8.GetBytes("1"), "text/xml"); + + [Test] + public async Task Attaches_the_body_to_every_row() + { + var source = SourceWithBodies("FailedMessages/one", "FailedMessages/two"); + + var result = await Load(source, "FailedMessages/one", "FailedMessages/two"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { "FailedMessages/one", "FailedMessages/two" })); + Assert.That(result.Rows.Select(row => row.Body), Is.All.SameAs(Body)); + Assert.That(result.Skipped, Is.Empty); + } + } + + [Test] + public async Task Retries_a_failing_read_three_times_then_skips_the_message_as_BodyUnreadable() + { + var source = SourceWithBodies("FailedMessages/one", "FailedMessages/two"); + source.FailBodyReads("FailedMessages/two", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + + var result = await Load(source, "FailedMessages/one", "FailedMessages/two"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/two"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { "FailedMessages/one" }), "a message with no body is a retry hazard, so it is left out rather than written without one"); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/two", MigrationSkipReason.BodyUnreadable) })); + } + } + + [Test] + public async Task Keeps_a_message_whose_read_succeeds_on_the_last_attempt() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Single().Body, Is.SameAs(Body)); + Assert.That(result.Skipped, Is.Empty); + } + } + + [Test] + public async Task Skips_a_message_the_source_has_no_body_for_as_BodyMissing_after_one_read() + { + var source = new InMemoryMigrationSource(); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(1), "a read that found nothing learns nothing by repeating"); + Assert.That(result.Rows, Is.Empty); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/one", MigrationSkipReason.BodyMissing) }), + "a missing body is not an unreadable one, and only a separate reason lets the refusal say no retry fixes it"); + } + } + + [Test] + public void Lets_a_defect_out_on_the_first_attempt_rather_than_skipping_the_message() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts, new NotSupportedException("this source cannot read bodies")); + + Assert.ThrowsAsync(() => Load(source, "FailedMessages/one"), + "a defect fails the same way on every attempt, so retrying it and skipping the message would blame the body store for a bug"); + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(1)); + } + + [Test] + public void Logs_a_defect_with_the_message_it_was_reading_before_letting_it_out() + { + var source = SourceWithBodies("FailedMessages/one"); + var defect = new NotSupportedException("this source cannot read bodies"); + source.FailBodyReads("FailedMessages/one", 1, defect); + var logger = new CapturingLogger(); + var batch = new MigrationBatch([new MigrationRow("FailedMessages/one", new object(), new Dictionary())], "FailedMessages/one"); + + Assert.ThrowsAsync(() => new MigrationBodyLoader(source, new FakeTimeProvider(), TimeSpan.Zero, logger).Load(Category, batch, TestContext.CurrentContext.CancellationToken)); + + var error = logger.Entries.Single(entry => entry.Level == LogLevel.Error); + + using (Assert.EnterMultipleScope()) + { + Assert.That(error.Message, Does.Contain("FailedMessages/one").And.Contain(Category.Id), "the exception that halts the category does not say which message it was reading, so this line is the only record of it"); + Assert.That(error.Exception, Is.SameAs(defect)); + } + } + + [Test] + public async Task Retries_a_cancellation_the_run_did_not_ask_for_and_then_skips_the_message_as_BodyUnreadable() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts, new OperationCanceledException("the client timed out")); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts), "a client timeout raised as a cancellation is a transient failure, not a shutdown"); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/one", MigrationSkipReason.BodyUnreadable) })); + } + } + + [Test] + public async Task Keeps_a_body_the_source_attached_and_never_reads_it_again() + { + var source = SourceWithBodies("FailedMessages/two"); + var attached = new MigrationBody(Encoding.UTF8.GetBytes("attached in Read"), "text/plain"); + + var batch = new MigrationBatch( + [ + new MigrationRow("FailedMessages/one", new object(), new Dictionary(), attached), + new MigrationRow("FailedMessages/two", new object(), new Dictionary()) + ], + "FailedMessages/two"); + + var result = await Loader(source).Load(Category, batch, TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.Zero, "a source that attaches bodies in Read would otherwise pay a second read per message for data it already had"); + Assert.That(result.Rows[0].Body, Is.SameAs(attached)); + Assert.That(result.Rows[1].Body, Is.SameAs(Body)); + } + } + + [Test] + public async Task Fetches_at_most_eight_bodies_at_a_time_and_keeps_the_batch_order() + { + var sourceIds = Enumerable.Range(0, 40).Select(index => $"FailedMessages/{index}").ToArray(); + var source = SourceWithBodies(sourceIds); + source.HoldBodyReads(); + + var load = Load(source, sourceIds); + + try + { + using var waitLimit = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await source.BodyReadsInFlight(MigrationBodyLoader.MaxConcurrentFetches, waitLimit.Token); + // Gives an unbounded loader time to open the rest of the forty reads, which it would do all at once. + await Task.Delay(TimeSpan.FromMilliseconds(100)); + + Assert.That(source.PeakBodyReadsInFlight, Is.EqualTo(MigrationBodyLoader.MaxConcurrentFetches), + "an unbounded loader opens all forty reads at once, which turns the copier into a load spike on a live source"); + } + finally + { + source.ReleaseBodyReads(); + } + + var result = await load.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(sourceIds), "the batch's order is the cursor's order, so it has to survive being fetched out of order"); + Assert.That(result.Skipped, Is.Empty); + Assert.That(source.PeakBodyReadsInFlight, Is.EqualTo(MigrationBodyLoader.MaxConcurrentFetches)); + } + } + + static InMemoryMigrationSource SourceWithBodies(params string[] sourceIds) + { + var source = new InMemoryMigrationSource(); + + foreach (var sourceId in sourceIds) + { + source.SetBody(sourceId, Body); + } + + return source; + } + + // Zero backoff because the FakeTimeProvider is never advanced, and a wait on it would never finish. + static MigrationBodyLoader Loader(InMemoryMigrationSource source) => + new(source, new FakeTimeProvider(), TimeSpan.Zero, NullLogger.Instance); + + static Task Load(InMemoryMigrationSource source, params string[] sourceIds) + { + var batch = new MigrationBatch( + [.. sourceIds.Select(sourceId => new MigrationRow(sourceId, new object(), new Dictionary()))], + sourceIds[^1]); + + return Loader(source).Load(Category, batch, TestContext.CurrentContext.CancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs index dbc94ad95a..103d4a2043 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -167,7 +167,13 @@ public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attem await engine.RunCategoryAsync(category); - Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failures[^1])); + var skipped = logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skipped.Exception, Is.SameAs(failures[^1])); + Assert.That(skipped.Message, Does.Contain(nameof(MigrationSkipReason.BodyUnreadable)), "the refusal counts skips by reason name, so the log line has to carry the same name for the operator to find them"); + } } [Test] @@ -235,6 +241,13 @@ public async Task A_body_store_outage_across_many_messages_halts_the_category() var checkpoint = await engine.RunCategoryAsync(category); - Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("exceeds the configured threshold"), + "the end-of-run accounting check halts a category whose body skips were never counted, so the state alone cannot tell the threshold fired"); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 200 }), + "the first batch's 100 skips only reach the 100-row floor, so the second batch's are what pass it"); + } } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs index 59b53c6a07..01dd62c96f 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs @@ -14,7 +14,8 @@ namespace ServiceControl.UnitTests.Migration; [TestFixture] class MigrationEngineCollisionTests { - static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary()); + // A body already attached, so the engine reads none and every row reaches the target. + static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary(), new MigrationBody(new byte[] { 1 }, "application/octet-stream")); [Test] public async Task Rows_already_present_in_the_target_are_counted_as_already_present_across_batches_not_skipped_or_overwritten() diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs index 0ece00a601..9e3a292a46 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -14,7 +14,8 @@ namespace ServiceControl.UnitTests.Migration; [TestFixture] class MigrationEngineHaltTests { - static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + // A body already attached, so the engine reads none and a row is lost only where a test rejects it. + static MigrationRow Row(string id) => new(id, new object(), new Dictionary(), new MigrationBody(new byte[] { 1 }, "application/octet-stream")); [Test] public async Task A_systemic_failure_halts_the_category_partway_through() diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 374a06de68..4053d7d71b 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -47,7 +47,7 @@ public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnread { var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; var source = new InMemoryMigrationSource(); - source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.Seed(category.Id, Row("msg-1"), Row("msg-2") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }); source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); var checkpointStore = new InMemoryMigrationCheckpointStore(); var target = new InMemoryMigrationTarget(checkpointStore); @@ -59,6 +59,32 @@ public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnread Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); } + [Test] + public async Task A_message_the_source_holds_no_body_for_is_counted_under_BodyMissing() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "msg-1 is the only skip, so a missing body has to count as a fault on its own"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyMissing] = 1 }), + "the refusal tells the operator which skips no retry can fix only from these reasons"); + Assert.That(logger.Entries.Where(entry => entry.Message.StartsWith("Skipped msg-1")).Select(entry => entry.Message), Has.Exactly(1).Contains("BodyMissing"), + "the checkpoint holds counts, not ids, so the log is the operator's only record of which message was left out"); + } + } + [Test] public async Task A_batch_that_loses_rows_two_different_ways_records_both_reasons() { @@ -66,7 +92,7 @@ public async Task A_batch_that_loses_rows_two_different_ways_records_both_reason // summing would leave the total right and the reasons wrong, and verification would still balance. var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; var source = new InMemoryMigrationSource(); - source.Seed(category.Id, Row("msg-1"), Row("msg-2"), Row("msg-3")); + source.Seed(category.Id, Row("msg-1"), Row("msg-2") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }, Row("msg-3") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }); // One the engine skips itself because the body will not read, one the target refuses. source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); var checkpointStore = new InMemoryMigrationCheckpointStore(); diff --git a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs index 2077493bff..91945ae6e9 100644 --- a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs +++ b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs @@ -125,11 +125,12 @@ public void Every_failed_required_category_is_named_with_its_skips_and_both_comm var withErrors = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) with { CopiedCount = 40, - SkippedCount = 5, + SkippedCount = 6, SkipReasons = new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2, - [MigrationSkipReason.BodyUnreadable] = 3 + [MigrationSkipReason.BodyUnreadable] = 3, + [MigrationSkipReason.BodyMissing] = 1 } }; var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.MessageRedirects) with { LastError = "TimeoutException at cursor r-7: the target did not answer" }; @@ -146,9 +147,10 @@ public void Every_failed_required_category_is_named_with_its_skips_and_both_comm } Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.MessageRedirects} is Failed (Halted)").And.Contain("the target did not answer")); - Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages} is Failed (CompleteWithErrors) after copying 40 and skipping 5")); + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages} is Failed (CompleteWithErrors) after copying 40 and skipping 6")); Assert.That(exception.Message, Does.Contain("RequiredValueMissing 2, no retry can fix"), "a row no retry can store is a reason to abandon, and the operator has to be told which"); Assert.That(exception.Message, Does.Contain("BodyUnreadable 3").And.Not.Contain("BodyUnreadable 3, no retry can fix"), "an unreadable body may read on the next try"); + Assert.That(exception.Message, Does.Contain("BodyMissing 1, no retry can fix"), "a body the source does not hold is missing again on every retry"); } } @@ -234,6 +236,49 @@ public void A_failed_category_whose_faults_no_retry_can_fix_is_pointed_at_abando }); } + [Test] + public void A_halted_category_whose_faults_no_retry_can_fix_is_told_abandon_gives_up_every_row_not_yet_read() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary { [MigrationSkipReason.MalformedValue] = 120 }) + with { LastError = "Halted: 120 of 127 rows skipped in this run exceeds the configured threshold of 5% and 100 rows." }], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("stopped before the end").And.Contain("every row not yet read"), + "a halted category never read the rows after its cursor, so abandon gives up far more than the skipped ones"); + Assert.That(entry.Message, Does.Contain(MigrationSettings.HaltThresholdPercentKey).And.Contain(MigrationSettings.HaltThresholdMinimumKey) + .And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}"), "raising the threshold and retrying copies every row the faults do not stop"); + Assert.That(entry.Message, Does.Contain("No retry can fix the MalformedValue ones")); + Assert.That(entry.Message, Does.Not.Contain("gives up the rest")); + }); + } + + [Test] + public void A_halted_category_with_faults_a_retry_can_fix_is_also_told_what_abandon_gives_up() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary { [MigrationSkipReason.BodyUnreadable] = 120 })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("every row not yet read").And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}")); + Assert.That(entry.Message, Does.Not.Contain("No retry can fix"), "an unreadable body may read on the next try"); + }); + } + [Test] public void A_category_that_failed_with_only_harmless_skips_is_still_offered_a_retry() { @@ -359,7 +404,7 @@ public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() { var logger = new CapturingLogger(); - var failedEarlier = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 2, new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 }) + var failedEarlier = Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 2, new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 }) with { SettledAt = RunStartedAt.AddMinutes(-5) }; diff --git a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs index 2f3b033642..4f5ff54295 100644 --- a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs +++ b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs @@ -55,6 +55,127 @@ public async Task A_stall_settles_the_stalled_category_failed_and_the_next_categ } } + [Test] + public async Task A_stall_in_a_call_that_ignores_its_token_still_settles_the_category_failed_without_waiting_for_the_call() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.CategoryId), Is.EqualTo(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects })); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].SettledAt, Is.EqualTo(clock.GetUtcNow().UtcDateTime)); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "one stuck call ended the whole start"); + } + + target.ReleaseIgnoredCall(); + + Assert.That(async () => await target.IgnoredCall, Throws.InstanceOf(), + "the stuck call saves from the version it read before the stall, so the store must refuse it"); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0]), "a late commit from the stuck call reopened a settled category"); + } + finally + { + target.ReleaseIgnoredCall(); + } + } + + [Test] + public async Task A_stalled_call_that_commits_while_the_stall_is_being_settled_still_leaves_the_category_failed() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + // The stuck call commits after the settle has read the row and before it saves, so the settle's save is from an old version. + store.BeforeUpsert = async checkpoint => + { + if (checkpoint.State == MigrationCategoryState.Halted) + { + target.ReleaseIgnoredCall(); + await target.IgnoredCall; + } + }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].CopiedCount, Is.EqualTo(1), "the settle must keep the batch the stuck call committed"); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes")); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0])); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "a late commit from our own stuck call ended the whole start"); + } + } + finally + { + target.ReleaseIgnoredCall(); + } + } + + [Test] + public async Task A_host_stopping_during_a_call_that_ignores_its_token_propagates_without_waiting_for_the_call() + { + var store = new InMemoryMigrationCheckpointStore(); + using var host = new CancellationTokenSource(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + var clock = new FakeTimeProvider(); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger(), host.Token)); + + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await host.CancelAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(async () => await run.WaitAsync(TimeSpan.FromSeconds(10)), Throws.InstanceOf(), + "a shutdown waited for a call that ignores its token"); + Assert.That((await store.Read(MigrationCategoryIds.KnownEndpoints))!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown settled the category"); + Assert.That((await store.Read(MigrationCategoryIds.MessageRedirects))!.State, Is.EqualTo(MigrationCategoryState.NotStarted), "a stopping host started the next category"); + } + } + finally + { + target.ReleaseIgnoredCall(); + } + } + [Test] public async Task A_host_stopping_mid_category_leaves_it_copying_and_the_stop_propagates() { @@ -173,6 +294,11 @@ sealed class CancellationHonouringCheckpointStore : IMigrationCheckpointStore { readonly InMemoryMigrationCheckpointStore inner = new(); + /// + /// Runs before each save that gets past the token check, with the checkpoint about to be saved. + /// + public Func? BeforeUpsert { get; set; } + public Task> ReadAll(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -185,10 +311,16 @@ public Task> ReadAll(CancellationToken cancel return inner.Read(categoryId, cancellationToken); } - public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - return inner.Upsert(checkpoint, cancellationToken); + + if (BeforeUpsert is not null) + { + await BeforeUpsert(checkpoint); + } + + return await inner.Upsert(checkpoint, cancellationToken); } } } diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs index 436431f6f8..45cf63cc1c 100644 --- a/src/ServiceControl/Hosting/Commands/RunCommand.cs +++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs @@ -15,10 +15,32 @@ using ServiceControl.Hosting.Auth; using ServiceControl.Hosting.Https; using ServiceControl.Migration; +#if DEBUG + using Microsoft.Extensions.Logging; + using ServiceControl.Infrastructure; + using ServiceControl.Migration.Checks; +#endif using ServicePulse; class RunCommand : AbstractCommand { +#if DEBUG + internal const string AllowUnreleasedMigrationVariable = "SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED"; + + /// + /// When is "true" in any letter case, registers + /// so a Debug build can copy past the release check for a developer's smoke test, and logs a warning so the bypass shows in the log. + /// + internal static void AllowUnreleasedMigrationWhenAsked(IServiceCollection services, string variableValue) + { + if (string.Equals(variableValue, "true", StringComparison.OrdinalIgnoreCase)) + { + services.AddSingleton(); + LoggerUtil.CreateStaticLogger().LogWarning("{Variable} is true, so this Debug build skips the check that the build carries the whole migration and will copy for real", AllowUnreleasedMigrationVariable); + } + } +#endif + public override Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) => Run(settings, customize: null, cancellationToken); @@ -44,6 +66,10 @@ internal static async Task Run(Settings settings, Action customize?.Invoke(hostBuilder); +#if DEBUG + AllowUnreleasedMigrationWhenAsked(hostBuilder.Services, Environment.GetEnvironmentVariable(AllowUnreleasedMigrationVariable)); +#endif + if (settings.MigrationEnabled) { hostBuilder.Services.AddHostedService(provider => new RequiredCopyBeforeTheHostOpens(provider, settings)); diff --git a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs index 68ab286716..6156b0bfc5 100644 --- a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs +++ b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs @@ -6,7 +6,9 @@ namespace ServiceControl.Migration.Checks; using ServiceControl.Persistence.DataMigration; /// -/// Registered by a test host that needs a copy to run on a build that does not yet carry the whole migration. +/// Registered by a test host, or by a Debug build whose SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED environment +/// variable is true, to let a copy run on a build that does not yet carry the whole migration. A Release build +/// has no way to register it. /// class AllowUnreleasedMigration; diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs index dc17bab1d1..f906c8d5b9 100644 --- a/src/ServiceControl/Migration/MigrationStartup.cs +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -32,9 +32,6 @@ internal static IReadOnlyCollection CopyableCategoryIds(IReadOnlyCollect /// the host. Call it after the host is built and before it starts, because the copy has to finish before /// anything else opens on the target. /// - /// The built host's services, which is where the target, the checkpoint store and the migration state come from. - /// The instance settings, read for the source and target persistence types. - /// Cancelled when the host is shutting down, which ends the copy without a refusal message. public static async Task RunRequiredCopy(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) { await MigrationStartupCheckRunner.Run( @@ -141,9 +138,6 @@ static async Task SeedMigrationState(IServiceProvider services, IReadOnlyCollect /// them. The order is what the operator sees: a check that costs nothing comes before one that connects to a /// database, and the source's own checks run last because they need it open. /// - /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. - /// The options read from the settings, which the window check parsed on its way past. - /// A check refused. The message names the check and says what to do, and this start has copied nothing. public static async Task RunChecksAndOpen(IServiceProvider services, Settings settings, IMigrationSource source, CancellationToken cancellationToken = default) { var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); @@ -160,12 +154,6 @@ public static async Task RunChecksAndOpen(IServiceProvid /// opening the target. A check that costs nothing comes before one that connects to a database. The target /// is open when this returns, and the source has not been touched. /// - /// The built host's services, which is where the target and its readiness checks come from. - /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. - /// Cancelled when the host is shutting down, which stops the checks without a refusal message. - /// The options read from the settings, which the window check parsed on its way past. - /// A check refused or the target did not open. The message names the check and says what to do, and this start has copied nothing. - /// The host is shutting down. public static async Task RunChecksAndOpenTarget(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) { var target = services.GetRequiredService(); @@ -191,11 +179,6 @@ .. readiness.ContributedChecks(), /// a category with no row gets a new not-started row carrying the error, and a row that is Done, Failed or /// Abandoned is left alone. The next start that resumes a row clears the error. /// - /// Where the rows are read and saved. - /// The optional categories this instance selected. - /// Why the source could not be opened. Its type and message go into the error. - /// Cancelled when the host is shutting down. - /// Another writer saved one of these rows since it was read. internal static async Task RecordSourceOutage(IMigrationCheckpointStore checkpointStore, IReadOnlyList optionalCategories, Exception exception, CancellationToken cancellationToken = default) { var error = $"The {PersistenceFactory.MigrationSourcePersistenceType} migration source could not be opened: {exception.GetType().Name}: {exception.Message.TrimEnd('.', ' ')}. This category stays as it is, and the next start tries the source again."; @@ -221,15 +204,6 @@ internal static async Task RecordSourceOutage(IMigrationCheckpointStore checkpoi /// stopped and settled Halted, and the categories after it still get their go. Before copying anything it saves /// a not-started checkpoint for every category that has none, as does. /// - /// Copies each category. - /// The categories to copy, in the order to copy them. - /// Where each category's row is seeded, watched and, after a stall, settled. - /// The clock the watchdog measures a stall on. - /// Receives the watchdog's progress lines and the stall. - /// The host's token. Cancelling it stops the copy and leaves the running category as its last committed batch left it, so the next start resumes it. - /// The checkpoint each category ended on. - /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. - /// The host is shutting down. internal static async Task> RunRequiredCategories( MigrationEngine engine, IReadOnlyList categories, @@ -255,40 +229,60 @@ internal static async Task> RunRequiredCatego { using var categoryCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); progress.Watch(category.Id, timeProvider.GetUtcNow().UtcDateTime, categoryCancellation); + var run = engine.RunCategoryAsync(category, categoryCancellation.Token); try { - results.Add(await engine.RunCategoryAsync(category, categoryCancellation.Token)); + // Ends the wait on a stall or a host stop even when a call underneath ignores its token. That call keeps running, and the version check refuses its late commit or the settle below re-reads past it. + results.Add(await run.WaitAsync(categoryCancellation.Token)); } // Only the watchdog cancels this source without the host's token, so the filter tells a stall from a shutdown. #pragma warning disable PS0020 catch (OperationCanceledException) when (categoryCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) #pragma warning restore PS0020 { - // The host's token, because the stall has already cancelled the category's and a store call on that would fail. - var stalled = await checkpointStore.Read(category.Id, cancellationToken); + MigrationCheckpoint settled; - results.Add(await checkpointStore.Upsert(stalled with + try { - State = MigrationCategoryState.Halted, - SettledAt = timeProvider.GetUtcNow().UtcDateTime, - LastError = StallExplanation(category.Id) - }, cancellationToken)); + settled = await SettleStalled(checkpointStore, category.Id, timeProvider, cancellationToken); + } + // The stalled call can commit between the settle's read and its save, so one re-read settles from that commit. A second conflict is another instance, and it propagates. + catch (MigrationCheckpointConflictException) + { + settled = await SettleStalled(checkpointStore, category.Id, timeProvider, cancellationToken); + } + + results.Add(settled); + + _ = run.ContinueWith( + ended => logger.LogWarning(ended.Exception.GetBaseException(), + "{CategoryId}: the call the stall stopped waiting for ended after the category was settled Halted, with {ExceptionType}. It changed nothing after the settle, because the checkpoint store refuses a save made from before it.", + category.Id, ended.Exception.GetBaseException().GetType().Name), + cancellationToken, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); } } return results; } + // Takes the host's token, because the stall has already cancelled the category's and a store call on that would fail. + static async Task SettleStalled(IMigrationCheckpointStore checkpointStore, string categoryId, TimeProvider timeProvider, CancellationToken cancellationToken) + { + var stalled = await checkpointStore.Read(categoryId, cancellationToken); + + return await checkpointStore.Upsert(stalled with + { + State = MigrationCategoryState.Halted, + SettledAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = StallExplanation(categoryId) + }, cancellationToken); + } + /// /// Waits for the copy and turns a second instance writing checkpoints to the same database into a refusal the /// operator can act on. A shutdown and every other failure come out as they are. /// - /// The copy, already running. - /// Read for the persistence type the refusal names. - /// What the copy returned. - /// A checkpoint saved by another instance. The message says what to do and how to go back. - /// The host is shutting down. #pragma warning disable PS0018 // The copy it waits on already runs under the host's token, so a token here would have nothing to cancel. internal static async Task> CopyOrExplainWhyItStopped( Task> copy, @@ -313,7 +307,6 @@ internal static async Task> CopyOrExplainWhyI /// Throws unless every attempted category finished. A category that reported no checkpoint at all counts as /// outstanding too, because nothing says how much of it was copied. /// - /// A category did not finish. The message names each one with its state and counts, names the commands that move each Failed one on, and says how to go back. internal static void RefuseIfAnyCategoryDidNotComplete( IReadOnlyList attempted, IReadOnlyList finished, @@ -344,9 +337,6 @@ internal static void RefuseIfAnyCategoryDidNotComplete( /// Says whether the host may open without the source: every category this start copies is finished, and no row /// outside the copy is unfinished. /// - /// Every checkpoint row in the target. - /// The categories this start copies. - /// True when nothing required is still outstanding. internal static bool RequiredCopyIsSettled(IReadOnlyList checkpoints, IReadOnlyList toCopy) => UnfinishedRowsOutsideTheCopy(checkpoints, toCopy).Count == 0 && toCopy.All(category => checkpoints.Any(checkpoint => checkpoint.CategoryId == category.Id && checkpoint.State.IsFinished())); @@ -355,11 +345,6 @@ internal static bool RequiredCopyIsSettled(IReadOnlyList ch /// Throws unless every attempted category finished and no row outside the copy is unfinished, naming each /// outstanding row in the same words. /// - /// The categories this start copied. - /// Where each attempted category ended. - /// The rows found. - /// Read for the persistence type the refusal names. - /// Something is outstanding. The message names it and says how to go back. internal static void RefuseIfAnyCategoryDidNotComplete( IReadOnlyList attempted, IReadOnlyList finished, @@ -372,9 +357,6 @@ internal static void RefuseIfAnyCategoryDidNotComplete( /// required because a newer build may have written it for a category that must finish, and a required row for a /// category this build cannot copy yet. A finished row, and a row known to be optional, hold nothing. /// - /// Every checkpoint row in the target. - /// The categories this start copies, whose rows the copy judges itself. - /// The rows that keep the host closed, in the order they were read. internal static IReadOnlyList UnfinishedRowsOutsideTheCopy(IReadOnlyList checkpoints, IReadOnlyList toCopy) => [.. checkpoints .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) @@ -402,7 +384,6 @@ static string LastErrorClause(MigrationCheckpoint checkpoint) => /// Logs one line per category saying what it copied and what it left behind. Skipped rows are warned about /// one at a time while the copy runs, and nothing else states the total or says what becomes of them. /// - /// When this start began, which is what tells a category this run finished from one an earlier run did. internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList finished, ILogger logger, DateTime runStartedAt) { foreach (var checkpoint in finished) @@ -455,19 +436,26 @@ internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList !reason.IsBenign()).ToArray() ?? []; var permanent = faults.Where(reason => reason.IsPermanent()).ToArray(); + var cannotFix = permanent.Length == 0 ? "" : $" No retry can fix the {string.Join(" or ", permanent)} ones."; + + // A Halted category never read the rows after its cursor, so abandoning it gives up far more than the skipped rows. + if (faults.Length > 0 && checkpoint.State == MigrationCategoryState.Halted) + { + return $"The copy stopped before the end of the category, so these and every row not yet read stay only in the source database. " + + $"If it stopped because the skips passed the halt threshold, raise {MigrationSettings.HaltThresholdPercentKey} or {MigrationSettings.HaltThresholdMinimumKey}, or fix their cause, then run --migration-retry {checkpoint.CategoryId}, which copies the category again from the start.{cannotFix} " + + $"--migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up every row not yet read, not only these."; + } if (faults.Length > 0 && permanent.Length == faults.Length) { return $"They stay only in the source database, and no retry can fix them, so --migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up the rest."; } - var cannotFix = permanent.Length == 0 ? "" : $" No retry can fix the {string.Join(" or ", permanent)} ones."; - return $"They stay only in the source database until --migration-retry {checkpoint.CategoryId} re-reads them once the cause is fixed.{cannotFix}"; } @@ -521,9 +509,6 @@ public ClosedWindowProgress(IMigrationCheckpointStore checkpointStore, TimeProvi /// Points the watchdog at the category that is starting now, in place of the one before it. Only this /// category is judged from here on, from the later of its last committed batch and . /// - /// The category starting now. - /// When this start began running it. A row an earlier start left carries an older stamp, which would read as a stall at once. - /// Cancelled when the category stalls. The category must run under its token, and nothing else should. public void Watch(string categoryId, DateTime runStartedAt, CancellationTokenSource stop) => running = new RunningCategory(categoryId, runStartedAt, stop); From 5a6d619228a99627b689a2f5988bfd112c5bfb6d Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Thu, 12 Nov 2026 14:16:27 +0100 Subject: [PATCH 18/28] Migration engine: Copy licensing throughput history to SQL (#6063) * Add licensing data migration * Update migration docs * Clarify licensing endpoint handling in migration overview documentation * Enhance licensing endpoints and throughput readers to support ordinal document-id order and cursor resumption; add tests for endpoint reading behavior. * Refactor licensing throughput handling to aggregate counts for endpoints with names differing only in case and trailing spaces; update related tests and documentation. * Enhance SQL Server migration dialect to support key comparison ignoring trailing spaces; introduce PaddedKeyComparer for improved equality checks. * Enhance RetentionSweepTests to support cancellation during database operations; add delay handling for command execution. * Clarify open decisions in migration documentation; update references to gaps and data handling in SQL migration process. --- .../ravendb-to-sql-migration-overview.md | 84 ++++--- .../ravendb-to-sql-migration-system-design.md | 18 +- .../FakeBrokerThroughputQuery.cs | 97 +++++++++ .../LicensingEndpointsMigrationTests.cs | 124 +++++++++++ .../LicensingSettingsMigrationTests.cs | 98 +++++++++ .../LicensingThroughputMigrationTests.cs | 205 ++++++++++++++++++ .../MigrationAcceptanceTest.cs | 68 ++++++ ...ughput_collector_runs_after_a_migration.cs | 47 ++++ .../PostgreSqlMigrationSqlDialect.cs | 27 +++ .../SqlServerMigrationSqlDialect.cs | 52 ++++- .../DataMigration/EFCoreMigrationTarget.cs | 6 +- .../LicensingEndpointRowMapper.cs | 29 +++ .../LicensingThroughputRowMapper.cs | 23 ++ .../DataMigration/SettingRowMapper.cs | 13 ++ .../Writers/LicensedEndpointDetailsWriter.cs | 31 +++ .../Writers/LicensingEndpointsWriter.cs | 54 +++++ .../Writers/LicensingReportMasksWriter.cs | 30 +++ .../Writers/LicensingThroughputWriter.cs | 85 ++++++++ .../Implementation/LicensingDataStore.cs | 2 +- .../Infrastructure/IMigrationSqlDialect.cs | 9 + .../Infrastructure/MigrationInsert.cs | 16 +- .../DataMigration/RavenMigrationSource.cs | 6 +- .../Readers/LicensedEndpointDetailsReader.cs | 47 ++++ .../Readers/LicensingEndpointsReader.cs | 19 ++ .../Readers/LicensingReportMasksReader.cs | 47 ++++ .../Readers/LicensingThroughputReader.cs | 56 +++++ .../DataMigration/ThroughputDocuments.cs | 112 ++++++++++ .../Throughput/LicensingDataStore.cs | 8 +- .../LicensingEndpointsSourceTests.cs | 95 ++++++++ .../LicensingSettingsSourceTests.cs | 119 ++++++++++ .../LicensingThroughputSourceTests.cs | 125 +++++++++++ .../RavenMigrationSourceTestBase.cs | 2 + .../RavenMigrationSourceTests.cs | 16 +- ...nsedEndpointDetailsMigrationTargetTests.cs | 29 +++ .../LicensingEndpointsMigrationTargetTests.cs | 63 ++++++ .../LicensingMigrationMapperCoverageTests.cs | 51 +++++ ...icensingReportMasksMigrationTargetTests.cs | 29 +++ ...LicensingThroughputMigrationTargetTests.cs | 193 +++++++++++++++++ .../Migration/MigrationInsertDateKeyTests.cs | 36 +++ .../Migration/MigrationTargetCoverageTests.cs | 22 +- .../EFCore/Migration/SettingRowMapperTests.cs | 55 +++++ .../DataMigration/ThroughputDayRow.cs | 10 + .../ServiceControl.Persistence.csproj | 1 + 43 files changed, 2209 insertions(+), 50 deletions(-) create mode 100644 src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index b6a81fa023..9c4bbb470e 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -20,7 +20,7 @@ This page describes the migration as designed, not what is built today. [The ins - **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. - **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. - **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. -- **The main thing that makes the outage long is a big backlog of unresolved failures.** They almost never age out (see the [6.18 exception](#skipped-rows)), they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They almost never age out (see the [6.18 exception](#rows-removed-after-the-copy)), they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. - **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds the unresolved failures through a RavenDB index, so while ServiceControl is closed it reads only them. If the index is still catching up, the copy waits for it first. If the index is broken, or leaves out some failed messages, the copy reads every failed message instead, archived and resolved ones included, and logs a warning saying so. Then a big archive does make the outage longer. ```mermaid @@ -161,7 +161,7 @@ There are eighteen *categories*: sixteen required and two optional. | Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been. RavenDB sends them in no particular order anyway, so no ordering is lost | | Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts | | Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | -| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. A comment on a group whose messages are all archived or resolved is removed by ServiceControl's usual clean-up soon after it opens, because that group has no failed messages in SQL yet | +| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. Some comments are removed after ServiceControl opens, see [Open decisions](#rows-removed-after-the-copy) | | Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | | Archived and resolved failed messages, with bodies | Optional | The biggest category by far, and most of the copying time | @@ -172,12 +172,12 @@ There are eighteen *categories*: sixteen required and two optional. - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. - The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. - Broker and audit service version details, which refill on the next throughput collection. -- Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if it fails again: on RavenDB it can then never be edited again, and after the move it can be edited once more. +- Failed message edit locks, which stop one failed message being edited twice. See [Open decisions](#values-that-change-or-merge-in-sql). - Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. **Written new by the copy rather than copied:** -- Retry claims in `FailedMessageRetries` are not copied. A claim ties a message to the retry that sent it, and while it exists, retrying the message from the failed messages list sends nothing. The copy writes a new claim for each retry-issued message it inserts, even one whose claim RavenDB had already released. As on RavenDB, the claim goes when the retried message succeeds, when it fails again, or when it is resent from Pending Retries. +- Retry claims in `FailedMessageRetries` are not copied. A claim ties a message to the retry that sent it, and while it exists, retrying the message from the failed messages list sends nothing. The copy writes a new claim for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). As on RavenDB, the claim goes when the retried message succeeds, when it fails again, or when it is resent from Pending Retries. ### Optional data windows @@ -189,17 +189,8 @@ There are eighteen *categories*: sixteen required and two optional. ### Data that changes shape -- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so it keeps the newest attempt and the number of attempts. A message that failed five times still shows five attempts and still shows as a repeated failure, but only the newest attempt keeps its headers and exception. This applies to every failed message, whatever its status. -- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. -- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. - - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. - - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. - - The copy logs a warning naming both endpoints and the one it kept. - - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. -- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. - **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. - -The dry run counts both kinds of merge before anything moves. +- **Other values change or merge on the way into SQL.** Each is listed under [Open decisions](#values-that-change-or-merge-in-sql). ### Skipped rows @@ -214,7 +205,7 @@ A skip is one of three kinds: - **Fault, no retry can fix it.** The RavenDB document itself cannot become a SQL row. A retry reads the same document and skips it again, so only [abandoning](#retry-and-abandon) gets past it. It is a fault because the row is lost, not because anything is broken. - **Harmless.** A row SQL would have removed anyway. -A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Data that changes shape](#data-that-changes-shape). +A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Open decisions](#values-that-change-or-merge-in-sql). **Fault skips a retry can fix:** @@ -249,7 +240,7 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied - **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. - - The exception: a message that failed again after being archived or resolved on version 6.18 or earlier kept its old expiry, and upgrading does not remove it. So a few unresolved messages can still expire during the copy. + - The exception is an unresolved message from version 6.18 or earlier, which can still expire during the copy. See [Open decisions](#rows-removed-after-the-copy). - A failed message that expires after the copy reads it, but before the copy reads its body, is a skip. It counts as a body RavenDB does not hold, so it is a fault skip and the category ends Failed. For unresolved messages this happens only with a source last written by version 6.18 or earlier. A retry gets past it, because the expired message is no longer there to read. - If the copy stops, and the message it stopped at expires before the next start, that start cannot carry on. The category stops with an error saying RavenDB holds no document with that id. If RavenDB is the right database, `--migration-retry` gets past it, because a retry reads the category from the start. - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. @@ -298,11 +289,11 @@ flowchart LR ### Writing to SQL -- **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, as described in [Data that changes shape](#data-that-changes-shape). +- **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). - **A failed message keeps its failure groups and its last-modified time.** `LastModified` is RavenDB's own `@last-modified`, copied as it is, so failure groups, and lists filtered by when a message last changed, show the same messages as before. - **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. - **`StatusChangedAt` is rebuilt.** - - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. `@expires` is the status-change time plus `ServiceControl/ErrorRetentionPeriod`, so the copy subtracts the current value. If you changed that setting since RavenDB stamped a message, its rebuilt time is off by the difference. It still ages out of SQL when RavenDB would have deleted it, because SQL adds the same current value back when it checks the row against the retention period. + - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. `@expires` is the status-change time plus `ServiceControl/ErrorRetentionPeriod`, so the copy subtracts the current value. A changed retention period makes this time wrong, see [Open decisions](#values-that-change-or-merge-in-sql). - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. - **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. - **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier decides inline or external storage with the same rule ingestion uses, because `IBodyStoragePersistence` does not apply it. The size limit for inline storage defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. @@ -369,12 +360,9 @@ A category is *Failed* when something went wrong. Everything already copied stay - **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. - **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. -**What the checks cannot see, and accept:** +**The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. -- **Nothing counts RavenDB during the copy.** So a read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read (see [the read-only commands](#the-dry-run)). -- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Tests on each reader cover those. -- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. -- **The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. +Some losses leave a category Done without any check noticing. They are listed under [Open decisions](#what-the-checks-do-not-catch). ### Retry and abandon @@ -418,7 +406,7 @@ Stop ServiceControl, run one of these, then start it again. In a container, run - **A worker or import that gets past the check marks the target as opened**, whether or not its own migration setting is on. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. > [!IMPORTANT] -> **Start workers only after the main host has opened.** One started at the same moment as the first migration start can slip in before the copy has written its first checkpoint, and nothing stops it. +> **Start workers only after the main host has opened.** Nothing stops one started at the same moment as the first migration start, see [Open decisions](#gaps-nothing-enforces). ### Going back to RavenDB @@ -428,7 +416,7 @@ Stop ServiceControl, run one of these, then start it again. In a container, run - **To try again later:** start from a new, empty SQL database (`--setup`). > [!IMPORTANT] -> Never reuse the old copy. It would skip every category already finished and miss anything RavenDB received since, and nothing can detect the reuse, because it looks exactly like a normal resume. +> Never reuse the old copy. It would skip every category already finished and miss anything RavenDB received since. Nothing detects the reuse, see [Open decisions](#gaps-nothing-enforces). > > **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. @@ -479,7 +467,7 @@ It says nothing about load on the source. - **RavenDB against the rows the copy read, never against SQL as it is now.** Once ServiceControl opens, normal use changes most categories' SQL rows within minutes: ingestion, heartbeats, retries, archiving, the licensing collectors, ServiceControl's own custom checks and the retention sweep. A SQL count then measures that use, not the copy. - **RavenDB holding more rows than the copy read means rows were never read.** Nothing writes to RavenDB once the copy starts, so it can only shrink. Verify flags these rows. They are not in SQL, so keep RavenDB. The same flag shows if something other than this migration wrote to RavenDB since, such as an old instance still running on it. - **RavenDB holding fewer is shown, not judged.** RavenDB's expiry keeps deleting archived and resolved messages, event log items and a few unresolved messages from 6.18 or earlier, after the copy has read them. -- **The same expiry can hide rows that were never read**, because a deleted row and a missed row cancel out. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. +- **The same expiry can hide rows that were never read**, see [Open decisions](#what-the-checks-do-not-catch). - **A windowed category is counted inside the window it started with**, on RavenDB only. - **Verify also prints how many rows SQL held for each category when it settled.** Before ServiceControl opens nothing else writes, so for a required category that is exactly what the copy left in SQL. For the two optional categories it includes what ServiceControl wrote while they copied. @@ -500,6 +488,50 @@ It says nothing about load on the source. - **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. - **Stopping a copy takes a restart.** +## Open decisions + +The design currently accepts each case below. Each one changes, merges or drops data, or lets a loss pass without failing a category. None is settled: each needs a decision before the migration ships. Where the design gives a reason for accepting a case, it is stated. + +### Values that change or merge in SQL + +- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so it keeps the newest attempt and the number of attempts. A message that failed five times still shows five attempts and still shows as a repeated failure, but only the newest attempt keeps its headers and exception. This applies to every failed message, whatever its status. +- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. +- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. + - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. + - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. + - The copy logs a warning naming both endpoints and the one it kept. + - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. +- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row.** SQL Server ignores trailing spaces when it compares the key. PostgreSQL keeps both. + - The copy keeps the endpoint whose RavenDB document id sorts first in ordinal order. + - A day that both endpoints counted holds the sum of their counts, as it would if SQL Server had recorded both endpoints from the start. + - This is not counted as a merge. The copy logs no warning, and the dry run does not list them. The second endpoint and its share of each shared day count as already present. + - The copy adds counts only for names that differ in case and trailing spaces. A key column given a collation that also ignores, for example, accents still makes one row of other names, and that row keeps one endpoint's count. The other endpoint's counts are lost. +- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** The copy subtracts the current value from `@expires`, so the rebuilt time is off by the difference. The message still ages out of SQL when RavenDB would have deleted it, because SQL adds the same current value back when it checks the row against the retention period. +- **Every retry-issued message gets a retry claim, even one whose claim RavenDB had already released.** While the claim exists, retrying the message from the failed messages list sends nothing. +- **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. + +The dry run counts the subscription and endpoint settings merges before anything moves. + +### Rows removed after the copy + +- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens.** ServiceControl's usual clean-up removes it, because that group has no failed messages in SQL until the background copy brings the archived and resolved ones. +- **Unresolved messages from version 6.18 or earlier can expire during the copy.** A message that failed again after being archived or resolved on 6.18 or earlier kept its old expiry, and upgrading does not remove it. One that RavenDB deletes before the copy reads it is not copied and not counted as a skip. + +### What the checks do not catch + +- **Nothing counts RavenDB during the copy.** A read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read. +- **Rows never read on a Done category are reported, not fixed.** Verify prints them, but its exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before ServiceControl opens. The current reasoning: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. +- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Only tests on each reader cover those. +- **Expiry can hide rows that were never read.** In verify, a deleted row and a missed row cancel out, so a flagged number is a floor. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. +- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. +- **A crash during a batch that carries bodies leaves orphan bodies in the body store.** External bodies are written before the transaction opens, so a crash leaves bodies no row points at, never rows pointing at missing bodies. + +### Gaps nothing enforces + +- **An ingestion worker started at the same moment as the first migration start can slip in before the copy writes its first checkpoint.** The ingestion gate cannot see a copy with no checkpoint row, so the worker ingests into a part-copied database. Only the instructions prevent this. +- **Reusing an old copy after going back to RavenDB cannot be detected**, because it looks exactly like a normal resume. The copy skips every category already finished and misses anything RavenDB received since. + ## Glossary - **Category:** one kind of data copied as a unit, such as known endpoints or the event log. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index 2aa0b511c3..1caf1d03c9 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -72,10 +72,10 @@ These run whether or not `Migration/Enabled` is set. Each one closes a path to d - **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import, flag on or off. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race it on the checkpoint version. - **`RecordHostOpenedOnTarget` on workers.** Every host that passes the ingestion gate also registers `RecordHostOpenedOnTarget`, flag on or off, and stamps `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so the end-of-migration guard must never offer a free abort after a worker has written. -The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row. +The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row, see [open decisions](ravendb-to-sql-migration-overview.md#gaps-nothing-enforces). > [!IMPORTANT] -> Workers must start only after the main host has opened. A worker started together with the first migration start can pass the gate before the first checkpoint row exists, and nothing enforces the order. +> Workers must start only after the main host has opened. ## The required copy @@ -172,9 +172,9 @@ Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader | Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | | Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | | Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | -| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; single-document readers load by id; licensing settings readers use `LicensingSettingsSource` | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; the two licensing endpoint readers use `ThroughputDocuments.Endpoints`; single-document readers load by id | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | | Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyMissing` after one attempt, a permanent fault no retry fixes, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | -| Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold, so a crash leaves orphan bodies, never dangling references | +| Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold. A crash leaves orphan bodies, see [open decisions](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch) | | Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `SetLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips and merges are logged after commit, and `Extend` adds the batch's `PreparedBatch.Merges` to `MergedCount`. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | | Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | | Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | @@ -257,7 +257,7 @@ Both operator commands rely on `MigrationCheckpointConflictException` to detect - **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `MergedCount` and `TargetCountAtSettle` are printed beside the sum instead. - **An abandoned category's sum covers its last pass only.** Rows an earlier pass copied before a retry are in `TargetCountAtSettle` but not in the sum, so RavenDB minus the sum overstates what was never copied. -**Rows never read on a Done category are accepted, not fixed.** Verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before the host opens: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. +For rows never read on a Done category, verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, and neither operator command takes a Done row. Whether to fix them is an [open decision](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch). ## Readers and writers @@ -274,10 +274,10 @@ Every reader derives from `MigrationCategoryReader` and every writer | 5 | `NotificationSettings` | Required | `NotificationSettingsReader`, `NotificationSettingsWriter` | One `Settings` row via `SettingRowMapper` | | 6 | `TrialEndDate` | Required | `TrialEndDateReader`, `TrialEndDateWriter` | One `Settings` row via `SettingRowMapper` | | 7 | `RetryOperations` | Required | `RetryOperationsReader`, `RetryOperationsWriter` | Two tables: historic rows store-keyed via `StoreKeyedInsert`, unacknowledged rows via `InsertMissing`, which sizes the batch. Deletes its rows on retry | -| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database | -| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. `SetLicensingEndpointThroughput` sets each day's count rather than incrementing, so re-reads are idempotent | -| 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | -| 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | +| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database. Reads only the `EndpointDocuments` collection, through `ThroughputDocuments.Endpoint`, because the throughput database gives its documents no id prefix and also holds the settings documents. With no prefix RavenDB streams in reverse etag order and ignores `startAfter`, so `ThroughputDocuments.Endpoints` reads every endpoint, sorts the ids in ordinal order and skips past the cursor itself | +| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. The reader gives one batch per group from `ThroughputDocuments.EndpointGroups` and ignores the batch size, because one endpoint can hold a year or more of days. A group is the endpoints of one source whose names match ignoring case and trailing spaces, and its cursor is its last id in ordinal order, so a resume never splits it. The writer adds the counts of a group's endpoints whose names the key column treats as the same. `SetLicensingEndpointThroughput` splits the batch into statements and sets each day's count rather than incrementing, so re-reads are idempotent | +| 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | +| 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | | 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator` | | 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | | 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | | diff --git a/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs b/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs new file mode 100644 index 0000000000..5ceb938368 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs @@ -0,0 +1,97 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Transports.BrokerThroughput; + +/// +/// A broker that reports the daily counts a test stages, only for days on or after the start date the collector asks from, +/// records the start date of every request, and counts how many whole collector passes have read it. +/// +class FakeBrokerThroughputQuery : IBrokerThroughputQuery +{ + readonly ConcurrentDictionary> staged = new(); + readonly ConcurrentQueue<(string Queue, DateOnly StartDate)> requests = new(); + int queuesStillReading; + int completedPasses; + + /// + /// How many times the collector has finished reading every staged queue's days. + /// + public int CompletedPasses => Volatile.Read(ref completedPasses); + + /// + /// Every queue the collector asked for days of, with the first day it asked for, in the order it asked. + /// + public IReadOnlyList<(string Queue, DateOnly StartDate)> Requests => [.. requests]; + + /// + /// Makes the broker report count messages on the queue for that day, replacing any count staged for the same day. + /// + public void Stage(string queue, DateOnly day, long count) => staged.GetOrAdd(queue, _ => new())[day] = count; + + public Dictionary Data { get; } = []; + + public string MessageTransport => "FakeBroker"; + + public string ScopeType => null; + + public KeyDescriptionPair[] Settings => []; + + public void Initialize(ReadOnlyDictionary settings) { } + + public bool HasInitialisationErrors(out string errorMessage) + { + errorMessage = string.Empty; + return false; + } + + public Task<(bool Success, List Errors, string Diagnostics)> TestConnection(CancellationToken cancellationToken = default) => + Task.FromResult((true, new List(), "Fake broker reachable")); + + public async IAsyncEnumerable GetQueueNames([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var queues = staged.Keys.ToArray(); + Volatile.Write(ref queuesStillReading, queues.Length); + + foreach (var queue in queues) + { + yield return new DefaultBrokerQueue(queue); + } + + await Task.CompletedTask; + } + + public async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + requests.Enqueue((brokerQueue.QueueName, startDate)); + + try + { + foreach (var (day, count) in staged[brokerQueue.QueueName].Where(entry => entry.Key >= startDate).OrderBy(entry => entry.Key)) + { + yield return new QueueThroughput { DateUTC = day, TotalThroughput = count }; + } + } + finally + { + // The collector records each day before asking for the next, so the last queue's enumeration ending is the pass ending. + if (Interlocked.Decrement(ref queuesStillReading) == 0) + { + Interlocked.Increment(ref completedPasses); + } + } + + await Task.CompletedTask; + } + + public string SanitizeEndpointName(string endpointName) => endpointName; + + public string SanitizedEndpointNameCleanser(string endpointName) => endpointName; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs new file mode 100644 index 0000000000..83e0f6a57a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs @@ -0,0 +1,124 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingEndpointsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_an_endpoint_and_reads_it_back_through_the_products_own_API() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) + { + SanitizedName = "Sales_Orders", + UserIndicator = "Sales team", + Scope = "Sales", + EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()] + }); + + await CopyCategory("LicensingEndpoints"); + + var fromTarget = await TargetLicensingDataStore.GetEndpoint("Sales.Orders", ThroughputSource.Broker); + var reportableEndpoints = await TargetLicensingDataStore.GetAllEndpoints(includePlatformEndpoints: false); + var row = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(fromTarget, Is.Not.Null); + Assert.That(fromTarget!.UserIndicator, Is.EqualTo("Sales team")); + Assert.That(fromTarget.Scope, Is.EqualTo("Sales")); + Assert.That(row.SanitizedName, Is.EqualTo("Sales_Orders")); + Assert.That(row.NormalizedSanitizedName, Is.EqualTo("sales_orders"), "GetEndpointThroughputByQueueName matches on this column, so an unnormalised value hides a migrated endpoint's throughput from the licensing report"); + Assert.That(fromTarget.EndpointIndicators, Is.EqualTo(new[] { EndpointIndicator.PlatformEndpoint.ToString() }), "the indicators column holds what EF itself writes: JSON on SQL Server, a text[] on PostgreSQL"); + Assert.That(reportableEndpoints, Is.Empty, "the product filters platform endpoints inside the database, so an indicator value EF cannot query puts a platform endpoint in the licensing report"); + } + } + + [Test] + public async Task The_same_name_under_two_sources_is_two_rows_and_not_a_collision() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputSource(new Endpoint("SALES.ORDERS", ThroughputSource.Audit) { SanitizedName = "sales.orders" }); + + var result = await CopyCategory("LicensingEndpoints"); + + Assert.That(result.Copied, Is.EqualTo(2), "the key is (NormalizedName, ThroughputSource), so the throughput source must survive into the key or the second endpoint is treated as a duplicate of the first"); + Assert.That(result.Skipped, Is.Zero); + } + + [Test] + public async Task An_endpoint_the_target_already_holds_is_already_present_rather_than_skipped() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + + await CopyCategory("LicensingEndpoints"); + await DeleteCheckpoint("LicensingEndpoints"); + var second = await CopyCategory("LicensingEndpoints"); + + Assert.That(second.AlreadyPresent, Is.EqualTo(1)); + Assert.That(second.Skipped, Is.Zero, "an endpoint the target already holds was not lost, and counting it as a skip makes --migration-verify subtract it twice"); + } + + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_never_fail_the_batch() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + var batch = new MigrationBatch( + [ + new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary()), + new MigrationRow("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + + var keyIgnoresTrailingSpace = (await QueryTarget(dbContext => Task.FromResult(dbContext.Database.ProviderName))).Contains("SqlServer"); + var storedNames = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().Select(row => row.Name).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space and keeps the first row; PostgreSQL's does not and keeps both"); + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is already present, not lost"); + Assert.That(result.Skipped, Is.Zero); + Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); + + if (keyIgnoresTrailingSpace) + { + Assert.That(storedNames, Is.EqualTo(new[] { "Sales.Orders " }), "the row whose id sorts first by ordinal order is the one a shared key keeps, whatever order the batch arrives in"); + } + } + } + + [Test] + public async Task An_endpoint_with_no_name_is_skipped_as_a_required_value_missing() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + var batch = new MigrationBatch( + [ + new MigrationRow("/Broker", new Endpoint(new EndpointIdentifier(null!, ThroughputSource.Broker)) { SanitizedName = "nameless" }, new Dictionary()), + new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = null! }, new Dictionary()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + var stored = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.SanitizedName, Is.Empty, "a null sanitized name takes the empty string the product itself stores"); + Assert.That(stored.NormalizedSanitizedName, Is.Empty); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "/Broker" })); + Assert.That(result.SkipReasons![MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "Name and NormalizedName are NOT NULL and the product has no value of its own for a nameless endpoint, so the row is skipped and counted rather than halting the category"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs new file mode 100644 index 0000000000..3655d52306 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs @@ -0,0 +1,98 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingSettingsMigrationTests : MigrationAcceptanceTest +{ + static LicensedEndpointDetails Details => new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + [Test] + public async Task Copies_report_masks_as_one_row_in_settings() + { + await SeedThroughputReportMasks(["contoso", "fabrikam"]); + + var result = await CopyCategory("LicensingReportMasks"); + + Assert.That(result.Copied, Is.EqualTo(1)); + var masks = await TargetLicensingDataStore.GetReportMasks(); + Assert.That(masks, Is.EquivalentTo(new[] { "contoso", "fabrikam" })); + } + + [Test] + public async Task Copies_the_licensed_endpoint_details_file_as_one_row_in_settings() + { + var seeded = Details; + await SeedThroughputLicensedEndpointDetails(seeded); + + var result = await CopyCategory("LicensedEndpointDetails"); + + Assert.That(result.Copied, Is.EqualTo(1)); + var details = await TargetLicensingDataStore.GetLicensedEndpointDetails(); + Assert.That(JsonSerializer.Serialize(details), Is.EqualTo(JsonSerializer.Serialize(seeded)), "every field, including the endpoint, queue and product arrays, must reach the SQL row"); + } + + [Test] + public async Task Report_masks_an_earlier_pass_copied_are_kept_and_reported_as_already_present() + { + await TargetLicensingDataStore.SaveReportMasks(["copied earlier"]); + await SeedThroughputReportMasks(["contoso"]); + + var result = await CopyCategory("LicensingReportMasks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(await TargetLicensingDataStore.GetReportMasks(), Is.EquivalentTo(new[] { "copied earlier" }), "a copy inserts a settings row only where the target has none, so a value an earlier pass copied is never overwritten"); + } + } + + [Test] + public async Task Licensed_endpoint_details_an_earlier_pass_copied_are_kept_and_reported_as_already_present() + { + await TargetLicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "copied earlier" }); + await SeedThroughputLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); + + var result = await CopyCategory("LicensedEndpointDetails"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That((await TargetLicensingDataStore.GetLicensedEndpointDetails())!.LicenseId, Is.EqualTo("copied earlier"), "a copy inserts a settings row only where the target has none, so a value an earlier pass copied is never overwritten"); + } + } + + [Test] + public async Task An_absent_document_writes_no_row() + { + var masks = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingReportMasks"); + var details = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensedEndpointDetails"); + + var masksResult = await CopyCategory(masks.Id); + var detailsResult = await CopyCategory(details.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(masksResult.Copied + detailsResult.Copied, Is.Zero); + Assert.That(await Target.Count(masks), Is.Zero, "an absent RavenDB document hands over nothing, so no empty masking list is invented on the SQL side"); + Assert.That(await Target.Count(details), Is.Zero); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs new file mode 100644 index 0000000000..a00b710498 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs @@ -0,0 +1,205 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingThroughputMigrationTests : MigrationAcceptanceTest +{ + [Test] + public void Writing_counts_before_their_endpoint_is_refused_naming_the_endpoint() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + var day = new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100); + var batch = new MigrationBatch([new MigrationRow("Sales.Orders/Broker|2026-01-01", day, new Dictionary())], "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var exception = Assert.ThrowsAsync(() => Target.Write(category, batch, checkpointToExtend)); + + Assert.That(exception!.Message, Does.Contain("Endpoint Sales.Orders from Broker does not exist"), "a missing endpoint is refused by name and source, as the product's own throughput write refuses it, rather than surfacing as a raw foreign key violation"); + } + + [Test] + public async Task An_endpoint_skipped_by_its_own_copy_blocks_the_counts_and_none_are_written() + { + await SeedThroughputSource(new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = new string('s', 451) }); + await SeedThroughputTimeSeries("Billing/Broker", new DateOnly(2026, 1, 1), 50); + + await CopyCategory("LicensingEndpoints"); + Assert.ThrowsAsync(() => CopyCategory("LicensingThroughput")); + + var endpoints = await ReadCheckpoint("LicensingEndpoints"); + var throughput = await ReadCheckpoint("LicensingThroughput"); + var written = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the endpoint whose sanitized name is too long is skipped as a fault"); + Assert.That(throughput.State, Is.EqualTo(MigrationCategoryState.Blocked), "a Failed predecessor blocks the counts, so the writer never meets a day whose endpoint was skipped"); + Assert.That(throughput.LastError, Does.Contain("must follow LicensingEndpoints, which is CompleteWithErrors")); + Assert.That(written, Is.Zero); + } + } + + [Test] + public async Task A_document_of_another_collection_is_not_read_so_the_endpoints_complete_and_the_counts_copy() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + + using (var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase)) + { + var stray = new { Setting = "left by an older version" }; + await session.StoreAsync(stray, "LegacySettings/1"); + session.Advanced.GetMetadataFor(stray)["@collection"] = "LegacySettings"; + await session.SaveChangesAsync(); + } + + var endpoints = await CopyCategory("LicensingEndpoints"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a document outside the EndpointDocuments collection is not an endpoint, so it is not skipped as a nameless one that blocks the counts"); + Assert.That(endpoints.Copied, Is.EqualTo(1)); + Assert.That(endpoints.Skipped, Is.Zero); + } + + var throughput = await CopyCategory("LicensingThroughput"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(throughput.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(throughput.Copied, Is.EqualTo(1)); + Assert.That(await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()), Is.EqualTo(1)); + } + } + + [Test] + public async Task The_registry_order_blocks_the_counts_until_their_endpoints_are_copied() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + + Assert.ThrowsAsync(() => CopyCategory("LicensingThroughput")); + var blocked = await ReadCheckpoint("LicensingThroughput"); + + await CopyCategory("LicensingEndpoints"); + var result = await CopyCategory("LicensingThroughput"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(blocked.LastError, Does.Contain("must follow LicensingEndpoints")); + Assert.That(result.Copied, Is.EqualTo(1)); + } + } + + [Test] + public async Task Running_the_category_twice_produces_the_same_count() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + await CopyCategory("LicensingEndpoints"); + + await CopyCategory("LicensingThroughput"); + await DeleteCheckpoint("LicensingThroughput"); + var second = await CopyCategory("LicensingThroughput"); + + var total = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == new DateOnly(2026, 1, 1)) + .Select(row => row.MessageCount) + .SingleAsync()); + + Assert.That(second.Copied, Is.EqualTo(1), "the second run wrote nothing, so the count below cannot tell setting from adding"); + Assert.That(total, Is.EqualTo(100), "running the category twice must set the day's count, not add to it"); + } + + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_add_their_counts_into_one_day_without_failing_the_batch() + { + var endpoints = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + var endpointBatch = new MigrationBatch( + [ + new MigrationRow("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary()), + new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary()) + ], + "Sales.Orders/Broker"); + await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + var day = new DateOnly(2026, 1, 1); + var batch = new MigrationBatch( + [ + new MigrationRow("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100), new Dictionary()), + new MigrationRow("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250), new Dictionary()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + + var keyIgnoresTrailingSpace = (await QueryTarget(dbContext => Task.FromResult(dbContext.Database.ProviderName))).Contains("SqlServer"); + var count = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == day) + .Select(row => row.MessageCount) + .SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space, so the two days are one row; PostgreSQL's does not, so they are two"); + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the day SQL Server's key folds into the other is already present, not lost or failed"); + Assert.That(result.Skipped, Is.Zero); + Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "in a single batch, every row reported as copied is a row in the table"); + Assert.That(count, Is.EqualTo(keyIgnoresTrailingSpace ? 350 : 250), "where the key makes the two endpoints one, their counts for the day are added, as the product's own throughput write adds them"); + } + } + + [Test] + public async Task The_copied_counts_are_what_the_collector_reads_as_already_collected() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 2), 200); + + await CopyCategory("LicensingEndpoints"); + await CopyCategory("LicensingThroughput"); + + var endpoint = await TargetLicensingDataStore.GetEndpoint(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)); + + Assert.That(endpoint!.LastCollectedDate, Is.EqualTo(new DateOnly(2026, 1, 2)), + "LastCollectedDate is derived from MAX(DateUtc) over the copied rows, and it is the only thing stopping the collector re-collecting these two days and adding to them"); + } + + [Test] + public async Task A_day_whose_endpoint_has_no_name_is_skipped_as_a_required_value_missing() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await CopyCategory("LicensingEndpoints"); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + var day = new DateOnly(2026, 1, 1); + var batch = new MigrationBatch( + [ + new MigrationRow("/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier(null!, ThroughputSource.Broker), day, 100), new Dictionary()), + new MigrationRow("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250), new Dictionary()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "/Broker|2026-01-01" })); + Assert.That(result.SkipReasons![MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "a nameless endpoint's day has no NormalizedName to key under, so it is skipped and counted rather than halting the category"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index 6268bbc41a..21c76e78a6 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -13,10 +13,14 @@ namespace ServiceControl.Migration.AcceptanceTests; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using NServiceBus.Extensibility; using NServiceBus.Transport; using NUnit.Framework; +using Particular.LicensingComponent.BrokerThroughput; +using Particular.LicensingComponent.Contracts; +using Particular.LicensingComponent.Persistence; using ServiceControl.Persistence.EFCore; using Particular.ServiceControl.Hosting; using ServiceBus.Management.Infrastructure.Settings; @@ -36,6 +40,7 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Persistence.Tests; using ServiceControl.Persistence.UnitOfWork; +using ServiceControl.Transports.BrokerThroughput; using TestHelper; // Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. using IDocumentStore = Raven.Client.Documents.IDocumentStore; @@ -52,6 +57,11 @@ abstract class MigrationAcceptanceTest ServiceProvider targetServices; IMigrationCheckpointStore targetCheckpointStore; + readonly FakeBrokerThroughputQuery brokerQuery = new(); + + // RavenDB refuses an incremental time series whose name does not start with "INC:". + // It must equal LicensingDataStore.ThroughputTimeSeriesName, which is internal to the RavenDB persister. + const string ThroughputTimeSeriesName = "INC: throughput data"; protected Settings Settings { get; private set; } protected HttpClient HttpClient { get; private set; } @@ -156,6 +166,64 @@ protected async Task SeedSource(string database, params (string Id, object Docum protected Task SeedSource(params (string Id, object Document)[] documents) => SeedSource(Source.PrimaryDatabase, documents); + protected ILicensingDataStore TargetLicensingDataStore => targetServices.GetRequiredService(); + + protected async Task SeedThroughputSource(Endpoint endpoint) + { + var documentId = $"{endpoint.Id.Name}/{endpoint.Id.ThroughputSource}"; + + var document = new + { + Id = documentId, + EndpointId = new { endpoint.Id.Name, endpoint.Id.ThroughputSource }, + endpoint.SanitizedName, + EndpointIndicators = endpoint.EndpointIndicators ?? [], + UserIndicator = endpoint.UserIndicator ?? string.Empty, + Scope = endpoint.Scope ?? string.Empty + }; + + using var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase); + await session.StoreAsync(document, documentId); + session.Advanced.GetMetadataFor(document)["@collection"] = "EndpointDocuments"; + await session.SaveChangesAsync(); + } + + protected async Task SeedThroughputTimeSeries(string documentId, DateOnly day, long count) + { + using var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase); + session.IncrementalTimeSeriesFor(documentId, ThroughputTimeSeriesName) + .Increment(day.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), count); + await session.SaveChangesAsync(); + } + + protected void BrokerReports(string queue, DateOnly day, long count) => brokerQuery.Stage(queue, day, count); + + protected IReadOnlyList<(string Queue, DateOnly StartDate)> BrokerRequests => brokerQuery.Requests; + + protected Task WaitForCollectorPass() => + WaitUntil(() => Task.FromResult(brokerQuery.CompletedPasses > 0), "the broker throughput collector completed a pass"); + + // The product adds the broker collector only when a query is registered before AddServiceControl, and customize runs after it. + protected void CollectFromTheBrokerImmediately(IServiceCollection services) + { + services.AddSingleton(brokerQuery); + services.AddHostedService(provider => new BrokerThroughputCollectorHostedService( + provider.GetRequiredService>(), + brokerQuery, + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService()) + { + DelayStart = TimeSpan.Zero + }); + } + + protected Task SeedThroughputReportMasks(IReadOnlyList maskedStrings) => + SeedSource(Source.ThroughputDatabase, ("ReportMasks", new { MaskedStrings = maskedStrings.ToList() })); + + protected Task SeedThroughputLicensedEndpointDetails(LicensedEndpointDetails details) => + SeedSource(Source.ThroughputDatabase, ("LicensedEndpointDetails", details)); + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => SeedSourceWithBodies((document, body, contentType)); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs new file mode 100644 index 0000000000..0bf218a543 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; + +[TestFixture] +[NonParallelizable] +class When_the_throughput_collector_runs_after_a_migration : MigrationAcceptanceTest +{ + [Test] + public async Task Copied_days_are_not_collected_a_second_time_and_added_to() + { + var time = new FakeTimeProvider(); + var yesterday = DateOnly.FromDateTime(time.GetUtcNow().UtcDateTime).AddDays(-1); + + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", yesterday, 100); + BrokerReports(queue: "sales.orders", day: yesterday, count: 70); + + await RunRequiredCopy(customize: services => + { + services.AddSingleton(time); + CollectFromTheBrokerImmediately(services); + }); + + await WaitForCollectorPass(); + + var total = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == yesterday) + .Select(row => row.MessageCount) + .SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(BrokerRequests, Is.EqualTo(new[] { ("sales.orders", yesterday.AddDays(1)) }), + "the collector asks only for days after the last copied one, so it never tried to collect the copied day again and a failed write cannot hide that it did"); + Assert.That(total, Is.EqualTo(100), + "100 is the copied count; 70 means the copy wrote nothing and the collector collected the day itself; 170 means the collector re-collected a copied day and the additive write path added the two, so the licensing report overstates usage"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs index 20c1542231..6c4ec5155a 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; using ServiceControl.Persistence.EFCore.Infrastructure; /// @@ -47,4 +48,30 @@ ON CONFLICT ({keyColumns}) DO NOTHING return inserted; } + + public async Task SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var keyColumns = string.Join(", ", insert.KeyColumns); + var written = 0; + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + written += (await insert.Execute( + dbContext, + $""" + INSERT INTO {insert.Table} ({string.Join(", ", insert.Columns)}) + VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ON CONFLICT ({keyColumns}) + DO UPDATE SET {string.Join(", ", insert.Columns.Except(insert.KeyColumns).Select(column => $"{column} = excluded.{column}"))} + RETURNING {keyColumns} + """, + chunk, + configureParameter: null, + cancellationToken)).Count; + } + + return written; + } } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs index 5de111af7b..0ebab54fc0 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -6,10 +6,11 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; using ServiceControl.Persistence.EFCore.Infrastructure; /// -/// The migration's SQL Server statements. Two keys that differ only in case can be one row here, because a +/// The migration's SQL Server statements. Two keys that differ only in case or trailing spaces can be one row here, because a /// column's collation decides what counts as equal, so the statements compare keys under the collation the /// schema gives them and the batch is de-duplicated the same way before it is offered to the table. /// @@ -54,7 +55,7 @@ FROM sys.columns AS c ?? throw new InvalidOperationException($"The migration target found no collation for {tableName}.{column}, so the SQL Server schema is missing or not current. Run ServiceControl with --setup against this database first."); collations[(table, sql.DelimitIdentifier(column))] = collation.Name; - comparers[(entityType.ClrType, property.Name)] = (collation.ComparisonStyle & IgnoreCaseStyle) == 0 ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; + comparers[(entityType.ClrType, property.Name)] = new PaddedKeyComparer((collation.ComparisonStyle & IgnoreCaseStyle) == 0 ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); } } @@ -113,6 +114,45 @@ WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Colum return inserted; } + public async Task SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var columns = string.Join(", ", insert.Columns); + var partitionBy = string.Join(", ", insert.KeyColumns.Select(column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column)); + var written = 0; + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // HOLDLOCK, or two writers can both find a day missing and both insert it. + // The output names the source's key values, because a stored name can differ from the chunk's by a trailing space and would match no row of it. + written += (await insert.Execute( + dbContext, + $""" + MERGE {insert.Table} WITH (HOLDLOCK) AS t + USING ( + SELECT {columns} + FROM ( + SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {partitionBy} ORDER BY [MigrationOrdinal] DESC) AS [MigrationDuplicate] + FROM (VALUES + {ParameterRowsWithOrdinal(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}, [MigrationOrdinal]) + ) AS deduplicated + WHERE [MigrationDuplicate] = 1 + ) AS s ({columns}) + ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} + WHEN MATCHED THEN UPDATE SET {string.Join(", ", insert.Columns.Except(insert.KeyColumns).Select(column => $"{column} = s.{column}"))} + WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) + OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"s.{column}"))}; + """, + chunk, + configureParameter: null, + cancellationToken)).Count; + } + + return written; + } + // EF declares unbounded text as nvarchar(4000) or nvarchar(max) by each value's length, and each new mix compiles its own plan. // Size -1 declares every such value as the nvarchar(max) its column already is. static void DeclareUnboundedTextAsMax(IProperty property, DbParameter parameter) @@ -130,4 +170,12 @@ sealed record OpenedKeys( IReadOnlyDictionary<(Type EntityType, string Property), IEqualityComparer> Comparers); sealed record KeyColumnCollation(string Name, int ComparisonStyle); + + // SQL Server pads the shorter value with spaces before it compares, so keys that differ only in trailing spaces are equal. + sealed class PaddedKeyComparer(StringComparer inner) : IEqualityComparer + { + public bool Equals(string? x, string? y) => inner.Equals(x?.TrimEnd(' '), y?.TrimEnd(' ')); + + public int GetHashCode(string obj) => inner.GetHashCode(obj.TrimEnd(' ')); + } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index 85c4bea2be..e0b683b1c8 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -26,7 +26,11 @@ sealed class EFCoreMigrationTarget( { new KnownEndpointsWriter(migrationDialect), new EndpointSettingsWriter(migrationDialect), - new FailedMessagesWriter(migrationDialect, storagePersistence, settings) + new FailedMessagesWriter(migrationDialect, storagePersistence, settings), + new LicensingEndpointsWriter(migrationDialect), + new LicensingThroughputWriter(migrationDialect), + new LicensingReportMasksWriter(migrationDialect), + new LicensedEndpointDetailsWriter(migrationDialect) }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs new file mode 100644 index 0000000000..b5772d85fa --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; + +/// +/// Turns a licensing endpoint read from the source into the row the target stores, with the same key and the same +/// stand-ins for missing values as the product's own save. +/// +static class LicensingEndpointRowMapper +{ + public static LicensingEndpointEntity Map(Endpoint endpoint) + { + var sanitizedName = endpoint.SanitizedName ?? string.Empty; + + return new LicensingEndpointEntity + { + NormalizedName = LicensingDataStore.Normalize(endpoint.Id.Name), + ThroughputSource = endpoint.Id.ThroughputSource, + Name = endpoint.Id.Name, + SanitizedName = sanitizedName, + NormalizedSanitizedName = LicensingDataStore.Normalize(sanitizedName), + UserIndicator = endpoint.UserIndicator ?? string.Empty, + Scope = endpoint.Scope, + EndpointIndicators = endpoint.EndpointIndicators?.ToList() ?? [] + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs new file mode 100644 index 0000000000..299fd8912d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; + +/// +/// Turns one endpoint's daily count read from the source into the row the target stores, keyed by the same +/// normalised name the product's own licensing store computes. +/// +static class LicensingThroughputRowMapper +{ + public static LicensingEndpointThroughputEntity Map(ThroughputDayRow row) + { + return new LicensingEndpointThroughputEntity + { + NormalizedName = LicensingDataStore.Normalize(row.Endpoint.Name), + ThroughputSource = row.Endpoint.ThroughputSource, + DateUtc = row.DateUtc, + MessageCount = row.MessageCount + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs new file mode 100644 index 0000000000..5e43817ff8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text.Json; +using ServiceControl.Persistence.EFCore.Entities; + +/// +/// Builds the Settings row for a value, storing it under the given key as the JSON text that +/// SettingsStore.StoreSetting writes. is the type the product stores under that key. +/// +static class SettingRowMapper +{ + public static SettingEntity Map(string key, T value) => new() { Key = key, Value = JsonSerializer.Serialize(value) }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs new file mode 100644 index 0000000000..9bcf8ec909 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the licensed endpoint details file as one row in Settings. A row the target already holds is left as it is +/// and counted as already present. +/// +sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; + + const string SettingKey = SettingKeys.LicensedEndpointDetails; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, LicensedEndpointDetails Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); + } + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs new file mode 100644 index 0000000000..cac5d17b6b --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs @@ -0,0 +1,54 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the licensing endpoints. An endpoint the target already holds, or a second one in the same batch whose key +/// the column treats as the same, is left as it is and counted as already present. +/// +sealed class LicensingEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.LicensingEndpoints; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpoints.LongCountAsync(cancellationToken); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, Endpoint Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var rows = new List(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, endpoint) in documents.OrderBy(document => document.Row.SourceId, StringComparer.Ordinal)) + { + if (endpoint.Id.Name is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "LicensingEndpoints.Name and NormalizedName are NOT NULL and the document has no endpoint name")); + continue; + } + + var entity = LicensingEndpointRowMapper.Map(endpoint); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); + } + + return Task.FromResult(new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + skips, + Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs new file mode 100644 index 0000000000..18823e068c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the report masks as one row in Settings. A row the target already holds is left as it is and counted as +/// already present. +/// +sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter> +{ + public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; + + const string SettingKey = SettingKeys.ReportMasks; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, List Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); + } + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs new file mode 100644 index 0000000000..5a03dd5353 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs @@ -0,0 +1,85 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Implementation; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the licensing endpoints' daily message counts, setting each day's count rather than adding to it, so a copy +/// run twice leaves the same numbers. A batch naming an endpoint the target does not hold is refused with an +/// naming that endpoint and its source. A day repeated for one endpoint keeps +/// its last row. Days of different endpoints whose names the key column treats as the same are added into one row, +/// as the product's own throughput write would add them. Either way the rows folded away are counted as already present. +/// +sealed class LicensingThroughputWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.LicensingThroughput; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpointThroughput.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, ThroughputDayRow Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var nameComparer = migrationDialect.KeyComparer(typeof(LicensingEndpointThroughputEntity), nameof(LicensingEndpointThroughputEntity.NormalizedName)); + var mapped = new List(documents.Count); + var endpoints = new List(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + // Date order is the order the product's own throughput write takes row locks in. + foreach (var (row, day) in documents.OrderBy(document => document.Document.DateUtc).ThenBy(document => document.Row.SourceId, StringComparer.Ordinal)) + { + if (day.Endpoint.Name is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "LicensingEndpointThroughput.NormalizedName is NOT NULL and the day's endpoint has no name")); + continue; + } + + var entity = LicensingThroughputRowMapper.Map(day); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + mapped.Add(entity); + endpoints.Add(day.Endpoint); + } + + foreach (var endpoint in endpoints.Distinct()) + { + var normalizedName = LicensingDataStore.Normalize(endpoint.Name); + + // Checked in SQL rather than in memory, so the name compares under the column's collation as the foreign key does. + if (!await dbContext.LicensingEndpoints.AnyAsync(row => row.NormalizedName == normalizedName && row.ThroughputSource == endpoint.ThroughputSource, cancellationToken)) + { + throw new InvalidOperationException($"Endpoint {endpoint.Name} from {endpoint.ThroughputSource} does not exist in the migration target, so its daily throughput cannot be copied. LicensingThroughput copies only after LicensingEndpoints has finished."); + } + } + + List rows = + [ + .. mapped + .GroupBy(day => (day.ThroughputSource, day.DateUtc)) + .SelectMany(sameDay => sameDay.GroupBy(day => day.NormalizedName, nameComparer).Select(AddEndpoints)) + ]; + + return new PreparedBatch((context, token) => migrationDialect.SetLicensingEndpointThroughput(context, rows, token), mapped.Count, skips, Merges: []); + } + + // The sum is set, not added to the stored count, so the reader must put every endpoint that can share a key in one batch. + static LicensingEndpointThroughputEntity AddEndpoints(IEnumerable sameKey) + { + var perEndpoint = sameKey.GroupBy(day => day.NormalizedName, StringComparer.Ordinal).Select(sameEndpoint => sameEndpoint.Last()).ToList(); + var kept = perEndpoint[0]; + kept.MessageCount = perEndpoint.Sum(day => day.MessageCount); + return kept; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index 08553a6686..9cf5f6e032 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -362,5 +362,5 @@ static Endpoint ToEndpoint(LicensingEndpointEntity row, DateOnly? lastCollectedD LastCollectedDate = lastCollectedDate ?? default }; - static string Normalize(string name) => name.ToLowerInvariant(); + internal static string Normalize(string name) => name.ToLowerInvariant(); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs index 67f875e037..951088d8cb 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; /// /// The migration's provider-specific SQL. Statements run inside the caller's transaction and insert only what is absent. @@ -39,4 +40,12 @@ public interface IMigrationSqlDialect /// Inserts the rows whose key the table does not hold, merges any whose keys the database treats as one, and returns the rows it inserted. /// Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class; + + /// + /// Sets each day's message count: a day the table does not hold is inserted, and a day it holds gets its count replaced, never added to. + /// The rows go to the database in the order given. Every row's endpoint must already be in the table, or the foreign key fails the statement. + /// Two rows with the same key in one call are not allowed on PostgreSQL, which refuses the statement. SQL Server keeps the last of them, which also covers names that differ only by a trailing space. + /// Returns how many rows the statements inserted or updated. It runs on the caller's open transaction, and throws when there is none or when a returned key matches no row given. + /// + Task SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs index 947e20afec..b53b885fa9 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using System.Data.Common; +using System.Reflection; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -16,6 +17,7 @@ public sealed class MigrationInsert where TEntity : class readonly IReadOnlyList properties; readonly IReadOnlyList keyProperties; + readonly IReadOnlyList keyReaders; readonly IReadOnlyList<(IProperty Property, string TableAndColumn, int MaxLength)> boundedText; readonly IReadOnlyList<(IProperty Property, string TableAndColumn)> text; @@ -24,6 +26,7 @@ public sealed class MigrationInsert where TEntity : class Table = quotedTable; this.properties = properties; this.keyProperties = keyProperties; + keyReaders = [.. keyProperties.Select(KeyReader)]; Columns = [.. properties.Select(property => quote(columnName(property)))]; KeyColumns = [.. keyProperties.Select(property => quote(columnName(property)))]; text = [.. properties.Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{tableName}.{columnName(property)}"))]; @@ -157,10 +160,9 @@ public async Task> Execute(ServiceControlDbContext dbCont await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - object?[] key = [.. keyProperties.Select((property, column) => FromProvider(property, reader.GetValue(column)))]; + object?[] key = [.. keyProperties.Select((property, column) => FromProvider(property, keyReaders[column].Invoke(reader, BindingFlags.DoNotWrapExceptions, null, [column], null)!))]; - // A key can come back as a different CLR type than it went in as, a date column read back as - // DateTime say, and a plain lookup would report that as a bare KeyNotFoundException. + // A returned key that no row of the chunk holds exactly would be a bare KeyNotFoundException with a plain lookup, so it is refused by name. inserted.Add(rowsByKey.TryGetValue(key, out var row) ? row : throw new InvalidOperationException($"The {Table} insert returned the key [{string.Join(", ", key.Select(Describe))}], which matches no row in the batch that produced it. Compare it against the key types the batch carried: [{string.Join(", ", keyProperties.Select(property => property.ClrType.Name))}].")); @@ -173,6 +175,14 @@ public async Task> Execute(ServiceControlDbContext dbCont static string Describe(object? value) => value is null ? "null" : $"{value} ({value.GetType().Name})"; + // SQL Server's GetValue returns a date column as DateTime, which no DateOnly key equals, so each key is read as the type its mapping stores. + static MethodInfo KeyReader(IProperty property) + { + var mapping = property.GetRelationalTypeMapping(); + + return typeof(DbDataReader).GetMethod(nameof(DbDataReader.GetFieldValue))!.MakeGenericMethod(mapping.Converter?.ProviderClrType ?? mapping.ClrType); + } + static object? FromProvider(IProperty property, object value) => property.GetRelationalTypeMapping().Converter is { } converter ? converter.ConvertFromProvider(value) : value; diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index 22eefa3bbf..cfd20d1582 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -27,7 +27,11 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig { new KnownEndpointsReader(lifecycle), new EndpointSettingsReader(lifecycle), - new FailedMessagesReader(lifecycle) + new FailedMessagesReader(lifecycle), + new LicensingEndpointsReader(lifecycle), + new LicensingThroughputReader(lifecycle), + new LicensingReportMasksReader(lifecycle), + new LicensedEndpointDetailsReader(lifecycle) }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs new file mode 100644 index 0000000000..477edb9b3b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs @@ -0,0 +1,47 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Threading; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; + +/// +/// Reads the licensed endpoint details document, which becomes one Settings row. A read gives one batch holding the +/// whole document, or no rows when the document does not exist. The batch's cursor is the document's id, and a read +/// resumed after that id gives nothing. +/// +sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + const string DocumentId = LicensingDataStore.LicencedEndpointDetailsDocumentId; + + public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadLicensedEndpointDetails(resumeAfter, cancellationToken); + + async IAsyncEnumerable ReadLicensedEndpointDetails(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (resumeAfter == DocumentId) + { + yield break; + } + + if (resumeAfter is not null) + { + throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); + } + + using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); + var document = await session.LoadAsync(DocumentId, cancellationToken); + + yield return new MigrationBatch( + document is null ? [] : [new MigrationRow(DocumentId, document, ReadOnlyDictionary.Empty)], + DocumentId); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs new file mode 100644 index 0000000000..0a0b9629e1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the licensing endpoints out of the throughput database, skipping every document outside the EndpointDocuments collection. +/// +sealed class LicensingEndpointsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.LicensingEndpoints; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ThroughputDocuments.Endpoints(Lifecycle, CategoryId, resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs new file mode 100644 index 0000000000..6b74a4e20a --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs @@ -0,0 +1,47 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Threading; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +/// +/// Reads the report masks document, whose list of masked strings becomes one Settings row. A read gives one batch +/// holding the list, or no rows when the document does not exist, and a list stored as null reads as an empty one. +/// The batch's cursor is the document's id, and a read resumed after that id gives nothing. +/// +sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader>(lifecycle) +{ + const string DocumentId = LicensingDataStore.ReportMasksDocumentId; + + public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadReportMasks(resumeAfter, cancellationToken); + + async IAsyncEnumerable ReadReportMasks(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (resumeAfter == DocumentId) + { + yield break; + } + + if (resumeAfter is not null) + { + throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); + } + + using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); + var document = await session.LoadAsync(DocumentId, cancellationToken); + + yield return new MigrationBatch( + document is null ? [] : [new MigrationRow(DocumentId, document.MaskedStrings ?? [], ReadOnlyDictionary.Empty)], + DocumentId); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs new file mode 100644 index 0000000000..0b87d2ee76 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs @@ -0,0 +1,56 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Session.TimeSeries; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; + +/// +/// Reads each licensing endpoint's daily message counts out of the throughput database, one batch per group from +/// , with the batch's cursor on the group's last document id. +/// +sealed class LicensingThroughputReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.LicensingThroughput; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadThroughputDays(resumeAfter, cancellationToken); + + async IAsyncEnumerable ReadThroughputDays(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // One group of endpoints per batch, because the batch size counts rows and one endpoint can bring a year or more of days, + // and because the writer adds the days of a group's endpoints only when they arrive in one batch. + await foreach (var endpoints in ThroughputDocuments.EndpointGroups(Lifecycle, CategoryId, resumeAfter, cancellationToken)) + { + var days = new List(); + + foreach (var endpoint in endpoints) + { + var id = ((Endpoint)endpoint.Document).Id; + + foreach (var point in await ReadThroughputSeries(endpoint.SourceId, cancellationToken)) + { + days.Add(new MigrationRow($"{endpoint.SourceId}|{point.Timestamp.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}", new ThroughputDayRow(id, DateOnly.FromDateTime(point.Timestamp), (long)point.Value), ReadOnlyDictionary.Empty)); + } + } + + yield return new MigrationBatch(days, endpoints[^1].SourceId); + } + } + + // Each series read is a request, and one session refuses more than MaxNumberOfRequestsPerSession of them. + async Task ReadThroughputSeries(string documentId, CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); + return await session.IncrementalTimeSeriesFor(documentId, LicensingDataStore.ThroughputTimeSeriesName).GetAsync(token: cancellationToken) ?? []; + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs new file mode 100644 index 0000000000..1207f6b3fb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs @@ -0,0 +1,112 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Conventions; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +/// +/// What the throughput database's readers share: which documents are endpoints, and how an endpoint document becomes a row. +/// +static class ThroughputDocuments +{ + static readonly string EndpointCollection = DocumentConventions.DefaultGetCollectionName(typeof(EndpointDocument)); + + /// + /// Reads the endpoints in ordinal document-id order, after , in batches of at most + /// rows, each with its cursor on the last endpoint's id. Throws + /// when the source holds no document with the id in . + /// + public static async IAsyncEnumerable Endpoints( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var endpoints = (await AllEndpoints(lifecycle, categoryId, resumeAfter, cancellationToken)) + .Where(row => resumeAfter is null || string.CompareOrdinal(row.SourceId, resumeAfter) > 0) + .OrderBy(row => row.SourceId, StringComparer.Ordinal); + + foreach (var chunk in endpoints.Chunk(batchSize)) + { + yield return new MigrationBatch(chunk, chunk[^1].SourceId); + } + } + + /// + /// Reads the endpoints in groups whose names can share a key in SQL: the same source, and the same name ignoring + /// case and trailing spaces. Groups come in the ordinal order of their last document id, which is each group's cursor, + /// and only groups after are read. Throws when + /// the source holds no document with the id in . + /// + public static async IAsyncEnumerable> EndpointGroups( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string? resumeAfter, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var groups = (await AllEndpoints(lifecycle, categoryId, resumeAfter, cancellationToken)) + .GroupBy(SharedKey) + .Select(group => group.OrderBy(row => row.SourceId, StringComparer.Ordinal).ToList()) + .Where(group => resumeAfter is null || string.CompareOrdinal(group[^1].SourceId, resumeAfter) > 0) + .OrderBy(group => group[^1].SourceId, StringComparer.Ordinal); + + foreach (var group in groups) + { + yield return group; + } + } + + // SQL Server's default collation ignores case and trailing spaces, and the product lower-cases the name it stores. + static (string? Name, ThroughputSource Source) SharedKey(MigrationRow row) + { + var id = ((Endpoint)row.Document).Id; + return (id.Name?.TrimEnd(' ').ToLowerInvariant(), id.ThroughputSource); + } + + // With no id prefix RavenDB streams in reverse etag order and ignores startAfter, so callers order the rows and apply the cursor. + static async Task> AllEndpoints(RavenReadOnlySourceLifecycle lifecycle, string categoryId, string? resumeAfter, CancellationToken cancellationToken) + { + var databaseName = lifecycle.Settings.ThroughputDatabaseName; + var endpoints = new List(); + + using var session = lifecycle.OpenSession(databaseName); + await RavenDocumentStream.RefuseMissingCursor(session, categoryId, databaseName, resumeAfter, cancellationToken); + + await using var stream = await session.Advanced.StreamAsync(string.Empty, token: cancellationToken); + + while (await stream.MoveNextAsync()) + { + if (Endpoint(stream.Current) is { } row) + { + endpoints.Add(row); + } + } + + return endpoints; + } + + /// + /// Turns one streamed document into a row that carries an . Returns null when the document is not in the EndpointDocuments collection, which is the only place the product reads endpoints from. + /// The throughput database gives its documents no id prefix, so the stream also hands back its singleton documents and anything else stored there. + /// + public static MigrationRow? Endpoint(StreamResult entry) => + IsEndpoint(entry) ? new MigrationRow(entry.Id, entry.Document.ToEndpoint(), ReadOnlyDictionary.Empty) : null; + + // RavenDB matches collection names ignoring case. + static bool IsEndpoint(StreamResult entry) => + entry.Metadata.TryGetValue(Constants.Documents.Metadata.Collection, out var collection) && string.Equals(collection?.ToString(), EndpointCollection, StringComparison.OrdinalIgnoreCase); +} diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 55364bb14c..046b871336 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -22,11 +22,11 @@ class LicensingDataStore( TimeProvider timeProvider) : ILicensingDataStore { internal const string ThroughputTimeSeriesName = "INC: throughput data"; - const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; + internal const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; - const string BrokerMetadataDocumentId = "BrokerMetadata"; - const string ReportMasksDocumentId = "ReportMasks"; - const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; + internal const string BrokerMetadataDocumentId = "BrokerMetadata"; + internal const string ReportMasksDocumentId = "ReportMasks"; + internal const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; const string LicencedEndpointDetailsAmendmentDocumentId = "LicensedEndpointDetailsAmendment"; static readonly AuditServiceMetadata DefaultAuditServiceMetadata = new([], []); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs new file mode 100644 index 0000000000..5dc3de34e7 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs @@ -0,0 +1,95 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +class LicensingEndpointsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_endpoints_and_skips_the_four_singleton_documents() + { + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)) + { + SanitizedName = "sales.orders" + }, "Sales.Orders/Broker"); + + await session.StoreAsync(new { }, RavenLicensingDataStore.AuditServiceMetadataDocumentId); + await session.StoreAsync(new { MaskedStrings = new[] { "secret" } }, RavenLicensingDataStore.ReportMasksDocumentId); + await session.StoreAsync(new { }, RavenLicensingDataStore.BrokerMetadataDocumentId); + await session.StoreAsync(new { LicenseId = "abc" }, RavenLicensingDataStore.LicencedEndpointDetailsDocumentId); + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + + var batches = await CollectBatches(source, category); + var rows = batches.SelectMany(batch => batch.Rows).ToList(); + + Assert.That(rows, Has.Count.EqualTo(1)); + var endpoint = (Endpoint)rows[0].Document; + Assert.That(endpoint.Id.Name, Is.EqualTo("Sales.Orders")); + } + + [Test] + public async Task Reads_endpoints_in_id_order_and_resumes_after_the_cursor_whatever_order_they_were_stored_in() + { + foreach (var name in new[] { "Z", "A", "M" }) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)) { SanitizedName = name.ToLowerInvariant() }, $"{name}/Broker"); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingEndpoints); + + var batches = await CollectBatches(source, category, batchSize: 1); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor, batchSize: 1); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "A/Broker", "M/Broker", "Z/Broker" }), "each batch cursors on its last endpoint, in id order rather than the order RavenDB stored them"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { "M/Broker", "Z/Broker" }), "a resume reads only the endpoints after the cursor"); + } + } + + [Test] + public async Task A_document_of_another_collection_is_neither_read_nor_counted() + { + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)) + { + SanitizedName = "sales.orders" + }, "Sales.Orders/Broker"); + + var stray = new { Setting = "left by an older version" }; + await session.StoreAsync(stray, "LegacySettings/1"); + session.Advanced.GetMetadataFor(stray)["@collection"] = "LegacySettings"; + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingEndpoints); + + var rows = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToList(); + var count = await source.Count(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Has.None.StartsWith("LegacySettings/"), "the product reads endpoints only from the EndpointDocuments collection, so a document of any other kind is not an endpoint"); + Assert.That(rows, Has.Count.EqualTo(1)); + Assert.That(count, Is.EqualTo(rows.Count), "the count reads what the copy reads, so --migration-verify expects no row for the stray document"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs new file mode 100644 index 0000000000..a2a7b1c7e3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs @@ -0,0 +1,119 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput.Models; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; + +class LicensingSettingsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_the_report_masks_document_as_its_list_of_masked_strings() + { + await Seed(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso", "fabrikam"] }); + + var batches = await Read(MigrationCategoryIds.LicensingReportMasks); + + var row = batches.SelectMany(batch => batch.Rows).Single(); + using (Assert.EnterMultipleScope()) + { + Assert.That(row.SourceId, Is.EqualTo("ReportMasks")); + Assert.That((List)row.Document, Is.EqualTo(new[] { "contoso", "fabrikam" })); + Assert.That(batches.Last().Cursor, Is.EqualTo("ReportMasks")); + } + } + + [Test] + public async Task Reads_the_licensed_endpoint_details_document_whole() + { + var seeded = Details; + await Seed(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, seeded); + + var batches = await Read(MigrationCategoryIds.LicensedEndpointDetails); + + var details = (LicensedEndpointDetails)batches.SelectMany(batch => batch.Rows).Single().Document; + using (Assert.EnterMultipleScope()) + { + Assert.That(JsonSerializer.Serialize(details), Is.EqualTo(JsonSerializer.Serialize(seeded)), "every field, including the endpoint, queue and product arrays, must survive the read"); + Assert.That(batches.Last().Cursor, Is.EqualTo("LicensedEndpointDetails")); + } + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails, "LicensedEndpointDetails")] + public async Task An_absent_document_gives_one_empty_batch_that_still_records_the_cursor(string categoryId, string documentId) + { + var batches = await Read(categoryId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { documentId })); + } + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails, "LicensedEndpointDetails")] + public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string categoryId, string documentId) + { + await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); + await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); + + var batches = await Read(categoryId, documentId); + + Assert.That(batches, Is.Empty); + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks)] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] + public async Task A_read_resumed_after_a_cursor_the_reader_never_issued_is_refused(string categoryId) + { + await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); + await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); + + var exception = Assert.ThrowsAsync(() => Read(categoryId, "Sales.Orders/Broker")); + + Assert.That(exception!.Message, Does.Contain(categoryId).And.Contain("'Sales.Orders/Broker'")); + } + + [Test] + public async Task A_report_masks_document_whose_list_is_null_reads_as_an_empty_list() + { + await Seed(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = null }); + + var batches = await Read(MigrationCategoryIds.LicensingReportMasks); + + var row = batches.SelectMany(batch => batch.Rows).Single(); + Assert.That(row.Document, Is.InstanceOf>().And.Empty, "the product shows a null list as no masks, and a null document halts the writer with a reader and writer mismatch"); + } + + static LicensedEndpointDetails Details => new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + async Task Seed(string id, object document) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(document, id); + await session.SaveChangesAsync(); + } + + async Task> Read(string categoryId, string resumeAfter = null) + { + await using var source = await OpenMigrationSource(); + return await CollectBatches(source, MigrationCategoryRegistry.All.Single(entry => entry.Id == categoryId), resumeAfter); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs new file mode 100644 index 0000000000..065b0fa62e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Conventions; +using ServiceControl.Persistence.DataMigration; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +class LicensingThroughputSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_one_endpoints_whole_time_series_as_one_batch_and_cursors_on_its_document_id() + { + var documentId = "Sales.Orders/Broker"; + + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName) + .Increment(new DateTime(2026, 1, 1), 100); + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName) + .Increment(new DateTime(2026, 1, 2), 250); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + + Assert.That(batches, Has.Count.EqualTo(1)); + Assert.That(batches[0].Cursor, Is.EqualTo(documentId)); + Assert.That(batches[0].Rows.Select(row => row.Document), Is.EqualTo(new[] + { + new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100), + new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 2), 250) + }), "each row carries the endpoint its count belongs to, because EndpointDailyThroughput does not, and the day and total RavenDB holds"); + Assert.That(batches[0].Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "Sales.Orders", "Sales.Orders" }), "the endpoint's name keeps its case, which the record's equality ignores"); + } + + [Test] + public async Task A_resume_reads_only_the_endpoints_after_the_cursor_whatever_order_they_were_stored_in() + { + foreach (var name in new[] { "Z", "A", "M" }) + { + var documentId = $"{name}/Broker"; + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName).Increment(new DateTime(2026, 1, 1), 100); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "A/Broker", "M/Broker", "Z/Broker" }), "one endpoint per batch, in id order rather than the order RavenDB stored them"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "M", "Z" }), "a resume reads only the days of endpoints after the cursor"); + } + } + + [Test] + public async Task Endpoints_whose_names_differ_only_in_case_and_trailing_spaces_come_in_one_batch() + { + foreach (var name in new[] { "Sales.Orders", "Sales.Orders.Archive", "SALES.ORDERS " }) + { + var documentId = $"{name}/Broker"; + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName).Increment(new DateTime(2026, 1, 1), 100); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "Sales.Orders.Archive/Broker", "Sales.Orders/Broker" }), "each group cursors on its last id in ordinal order, so a resume never splits a group"); + Assert.That(batches[1].Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EquivalentTo(new[] { "Sales.Orders", "SALES.ORDERS " }), "the writer can add the two endpoints' days only when they arrive in one batch"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EquivalentTo(new[] { "Sales.Orders", "SALES.ORDERS " }), "a resume after the first group reads the whole second group"); + } + } + + [Test] + public async Task Reads_more_endpoints_than_one_session_accepts_requests() + { + var endpointCount = new DocumentConventions().MaxNumberOfRequestsPerSession + 1; + + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + for (var index = 0; index < endpointCount; index++) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier($"Endpoint{index}", ThroughputSource.Broker)), $"Endpoint{index}/Broker"); + } + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + + Assert.That(batches, Has.Count.EqualTo(endpointCount), "each endpoint's series is its own request, and one session refuses this many"); + Assert.That(await source.Count(category), Is.Zero); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs index 358c3f9943..f940d6f8ba 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs @@ -13,6 +13,8 @@ abstract class RavenMigrationSourceTestBase : RavenPersistenceTestBase static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); static readonly object SettingsGate = new(); + protected string ThroughputDatabaseName => ((RavenPersisterSettings)PersistenceSettings).ThroughputDatabaseName; + protected async Task OpenMigrationSource() { var settings = (RavenPersisterSettings)PersistenceSettings; diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index e97f52a466..463c008a2f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -5,9 +5,12 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using System.Linq; using System.Threading.Tasks; using NUnit.Framework; +using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations.Indexes; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput.Models; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; class RavenMigrationSourceTests : RavenMigrationSourceTestBase { @@ -126,7 +129,11 @@ public async Task Reading_every_category_creates_no_index() { [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }), - [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()) + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()), + [MigrationCategoryIds.LicensingEndpoints] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), + [MigrationCategoryIds.LicensingThroughput] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), + [MigrationCategoryIds.LicensingReportMasks] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso"] }), + [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }) }; string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; @@ -138,6 +145,13 @@ async Task IndexNames() => static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.Find("EndpointSettings"); + async Task SeedThroughputDocument(string id, object document) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(document, id); + await session.SaveChangesAsync(); + } + async Task SeedEndpointSettings(int count) { for (var index = 0; index < count; index++) diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs new file mode 100644 index 0000000000..1c6cba5d3d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +class LicensedEndpointDetailsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory LicensedEndpointDetailsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensedEndpointDetails); + + [Test] + public async Task Count_is_the_licensed_endpoint_details_row_alone() + { + await LicensingDataStore.SaveReportMasks(["contoso"]); + Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.Zero, "the report masks row is not a licensed endpoint details row"); + + await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); + + Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the report masks row and every other row in the table"); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs new file mode 100644 index 0000000000..2eb2c74e25 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingEndpointsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory LicensingEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + + [Test] + public async Task Count_is_the_licensing_endpoint_rows_the_target_holds() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Audit) { SanitizedName = "sales.orders" }); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 2), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Audit, new DateOnly(2026, 1, 1), 5); + + Assert.That(await Target.Count(LicensingEndpointsCategory), Is.EqualTo(2), "one row per name and source, and not the three throughput days that hang off them"); + } + + [Test] + public async Task A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fitsExactly = new Endpoint(new string('n', 300), ThroughputSource.Broker) { SanitizedName = new string('s', 450) }; + var longName = new Endpoint(new string('n', 301), ThroughputSource.Audit) { SanitizedName = "sales.orders" }; + var longSanitizedName = new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = new string('s', 451) }; + var batch = BatchOf(("Endpoint/1", fitsExactly), ("Endpoint/2", longName), ("Endpoint/3", longSanitizedName)); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(LicensingEndpointsCategory, batch, CheckpointAfter("Endpoint/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "Endpoint/2", "Endpoint/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await LicensingDataStore.GetAllEndpoints(includePlatformEndpoints: true)).Select(endpoint => endpoint.Id.Name), Is.EqualTo(new[] { new string('n', 300) })); + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details[0], Does.EndWith("takes at most 300 characters and the value holds 301, on document 'Endpoint/2'")); + Assert.That(details[1], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Endpoint/3'")); + } + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs new file mode 100644 index 0000000000..f7c2d37322 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class LicensingMigrationMapperCoverageTests : PersistenceTestBase +{ + [Test] + public async Task The_endpoint_mapper_sets_every_mapped_property() + { + var endpoint = new Endpoint("Sales.Orders", ThroughputSource.Audit) + { + SanitizedName = "Sales.Orders", + EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()], + UserIndicator = "Sales team", + Scope = "Sales" + }; + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(await Model(), LicensingEndpointRowMapper.Map(endpoint)); + } + + [Test] + public async Task The_throughput_mapper_sets_every_mapped_property() + { + var day = new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Audit), new DateOnly(2026, 1, 1), 100); + + var row = LicensingThroughputRowMapper.Map(day); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(await Model(), row); + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NormalizedName, Is.EqualTo("sales.orders"), "the key must be the name the endpoint row is stored under, or the foreign key refuses the day"); + Assert.That(row.ThroughputSource, Is.EqualTo(ThroughputSource.Audit)); + Assert.That(row.DateUtc, Is.EqualTo(new DateOnly(2026, 1, 1))); + Assert.That(row.MessageCount, Is.EqualTo(100)); + } + } + + async Task Model() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + return scope.ServiceProvider.GetRequiredService().Model; + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs new file mode 100644 index 0000000000..45faa38e2e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +class LicensingReportMasksMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory LicensingReportMasksCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingReportMasks); + + [Test] + public async Task Count_is_the_report_masks_row_alone() + { + await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); + Assert.That(await Target.Count(LicensingReportMasksCategory), Is.Zero, "the licensed endpoint details row is not a report masks row"); + + await LicensingDataStore.SaveReportMasks(["contoso"]); + + Assert.That(await Target.Count(LicensingReportMasksCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the licensed endpoint details row and every other row in the table"); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs new file mode 100644 index 0000000000..bcae1c840f --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs @@ -0,0 +1,193 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingThroughputMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory LicensingThroughputCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingThroughput); + + [Test] + public async Task Count_is_the_daily_throughput_rows_the_target_holds() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await LicensingDataStore.SaveEndpoint(new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = "billing" }); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 2), 5); + await LicensingDataStore.RecordEndpointThroughput("Billing", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + + Assert.That(await Target.Count(LicensingThroughputCategory), Is.EqualTo(3), "one row per endpoint and day, and not the two endpoint rows they hang off"); + } + + [Test] + public async Task A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100)), + ("long/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier(new string('n', 301), ThroughputSource.Broker), day, 250))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new LicensingThroughputWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("long/Broker|2026-01-01")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "long/Broker|2026-01-01" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 1 }), "the target cannot hold an endpoint with this name, so the day must be skipped here rather than fail the whole batch at the endpoint check"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(LicensingEndpointThroughputEntity.NormalizedName))} takes at most 300 characters and the value holds 301, on document 'long/Broker|2026-01-01'" + })); + } + } + + [Test] + public async Task A_day_repeated_in_one_batch_keeps_its_last_row_and_counts_the_other_as_already_present() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01|b", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250)), + ("Sales.Orders/Broker|2026-01-01|a", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100))); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker|2026-01-01|b")); + await using var scope = ServiceProvider.CreateAsyncScope(); + var stored = await scope.ServiceProvider.GetRequiredService().LicensingEndpointThroughput.Select(row => row.MessageCount).SingleAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "PostgreSQL refuses a statement that updates one row twice, so the repeat is dropped before it and counted rather than failing the batch"); + Assert.That(stored, Is.EqualTo(250), "the row whose id sorts last by ordinal order sets the day, whatever order the batch arrives in"); + } + } + + [Test] + public async Task Days_of_two_endpoints_the_key_treats_as_one_are_added_when_they_arrive_in_one_batch() + { + var endpoints = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + var endpointBatch = BatchOf( + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100)), + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))); + + var first = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + var again = await Target.Write(LicensingThroughputCategory, batch, first.Saved); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var keyIgnoresTrailingSpace = scope.ServiceProvider.GetRequiredService().Database.ProviderName!.Contains("SqlServer"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2)); + Assert.That(first.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the endpoint folded into the other's row is counted, not lost"); + Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 350L } : new[] { 100L, 250L }), "SQL Server's key makes the two one day, which holds both counts, as the product's own throughput write would"); + Assert.That(again.Copied, Is.EqualTo(first.Copied), "copying the batch again sets the same sum rather than adding to it"); + } + } + + [Test] + public async Task A_later_batch_replaces_the_count_an_earlier_batch_stored_for_the_same_day() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + + var first = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100))), CheckpointAfter("Sales.Orders/Broker")); + var second = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))), first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.EqualTo(1), "an update is a row the statement wrote"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 250L }), "the count RavenDB holds now replaces the stored one, and is never added to it or ignored"); + } + } + + [Test] + public async Task One_endpoint_with_more_days_than_one_statement_takes_stores_every_day_with_its_own_count() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var dialect = ServiceProvider.GetRequiredService(); + int dayCount; + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + dayCount = dialect.RowsPerStatementFor(scope.ServiceProvider.GetRequiredService()) + 1; + } + + var firstDay = new DateOnly(2026, 1, 1); + var days = Enumerable.Range(0, dayCount).Select(offset => (Date: firstDay.AddDays(offset), Count: 1000L + offset)).ToArray(); + var batch = BatchOf([.. days.Select(day => ($"Sales.Orders/Broker|{day.Date:yyyy-MM-dd}", (object)new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day.Date, day.Count)))]); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + await using var readScope = ServiceProvider.CreateAsyncScope(); + var stored = await readScope.ServiceProvider.GetRequiredService().LicensingEndpointThroughput + .OrderBy(row => row.DateUtc) + .Select(row => new { row.DateUtc, row.MessageCount }) + .ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(dayCount), "the batch runs as two statements, and Copied adds up what both wrote"); + Assert.That(result.AlreadyPresent, Is.Zero, "a day the second statement wrote must not be counted as already present"); + Assert.That(stored.Select(row => (row.DateUtc, row.MessageCount)), Is.EqualTo(days), "every day is stored with its own count, including the day past the first statement"); + } + } + + [Test] + public async Task A_later_batch_sets_a_day_stored_under_a_name_that_differs_only_by_a_trailing_space() + { + var endpoints = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + var endpointBatch = BatchOf( + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); + var day = new DateOnly(2026, 1, 1); + + var first = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100))), CheckpointAfter("Sales.Orders /Broker")); + var second = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))), first.Saved); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var keyIgnoresTrailingSpace = scope.ServiceProvider.GetRequiredService().Database.ProviderName!.Contains("SqlServer"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.EqualTo(1), "the second batch's day is matched back to the row that produced it, not refused because the stored name keeps its trailing space"); + Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 250L } : new[] { 100L, 250L }), "SQL Server's key makes the two names one day, which the later count sets; PostgreSQL's keeps two"); + } + } + + async Task StoredCounts() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + return await scope.ServiceProvider.GetRequiredService().LicensingEndpointThroughput.Select(row => row.MessageCount).OrderBy(count => count).ToArrayAsync(); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs new file mode 100644 index 0000000000..629d6d1684 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationInsertDateKeyTests : PersistenceTestBase +{ + [Test] + public async Task A_row_whose_key_holds_a_date_is_matched_back_to_the_row_that_produced_it() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var dialect = ServiceProvider.GetRequiredService(); + var day = new LicensingEndpointThroughputEntity { NormalizedName = "sales.orders", ThroughputSource = ThroughputSource.Broker, DateUtc = new DateOnly(2026, 1, 1), MessageCount = 100 }; + + await using var scope = ServiceProvider.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + await dialect.Open(dbContext); + + var inserted = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + var rows = await dialect.InsertMissing(dbContext, [day]); + await transaction.CommitAsync(); + return rows; + }); + + Assert.That(inserted, Is.EqualTo(new[] { day }), "a date key comes back from the database as DateTime unless it is read as the type its mapping stores, and the exact lookup then refuses the row"); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index eb7caebdcc..0cf8345618 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -1,11 +1,15 @@ namespace ServiceControl.Persistence.Tests; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; class MigrationTargetCoverageTests : PersistenceTestBase { @@ -30,15 +34,29 @@ public void Every_category_the_target_supports_has_a_test_for_a_value_longer_tha { var target = ServiceProvider.GetRequiredService(); - Assert.That(target.SupportedCategoryIds, Is.EquivalentTo(ValueTooLongTests.Keys), + Assert.That(target.SupportedCategoryIds, Is.EquivalentTo(ValueTooLongTests.Keys.Concat(NoValueCanBeTooLong)), "a writer that does not run the length check sends an over-long value to the insert, which fails the whole statement and halts the category; give the new category a ValueTooLong test like those listed here, then add it to this list"); } + [Test] + public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_can_be_too_long() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + Assert.That(model.FindEntityType(typeof(SettingEntity))!.FindProperty(nameof(SettingEntity.Value))!.GetMaxLength(), Is.Null, "NoValueCanBeTooLong exempts the settings categories because of this; give them a value-too-long test if the column gains a limit"); + } + + // Each is one Settings row under a constant key whose value column has no maximum length, so no source value can be too long for it. + static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails]; + // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. static readonly Dictionary ValueTooLongTests = new() { [MigrationCategoryIds.KnownEndpoints] = nameof(KnownEndpointsMigrationTargetTests.A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), - [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy) + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), + [MigrationCategoryIds.LicensingEndpoints] = nameof(LicensingEndpointsMigrationTargetTests.A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs new file mode 100644 index 0000000000..84b27862f8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SettingRowMapperTests : PersistenceTestBase +{ + static readonly List ReportMasks = ["contoso", "fabrikam"]; + + static readonly LicensedEndpointDetails Details = new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, SettingRowMapper.Map(SettingKeys.ReportMasks, ReportMasks)); + } + + [Test] + public async Task Licensing_settings_read_back_through_GetSetting_as_the_product_reads_them() + { + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.Settings.AddRange(SettingRowMapper.Map(SettingKeys.ReportMasks, ReportMasks), SettingRowMapper.Map(SettingKeys.LicensedEndpointDetails, Details)); + await dbContext.SaveChangesAsync(); + } + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + + Assert.That(await dbContext.GetSetting>(SettingKeys.ReportMasks), Is.EqualTo(ReportMasks)); + Assert.That(JsonSerializer.Serialize(await dbContext.GetSetting(SettingKeys.LicensedEndpointDetails)), Is.EqualTo(JsonSerializer.Serialize(Details))); + } + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs b/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs new file mode 100644 index 0000000000..bcbfb5431c --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using Particular.LicensingComponent.Contracts; + +/// +/// One endpoint's message count for one UTC day, as a licensing throughput row crosses from source to target. +/// Endpoint is the endpoint the count belongs to, DateUtc is the day, and MessageCount is that day's whole total. +/// +public sealed record ThroughputDayRow(EndpointIdentifier Endpoint, DateOnly DateUtc, long MessageCount); diff --git a/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj b/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj index 131f2cfa0e..876ea8a78a 100644 --- a/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj +++ b/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj @@ -6,6 +6,7 @@ + From ea4bbb9ed76f417f57b9cdab7605547007ad8f0c Mon Sep 17 00:00:00 2001 From: Rhys Bevilaqua Date: Fri, 13 Nov 2026 09:13:54 +0100 Subject: [PATCH 19/28] Change offline migration step to an interactive cli action (#6106) * Change offline migration step to an interactive cli action * Split step * Refactor migration cancellation token handling in MigrateCommand and OptionalCategoryCopier * Refine migration documentation by removing redundant statements and clarifying instructions * Remove redundant exception handling in MigrateCommand * Update exception message in TargetHoldsNoServiceControlDataCheck for clarity on migration steps * Remove CheckpointMigrationState and IMigrationState interfaces along with related tests. Not needed now with the --migrate command * Enhance migration documentation and error messages for clarity * Add "RecordHostOpenedOnTarget" to the list of services in When_hosting_error_ingestion_only test for idempotent upsert * Update migration commands from --migrate to --start-data-migration across documentation and codebase --------- Co-authored-by: Warwick Schroeder --- docs/README.md | 1 + .../ravendb-to-sql-migration-instructions.md | 97 ++++++++++----- .../ravendb-to-sql-migration-overview.md | 68 ++++++----- .../ravendb-to-sql-migration-system-design.md | 55 +++++---- .../When_hosting_error_ingestion_only.cs | 3 +- .../MigrationAcceptanceTest.cs | 34 ++++-- .../When_a_copy_is_killed_mid_category.cs | 22 +--- .../When_a_startup_check_fails.cs | 12 +- ...When_categories_change_between_restarts.cs | 6 +- .../When_endpoint_settings_are_migrated.cs | 2 +- ..._api_is_called_during_the_required_copy.cs | 34 +++--- .../When_the_host_opens_on_the_target.cs | 46 +++----- ...unresolved_failed_messages_are_migrated.cs | 6 +- .../TargetHoldsNoServiceControlDataCheck.cs | 2 +- .../DataMigration/CheckpointMigrationState.cs | 42 ------- .../DataMigration/IMigrationStartupCheck.cs | 4 +- .../DataMigration/IMigrationState.cs | 13 -- .../DataMigration/MigrationSettings.cs | 6 - ...rovals.PlatformSampleSettings.approved.txt | 1 - .../MigrationSourceReportArgumentTests.cs | 8 ++ .../CheckpointMigrationStateTests.cs | 111 ------------------ .../MigrationBeforeTheHostOpensTests.cs | 72 ++++++++++++ .../MigrationEnabledSettingsTests.cs | 25 +--- .../MigrationIsReleasedCheckTests.cs | 6 +- .../Migration/OptionalCategoryCopierTests.cs | 78 ++++++++++++ .../Migration/StoppedCopyExplanationTests.cs | 2 +- .../HostApplicationBuilderExtensions.cs | 5 - .../Commands/ErrorIngestionOnlyCommand.cs | 5 +- .../Commands/ImportFailedErrorsCommand.cs | 1 + .../Hosting/Commands/MigrateCommand.cs | 63 ++++++++++ .../Hosting/Commands/RunCommand.cs | 29 +---- src/ServiceControl/Hosting/Help.txt | 10 ++ src/ServiceControl/Hosting/HostArguments.cs | 20 ++++ .../Infrastructure/Settings/Settings.cs | 2 - .../Checks/MigrationIsReleasedCheck.cs | 2 +- .../Checks/MigrationPairIsSupportedCheck.cs | 2 +- .../Checks/RetryHistoryDepthIsSafeCheck.cs | 2 +- .../FinishedCopyBeforeAnIngestionNodeOpens.cs | 12 +- .../Migration/MigrationBeforeTheHostOpens.cs | 32 +++++ .../Migration/MigrationStartup.cs | 47 +++----- .../Migration/MigrationStartupCheckRunner.cs | 4 +- .../Migration/OptionalCategoryCopier.cs | 88 ++++++++++++++ .../Migration/RecordHostOpenedOnTarget.cs | 9 +- .../RequiredCopyBeforeTheHostOpens.cs | 30 ----- 44 files changed, 630 insertions(+), 489 deletions(-) delete mode 100644 src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs delete mode 100644 src/ServiceControl.Persistence/DataMigration/IMigrationState.cs delete mode 100644 src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs create mode 100644 src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs create mode 100644 src/ServiceControl/Hosting/Commands/MigrateCommand.cs create mode 100644 src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs create mode 100644 src/ServiceControl/Migration/OptionalCategoryCopier.cs delete mode 100644 src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs diff --git a/docs/README.md b/docs/README.md index 4c6ebc1521..3c8cd29743 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,6 +26,7 @@ This section points to sources that explain why ServiceControl is designed the w - [Message body storage](message-body-storage.md) — where the Audit SQL Server and PostgreSQL persisters keep message bodies, how retention removes them, and which settings are optional - [Bulk retries design](bulk-retries-design.md) — how ServiceControl retries failed messages in bulk - [Retries over Azure Storage Queues transport](retries-asq-transport.md) — how retries behave with one or several storage accounts +- [RavenDB-to-SQL migration design](migration/ravendb-to-sql-migration-overview.md) — why required data copies offline and persisted checkpoints drive online completion - [Data versioning design](data-versioning-design.md) — which invariant keeps a cached API response from going stale - [Event log design](eventlog-design.md) — what the event log is and what it records - [Multiple ServiceControl instances communication](multipleservicecontrolinstancescommunication.md) — how primary, audit, and monitoring instances talk to each other diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index 88be796b03..d1879cf889 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -1,21 +1,61 @@ # Migration Engine: Instructions > [!NOTE] -> This build does not copy data yet. Only `--migration-source-report` works, and ServiceControl does not start if `ServiceControl/Migration/Enabled` is `true`. +> Production copying remains blocked until the complete migration ships. `--migration-source-report` works; `--start-data-migration` reports that copying is unavailable. -This page tells you how to move the data of a ServiceControl error instance from RavenDB to SQL Server or PostgreSQL. ServiceControl copies the data itself, so you do not need a separate tool. [The migration overview](ravendb-to-sql-migration-overview.md) explains how the migration works and why. +This page tells you how to move the data of a ServiceControl error instance from RavenDB to SQL Server or PostgreSQL. The ServiceControl executable provides the offline command and the online copier, so you do not need a separate tool. [The migration overview](ravendb-to-sql-migration-overview.md) explains how the migration works and why. The migration does not move the audit instance or the monitoring instance. The audit instance has no migration path and the monitoring instance keeps its data in memory, so it has nothing to move. ## How the migration runs -ServiceControl copies the data it needs to open while it is closed. **This part is your outage**. Then it opens and copies the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. +Run `--start-data-migration` with ServiceControl stopped to copy the data it needs to open. **This part is your outage**. After the command succeeds, start ServiceControl to copy the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. > [!IMPORTANT] > Until ServiceControl opens, you can go back to RavenDB and lose nothing. After it opens, you cannot go back. The migration copies the data in categories, and each category is Copying, Done, Failed or Abandoned. [The overview](ravendb-to-sql-migration-overview.md#when-something-goes-wrong-you-decide-what-happens-next) explains the categories and the states. +The steps below follow this path: + +```mermaid +flowchart TD + subgraph prepare["ServiceControl runs on RavenDB"] + S1["1. Upgrade and run --migration-source-report"] + S2["2. Create an empty SQL database and run --setup"] + S3["3. Choose the optional data windows"] + S4["4. Run --migration-dry-run and book the outage"] + S5["5. Back up RavenDB"] + S1 --> S2 --> S3 --> S4 --> S5 + end + + subgraph outage["ServiceControl stopped: the outage"] + S6["6. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] + Q1{"Did --start-data-migration succeed?"} + Fix["Correct the reported cause"] + RetryAbandon1["Run --migration-retry or --migration-abandon for the Failed category"] + Back["Go back: set PersistenceType to RavenDB and start ServiceControl"] + S6 --> Q1 + Q1 -->|"No: a check failed or the command stopped"| Fix --> S6 + Q1 -->|"No: a required category is Failed"| RetryAbandon1 --> S6 + Q1 -.->|"Give up, nothing is lost"| Back + end + + subgraph online["ServiceControl open on SQL: no way back to RavenDB"] + S7["7. Start ServiceControl"] + S8["8. Watch the background copy with --migration-status"] + Q2{"Is an optional category Failed?"} + RetryAbandon2["Stop ServiceControl, run --migration-retry or --migration-abandon, then start it"] + S9["9. Stop ServiceControl, run --migration-verify, then start it"] + S7 --> S8 --> Q2 + Q2 -->|Yes| RetryAbandon2 --> S8 + Q2 -->|"No: every category is Done or Abandoned"| S9 + end + + S5 --> S6 + Q1 -->|Yes| S7 +``` + ## Before you start Make sure that your setup is supported: @@ -59,7 +99,7 @@ In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You > [!IMPORTANT] > > - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. -> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database without the migration turned on, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. +> - Do not start ServiceControl again until step 7. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: @@ -96,10 +136,9 @@ The report exits with 0 when it prints a report, and with 1 when it cannot. If t 4. Set `ServiceControl/MessageBody/StorageType` to `FileSystem`, `AzureBlob` or `S3`, and set the settings for that storage type. 5. Run `ServiceControl.exe --setup`. This creates the SQL schema, which includes the table where the migration saves its progress. -## Step 3: Turn the migration on +## Step 3: Choose the optional data -1. Set `ServiceControl/Migration/Enabled` to `true`. -2. If you want less optional data, set a shorter window for the optional categories. A window is how far back an optional category copies. Set it to `0` to turn the category off. +If you want less optional data, set a shorter window for the optional categories. A window is how far back an optional category copies. Set it to `0` to turn the category off. The SQL checkpoints record the migration. | Setting | Default | What it does | | --- | --- | --- | @@ -121,21 +160,27 @@ Book the outage from the estimate in the report, and add a margin. It is an esti RavenDB keeps deleting old failed messages and event log items during the migration and after it. Without a backup, the RavenDB database that you keep to go back to loses data every day. -## Step 6: Start ServiceControl +## Step 6: Run the offline command + +1. Stop ServiceControl and every ingestion worker. +2. Run `ServiceControl.exe --start-data-migration` from the instance installation folder. In a container deployment, run the command in a one-off container against the same databases. +3. Wait for the command to finish. It runs the startup checks, copies every required category, and saves progress in SQL. It never starts the API, ingestion, or other hosted services. +4. If the command fails, correct the reported cause. Run `--start-data-migration` again to resume. A Failed category still needs retry or abandon first. + +The command exits without starting ServiceControl. + +## Step 7: Start ServiceControl -Start the ServiceControl service, or the container. ServiceControl then does these things in order: +After the offline command reports success, start the ServiceControl service or container. ServiceControl detects the migration from SQL checkpoints and copies optional categories in the background. -1. It runs every startup check. If a check fails, ServiceControl does not start and names the check. It copies nothing. -2. It copies every required category while it is closed. If a required category fails, ServiceControl stays closed. It says which category failed, why, and which command to run. -3. It opens when every required category is Done or Abandoned. You do not need to restart it. -4. It copies the optional categories in the background. +Normal startup refuses an unfinished required copy and tells you to run `--start-data-migration`; it never resumes that copy itself. > [!IMPORTANT] -> Do not start any `--error-ingestion-only` worker until ServiceControl opens. A worker refuses to start while a required category is unfinished, but a worker that starts at the same time as ServiceControl can start before the copy saves its first progress, and nothing stops it. +> Do not start any `--error-ingestion-only` worker until the offline command succeeds and the main ServiceControl instance opens. A worker cannot detect a migration before its first checkpoint exists. If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). -## Step 7: Watch the background copy +## Step 8: Watch the background copy ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: @@ -146,28 +191,27 @@ ServiceControl is open and works as usual while the optional categories copy. Yo If the copy slows down your production work, raise `ServiceControl/Migration/ThrottlePauseMilliseconds` and restart. This is the pause between two background batches. The default is 100 milliseconds. > [!IMPORTANT] -> Turning the migration off does not stop the copy, because ServiceControl refuses to start until every category is Done or Abandoned. See [how to abandon a category](#a-category-is-failed). +> To give up an optional category, use [the abandon command](#to-abandon-a-category). Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. -## Step 8: Finish the migration +## Step 9: Finish the migration 1. Stop ServiceControl. 2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. -3. Set `ServiceControl/Migration/Enabled` to `false`. -4. Start ServiceControl. +3. Start ServiceControl. Finished checkpoints remain in SQL, but no background copy runs and RavenDB is not opened. Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time. If verify shows rows that the copy did not read, keep RavenDB. Those rows are only in RavenDB. RavenDB can show fewer rows than the copy read when nothing is wrong, because RavenDB deletes old rows. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) explains why. -If you turn the migration off before every category is Done or Abandoned, ServiceControl refuses to start. The refusal names each unfinished category and the ways to continue. +No configuration change is needed to finish. Once every category is Done or Abandoned, subsequent starts leave RavenDB unopened. ## If something goes wrong ### A startup check fails -ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then start ServiceControl again. The dry run runs the same checks, so you can use it to test your correction. +ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then run `--start-data-migration` again. The dry run runs the same checks, so you can use it to test your correction. ### A category is Failed @@ -179,13 +223,13 @@ A category becomes Failed when it cannot copy rows, or when an error stops it. T 2. Read why the category failed. The reason is in the startup refusal and in the output of `--migration-status`. 3. Correct the cause. It is usually outside the migration, for example the body storage cannot be reached, a certificate expired, a database is down, or a disk is full. 4. Run `--migration-retry ` to reset the migration progress for that category. -5. Start ServiceControl. It copies the category again from the start. +5. For a required category, run `--start-data-migration` again before starting ServiceControl. For an optional category, start ServiceControl. The category copies again from the start. #### To abandon a category 1. Stop ServiceControl. 2. Run `--migration-abandon `. When you abandon a category, the categories that depend on it are abandoned too. -3. Start ServiceControl. +3. If any required category remains unfinished, run `--start-data-migration` again. Otherwise, start ServiceControl. > [!IMPORTANT] > **Abandon is final**. The rows that the category already copied stay in SQL, and the rest are not copied. Abandon a category when a retry cannot fix it, or when the data is not worth the time. You cannot retry the event log, so you can only abandon it. @@ -196,9 +240,8 @@ You can abandon a required category only after it starts copying. You can abando You can go back to RavenDB without losing data only while ServiceControl is still closed: -1. Set `ServiceControl/Migration/Enabled` to `false`. -2. Set `ServiceControl/PersistenceType` back to RavenDB. -3. Start ServiceControl. +1. Set `ServiceControl/PersistenceType` back to RavenDB. +2. Start ServiceControl. You lose only the copy. To try the migration again later, start from step 2 with a new, empty SQL database. @@ -228,7 +271,7 @@ On an embedded source, stop ServiceControl before you run a command that opens R In a container, run each command as a one-off container of the same image, against the same databases: ```shell -docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-status +docker run --rm -it --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --start-data-migration ``` The RavenDB source must be an external server. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 9c4bbb470e..1b3c015e16 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -10,7 +10,7 @@ This page describes the migration as designed, not what is built today. [The ins ### What this is - The migration moves an **error instance's** data from RavenDB to SQL Server or PostgreSQL, so a customer can switch persisters and keep their data. -- It runs inside ServiceControl itself. There is no separate tool. +- The ServiceControl executable provides an interactive offline command and an online background copier. There is no separate tool. - **The audit instance is out of scope.** There is no migration path for the audit instance. - **The monitoring instance is out of scope.** It keeps its data in memory, so there is nothing to move. @@ -25,9 +25,10 @@ This page describes the migration as designed, not what is built today. [The ins ```mermaid flowchart LR - A["Checks"] --> B["Copy required data
ServiceControl closed"] - B --> V["Required all Done
or Abandoned"] --> C["ServiceControl opens
optional data copies
in the background"] - C --> D["Everything Done or
Abandoned: turn
migration off"] + A["--start-data-migration: checks"] --> B["Copy required data
ServiceControl stopped"] + B --> V["Required all Done
or Abandoned"] --> P["Command exits:
start ServiceControl"] + P --> C["ServiceControl detects checkpoints
optional data copies
in the background"] + C --> D["Everything Done or
Abandoned: no further copying"] ``` ### When something goes wrong, you decide what happens next @@ -37,7 +38,7 @@ The end goal is simple: **either all the data comes across, or you explicitly gi - **Each *category* of data is in one of four states:** Copying, Done, Failed or Abandoned. - **A category goes Failed when something went wrong**, for example a row it could not copy. It stays Failed, and the status says why. - **You act with one of two commands**, with ServiceControl stopped: - - `--migration-retry `, after fixing the cause: the next start copies the category again from the beginning. + - `--migration-retry `, after fixing the cause: rerun `--start-data-migration` for a required category, or restart ServiceControl for an optional category. - `--migration-abandon `: give up on what it has not copied, and keep what it has. - **A restart never decides anything for you.** A Failed category stays Failed until you run a command, so a restart policy cannot loop or lose data. - **ServiceControl opens once every required category is Done or Abandoned**, and the migration is finished once every category is. @@ -98,8 +99,8 @@ flowchart LR [The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. -- **The [free abort](#going-back-to-ravendb) window runs from the first start with migration on until ServiceControl opens.** Until then, you can go back to RavenDB and lose only the copy. -- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The required copy runs before any of them starts, so they start only once it finishes. Optional categories never hold them back. +- **The [free abort](#going-back-to-ravendb) window runs from the first `--start-data-migration` invocation until ServiceControl opens.** A successful command does not end this window; starting ServiceControl does. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command builds a persistence-only host and never starts it. Normal startup refuses unfinished required checkpoints before any hosted service starts. Optional categories never hold them back. ## Design @@ -249,7 +250,7 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied In this order, cheapest first: -1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. A RavenDB instance with migration on is refused here, because RavenDB is only ever the source. +1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. `--start-data-migration` refuses a RavenDB target, because RavenDB is only ever the source. 2. The optional category windows are valid time spans of zero or more. 3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history, and no row count would ever show it. 4. The SQL schema is current. This also refuses a database upgraded without `--setup`. @@ -262,18 +263,18 @@ In this order, cheapest first: - an external server is at least the RavenDB client's version; - both databases load, within 5 minutes each on an embedded source. - If every required category is already Done or Abandoned, a source that will not open does not stop the start. ServiceControl opens, and the optional categories wait for the source. + If every required category is already Done or Abandoned, the offline command can succeed without opening the source. Normal startup does not need the source; only the online copier opens it. Checks stop at the first one that refuses, and the refusal names it. Once open, the source can add checks of its own. The RavenDB source adds none. ```mermaid flowchart LR - A["Checks pass?"] -->|No| X["Host stays closed
and says why"] + A["--start-data-migration: checks pass?"] -->|No| X["Command fails
ServiceControl stays stopped"] A -->|Yes| B["Copy every required
category"] B --> C["All Done
or Abandoned?"] C -->|No| X - C -->|Yes| D["ServiceControl opens"] - D --> E["Copy optional
categories"] + C -->|Yes| D["Command succeeds:
prompt operator to start ServiceControl"] + D --> E["Start ServiceControl:
copy optional categories"] ``` - **The host also stays closed** if a second instance is writing the same checkpoints. @@ -307,7 +308,7 @@ flowchart LR - **Batch size comes from the database.** SQL Server divides its parameter budget by the column count, PostgreSQL uses a flat 50 rows. - **Background batches have a pause between them**, 100 ms by default, set by `ServiceControl/Migration/ThrottlePauseMilliseconds`. The first batch of each category is not paused. -- **Raise the pause if the copy competes with production.** Turning migration off is not a remedy: the host refuses to start until every category is Done or Abandoned. +- **Raise the pause if the copy competes with production.** To stop an optional category permanently, abandon it. There is no enable switch. - **The required copy is never paused**, because ServiceControl is closed and nothing competes with it. ### Progress and checkpoints @@ -329,7 +330,7 @@ sequenceDiagram ``` - **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. -- **A crash costs only the batch in flight.** The next start carries on from the committed cursor. +- **A crash costs only the batch in flight.** Rerun `--start-data-migration` for required data, or restart ServiceControl for optional data. The copier resumes from the committed cursor. - **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. Unreadable bodies stay off the checkpoint until the write commits. - **Every skip has a reason, or the save is refused.** The per-reason counts must add up exactly to the skipped count, and the target's counts must match what it committed. - **The stored counts always describe the rows in SQL.** The target adds its own outcome and saves the result beside the rows, so nothing is saved later or separately. @@ -351,7 +352,7 @@ A category is *Failed* when something went wrong. Everything already copied stay **What does not make it Failed:** - **Harmless skips.** -- **A shutdown or a crash.** The category stays Copying and the next start carries on from the cursor. +- **A shutdown or a crash.** The category stays Copying. The next `--start-data-migration` invocation resumes required data; the next ServiceControl start resumes optional data. - **An exception or a stall in an optional category.** The error is written on the category, shown in status, and the next start carries on from the cursor. A RavenDB outage after opening is handled the same way. So a network blip never forces anything to be abandoned. If the error keeps coming back, retry or abandon the category. **What a Failed category costs:** @@ -366,18 +367,19 @@ Some losses leave a category Done without any check noticing. They are listed un ### Retry and abandon -Stop ServiceControl, run one of these, then start it again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. +Stop ServiceControl before running either command. Rerun `--start-data-migration` for unfinished required data before starting the service again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. | Command | Works on | What it does | | --- | --- | --- | -| `--migration-retry ` | A Failed category, or an optional one still copying with an error | The next start copies it again from the beginning | +| `--migration-retry ` | A Failed category, or an optional one still copying with an error | The next `--start-data-migration` invocation copies a required category again; the next ServiceControl start copies an optional category again | | `--migration-abandon ` | A Failed or started required category, or any optional one | Gives up on what it has not copied, and on any category whose rows hang off it. What they copied stays in SQL. Final | **To retry:** 1. Read the reason in the refusal, the log or `--migration-status`. It names the skipped rows by reason, or the error. 2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or target down, or the disk full. -3. Run `--migration-retry `, then start ServiceControl. +3. Run `--migration-retry `. +4. For a required category, run `--start-data-migration` again before starting ServiceControl. For an optional category, start ServiceControl. **What a retry does:** @@ -400,19 +402,19 @@ Stop ServiceControl, run one of these, then start it again. In a container, run ### Ingestion workers - **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, as before, because only SQL can scale out. -- **On SQL it starts only when every required category is Done or Abandoned**, whether or not its own migration setting is on. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. -- **That check runs even on a worker someone forgot to flag**, because ingesting into a part-copied database would make going back a loss. +- **On SQL it starts only when every required category is Done or Abandoned**, based on SQL checkpoints. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. +- **Every SQL writer runs that check**, because ingesting into a part-copied database would make going back a loss. - **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. -- **A worker or import that gets past the check marks the target as opened**, whether or not its own migration setting is on. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. +- **A worker or import that gets past the check marks the target as opened**, without an enable setting. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. > [!IMPORTANT] -> **Start workers only after the main host has opened.** Nothing stops one started at the same moment as the first migration start, see [Open decisions](#gaps-nothing-enforces). +> **Stop workers before running `--start-data-migration`, and restart them only after the main host has opened.** A worker cannot detect a migration before its first checkpoint exists, see [Open decisions](#gaps-nothing-enforces). ### Going back to RavenDB **While ServiceControl is closed**, only the copier has written to SQL and the migration has written nothing to RavenDB, which is still the source of truth. This is the *free abort*. RavenDB's own expiration still runs, though, unless you disabled it. -- **To go back:** set `ServiceControl/Migration/Enabled=false`, point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. +- **To go back:** point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. - **To try again later:** start from a new, empty SQL database (`--setup`). > [!IMPORTANT] @@ -420,16 +422,13 @@ Stop ServiceControl, run one of these, then start it again. In a container, run > > **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. -**Turning migration off before every category is Done or Abandoned is refused**, not a quiet exit. That is what stops a migration ending by accident. +- **Every SQL start checks the checkpoint table.** A RavenDB instance has no checkpoint store and skips the check. +- **An unfinished required category keeps the host closed.** Run `--start-data-migration` again, after retrying or abandoning Failed categories. Startup never runs the required copy. +- **Outstanding optional checkpoints activate the online copier.** They do not prevent ServiceControl from opening. Failed optional categories remain Failed until an operator acts. +- **With no outstanding categories, ServiceControl does not open RavenDB.** Completed checkpoints remain as the migration record. +- **Before starting the copy, the offline command seeds required and selected optional checkpoints.** The online phase does not depend on a setting that an operator must remember to retain. -- **Every SQL start checks the checkpoint table, with migration on or off.** It costs nothing when nothing is outstanding, and a RavenDB instance never reaches it. -- **With nothing outstanding, nothing changes.** A SQL instance starts exactly as before, without opening RavenDB. -- **An optional category that never started and whose window is `0` is not outstanding.** It was turned off before it began. -- **With anything outstanding, the host refuses.** It names each category with its state, counts and last error, and the ways out: - - turn migration back on to finish, after `--migration-retry` for any Failed category; - - `--migration-abandon` what you are giving up on; - - point back at RavenDB, which is free only if ServiceControl never opened on SQL (the refusal says which); - - start over against a new, empty SQL database. +Before ServiceControl opens, the other exits remain pointing back at RavenDB or starting over against a new, empty SQL database. **Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl has opened, nothing can start without it. After, ServiceControl still opens, and the optional categories wait for RavenDB to come back, or can be abandoned. @@ -475,7 +474,6 @@ It says nothing about load on the source. | Setting | Default | What it does | | --- | --- | --- | -| `ServiceControl/Migration/Enabled` | `false` | Turns the migration on | | `ServiceControl/Migration/EventLogWindow` | `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. `0` turns it off | | `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | `ServiceControl/ErrorRetentionPeriod` | How far back the archived and resolved copy goes. `0` turns it off | | `ServiceControl/Migration/ThrottlePauseMilliseconds` | `100` | Pause between background batches | @@ -484,7 +482,7 @@ It says nothing about load on the source. - **The source needs no new settings.** It reads the instance's existing RavenDB settings. - **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. -- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. A start with migration still on after every category is Done or Abandoned logs a warning to turn it off. There is no migration custom check, because custom checks are being removed from ServiceControl. +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. Finished checkpoints deactivate copying automatically. There is no migration custom check, because custom checks are being removed from ServiceControl. - **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. - **Stopping a copy takes a restart.** @@ -542,7 +540,7 @@ The dry run counts the subscription and endpoint settings merges before anything - **Checkpoint:** the row in SQL recording one category's progress, saved in the same transaction as the rows it describes. - **Cursor:** the position in RavenDB the copy has reached in a category. - **State:** one of Copying, Done, Failed or Abandoned. -- **Copying:** not finished yet, including waiting to start, or stopped on an error that the next start will try again. +- **Copying:** not finished yet, including waiting to start, or interrupted until the next offline command or online copier invocation. - **Done:** every row is in SQL, or was left out for a harmless reason, and the counts balance. - **Failed:** something went wrong. Stays Failed until you run `--migration-retry` or `--migration-abandon`. - **Abandoned:** you gave up on what a category had not copied, or on one it depends on. Final. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index 1caf1d03c9..e95612915f 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -1,7 +1,7 @@ # Migration Engine: System Architecture Design > [!NOTE] -> This build does not carry the whole migration yet. This page describes it as it will be when it ships. +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. The offline command, checkpoint-based startup gate, and online orchestration are present, but production copying remains blocked by `MigrationIsReleasedCheck`. [The overview](ravendb-to-sql-migration-overview.md) covers behaviour and guarantees. [The instructions](ravendb-to-sql-migration-instructions.md) cover the operator procedure. This page maps both onto types, call order and assembly boundaries. @@ -13,9 +13,9 @@ ```mermaid flowchart LR subgraph host["ServiceControl host"] - req["RequiredCopyBeforeTheHostOpens
then MigrationStartup"] + req["MigrateCommand
then MigrationStartup"] opt["OptionalCategoryCopier"] - guard["EndOfMigrationGuard"] + guard["MigrationBeforeTheHostOpens"] cmd["Migration commands"] end subgraph neutral["ServiceControl.Persistence"] @@ -54,7 +54,7 @@ Dependencies point inward: each persister knows only its own store, the neutral | `ServiceControl.Persistence` | `MigrationEngine`, `MigrationBodyLoader`. Contracts: `IMigrationSource`, `IMigrationTarget`, `IMigrationCheckpointStore`, `IMigrationStartupCheck`, `IMigrationTargetReadiness`, `IMigrationSourceFactory`. Data: `MigrationCategory`, `MigrationCategoryKind`, `MigrationCategoryIds`, `MigrationCategoryRegistry`, `MigrationBatch`, `MigrationRow`, `MigrationBody`, `MigrationCheckpoint`, `MigrationWriteResult`, `MigrationSkipReason`, `MigrationSourceDescription`, `MigrationSourceFact`. Rules: `MigrationCategoryStateExtensions`, `MigrationSkipReasonExtensions`, `MigrationCheckpointRules`, `HaltThreshold`. Options: `MigrationEngineOptions`, `MigrationSettings`. `MigrationCheckpointConflictException`. Read-only views: `MigrationStatusView`, `MigrationVerification`, `MigrationDryRunArithmetic` | Referenced by both persisters and the host, references neither persister, so the engine is unit-tested against in-memory fakes in `ServiceControl.UnitTests/Migration` | | `ServiceControl.Persistence.RavenDB` | `RavenMigrationSource`, `RavenReadOnlySourceLifecycle`, `RavenDocumentStream`, `IMigrationCategoryReader`, `IWindowedMigrationCategoryReader`, one reader per category | Owns document id prefixes, sessions and the embedded server | | `ServiceControl.Persistence.EFCore` | `EFCoreMigrationTarget`, `EFCoreMigrationTargetReadiness`, `IMigrationCategoryWriter`, `IStoreKeyedCategoryWriter`, one writer per category, `PreparedBatch`, `IMigrationSqlDialect`, `MigrationInsert`, `StoreKeyedInsert`, `EFMigrationCheckpointStore`, `MigrationCheckpointExtensions`, `EFCoreMigrationRowAssessor`, and the three target readiness checks | Owns tables, column widths and provider parameter limits | -| `ServiceControl` (host) | `RequiredCopyBeforeTheHostOpens`, `RecordHostOpenedOnTarget`, `MigrationStartup` with nested `ClosedWindowProgress`, `MigrationStartupCheckRunner`, the four host checks, `OptionalCategoryCopier`, `RequiredStartSourceOutage`, `MigrationComponent`, `EndOfMigrationGuard`, `FinishedCopyBeforeAnIngestionNodeOpens`, `MigrationProgressReporter`, the six migration commands with `MigrationOperatorCommands` and `MigrationSourceDescriptionPrinter`, `PersistenceFactory.CreateMigrationSource` and `OpenMigrationSource` | Pair support is the one question neither persister can answer, so it lives here | +| `ServiceControl` (host) | `MigrateCommand`, `MigrationBeforeTheHostOpens`, `RecordHostOpenedOnTarget`, `MigrationStartup` with nested `ClosedWindowProgress`, `MigrationStartupCheckRunner`, the four host checks, `OptionalCategoryCopier`, `FinishedCopyBeforeAnIngestionNodeOpens`, `MigrationProgressReporter`, the migration commands with `MigrationOperatorCommands` and `MigrationSourceDescriptionPrinter`, `PersistenceFactory.CreateMigrationSource` and `OpenMigrationSource` | Pair support is the one question neither persister can answer, so it lives here | Supporting assemblies: @@ -65,17 +65,15 @@ The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadines ## What runs on every SQL instance -These run whether or not `Migration/Enabled` is set. Each one closes a path to data loss or to a migration ending unnoticed. - - **Registrations.** `BasePersistence.RegisterDataStores`, reached through each provider's `AddPersistence`, registers `IMigrationCheckpointStore` as `EFMigrationCheckpointStore`, `IMigrationTargetReadiness` as `EFCoreMigrationTargetReadiness` and `IMigrationTarget` as `EFCoreMigrationTarget`, all singletons. -- **`EndOfMigrationGuard`.** With `Migration/Enabled` off, `RunCommand` registers a hosted service in place of `RequiredCopyBeforeTheHostOpens` that runs `MigrationStartup.RunEndOfMigrationGuard` in `StartingAsync`, before any other hosted service and before Kestrel binds. It reads every checkpoint row and refuses if any category `IsOutstanding`. `IMigrationTargetReadiness.HasHostOpened` only selects the refusal text: whether going back to RavenDB is still free. With no checkpoint store registered (RavenDB) it returns immediately. -- **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import, flag on or off. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race it on the checkpoint version. -- **`RecordHostOpenedOnTarget` on workers.** Every host that passes the ingestion gate also registers `RecordHostOpenedOnTarget`, flag on or off, and stamps `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so the end-of-migration guard must never offer a free abort after a worker has written. +- **`MigrationBeforeTheHostOpens`.** `RunCommand` registers a lifecycle service that checks required checkpoints in `StartingAsync`, before any hosted service starts or Kestrel binds. It delegates to `FinishedCopyBeforeAnIngestionNodeOpens`, treating unknown ids as required. An unfinished required row refuses startup and names `--start-data-migration`. The service never copies data. With no checkpoint store registered (RavenDB), it returns immediately. +- **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race the offline command on the checkpoint version. +- **`RecordHostOpenedOnTarget` on every writer.** The main host, ingestion workers, and SQL imports register `RecordHostOpenedOnTarget` and stamp `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so rollback advice must never offer a free abort after a worker has written. The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row, see [open decisions](ravendb-to-sql-migration-overview.md#gaps-nothing-enforces). > [!IMPORTANT] -> Workers must start only after the main host has opened. +> Stop workers before running `--start-data-migration`. Restart them only after the command succeeds and the main host opens. A worker cannot detect a migration before its first checkpoint exists. ## The required copy @@ -87,18 +85,20 @@ Unresolved and retry-issued failed messages copy while the host is closed becaus Archived and resolved messages are safe to copy after the host opens. If one of them fails again, the live unresolved row should win, and an insert-only copy gives exactly that. Copying unresolved messages in the background would need a writer that merges by attempt time, and retry confirmations held until their row arrives. Both change live ingestion. +For developer smoke tests, Debug builds honor `SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED=true` in both `--start-data-migration` and normal startup. Release builds ignore this override. + ```mermaid sequenceDiagram - participant Run as RunCommand - participant Copy as RequiredCopyBeforeTheHostOpens + participant Run as Operator + participant Copy as MigrateCommand participant Start as MigrationStartup participant Checks as MigrationStartupCheckRunner participant Source as RavenMigrationSource participant Target as EFCoreMigrationTarget participant Engine as MigrationEngine - Run->>Run: registers Copy as a hosted service, then hostBuilder.Build() - Run->>Copy: StartingAsync, before any other hosted service starts + Run->>Copy: --start-data-migration, with ServiceControl stopped + Copy->>Copy: build persistence-only host; never StartAsync Copy->>Start: RunRequiredCopy(app.Services, settings) Start->>Checks: MigrationPairIsSupportedCheck Start->>Start: PersistenceFactory.CreateMigrationSource, not connected @@ -111,14 +111,16 @@ sequenceDiagram Engine-->>Start: one checkpoint per category Start->>Start: ReportWhatTheCopyLeftBehind Start->>Start: RefuseIfAnyCategoryDidNotComplete - Start-->>Copy: returns, or throws and the host never opens - Note over Run: RecordHostOpenedOnTarget stamps the target in StartedAsync + Start-->>Copy: returns, or throws and the command fails + Copy-->>Run: success: start ServiceControl to complete online migration + Run->>Run: start service or container + Note over Run: Startup checks required checkpoints; StartedAsync stamps the target ``` Each position in the sequence is deliberate: -1. **Inside host start.** `RunCommand` registers `RequiredCopyBeforeTheHostOpens` only when `Migration/Enabled` is true, before `hostBuilder.Build()`. `app.RunAsync` drives its `StartingAsync` ahead of every other hosted service, so the retention sweeper, heartbeat settings sync, throughput collectors and the API cannot run while a required category is unfinished. Running inside `RunAsync` also keeps the Windows Service Control Manager's 30-second start timeout satisfied. -2. **`MigrationPairIsSupportedCheck` first.** It needs no source, target or options. It resolves `PersistenceType` through `PersistenceManifestLibrary` and checks it against `PersistenceFactory.SqlPersistenceNames`; the source type is fixed at `PersistenceFactory.MigrationSourcePersistenceType`. RavenDB with the flag on is rejected here. +1. **Inside the offline command.** `MigrateCommand` builds a host containing persistence, logging, and the clock. It never calls `StartAsync` or `RunAsync`, so no hosted services run. It calls `MigrationStartup.RunRequiredCopy` directly and prints the instruction to start ServiceControl only after success. Ctrl+C cancels the command; committed batches remain resumable through another `--start-data-migration` invocation. The Windows service is not starting during this work, so its start timeout does not apply. +2. **`MigrationPairIsSupportedCheck` first.** It needs no source, target or options. It resolves `PersistenceType` through `PersistenceManifestLibrary` and checks it against `PersistenceFactory.SqlPersistenceNames`; the source type is fixed at `PersistenceFactory.MigrationSourcePersistenceType`. `--start-data-migration` against a RavenDB target is rejected here. 3. **`PersistenceFactory.CreateMigrationSource`** resolves the source persistence and hard-casts its configuration to `IMigrationSourceFactory`, implemented only by `RavenPersistenceConfiguration`. A non-implementing persister fails with `InvalidCastException`. No I/O happens yet. 4. **`MigrationStartup.RunChecksAndOpenTarget`**, the target half, in order: - `OptionalCategoryWindowsAreValidCheck` runs `MigrationEngineOptions.FromSettings`, refusing a window that is not a non-negative `TimeSpan` and naming the key. A zero window removes the category from `SelectedOptionalCategoryIds`. @@ -137,27 +139,27 @@ Each position in the sequence is deliberate: > [!IMPORTANT] > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexCreation`. -7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the copy is skipped and the host opens. `OptionalCategoryCopier` then stays idle for that start. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. +7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the command skips the copy and succeeds. The operator can then start ServiceControl; its online copier handles any later source outage independently. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. 8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. -9. **`MigrationStartup.RunRequiredCategories`** upserts `NotStarted` rows, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. +9. **Checkpoint initialization.** The command seeds `NotStarted` rows for required and selected optional categories before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. 10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. 11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. 12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row as harmless. 13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. -14. **`RecordHostOpenedOnTarget`** is registered beside `RequiredCopyBeforeTheHostOpens` with the flag on. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. +14. **`RecordHostOpenedOnTarget`** is registered on normal startup, never on the offline command. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. ## The background copy -`OptionalCategoryCopier` is a `BackgroundService` registered by `MigrationComponent` only with the flag on and only on the main instance; `--error-ingestion-only` passes its own component list. +`OptionalCategoryCopier` is a `BackgroundService` registered by `RunCommand` only on the main instance. It waits for `ApplicationStarted`, then reads checkpoints. With no unfinished optional rows, it exits without creating a source or opening a target. -- Every 20 seconds it runs the optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. -- It opens its own source through `PersistenceFactory.OpenMigrationSource`, only when a category is still copying. On failure it calls `RecordSourceOutage` and idles for the rest of the start. It also idles when the required start already recorded an outage, which it learns from the in-process `RequiredStartSourceOutage` singleton (`Recorded`, `Error`) that `RunRequiredCopy` sets where it calls `RecordSourceOutage`. The checkpoint rows cannot carry this signal, because an optional row's `LastError` can survive from an earlier start. +- It runs the selected, supported optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. The offline command has already seeded the rows that identify the online work. +- It creates and opens its own source only when a supported category is still copying. On failure it calls `RecordSourceOutage` and exits for this start. It does not depend on in-process state from `--start-data-migration`, which ran in another process. - An exception leaves the category `InProgress` with `LastError`, untouched until the next start, so a transient failure never forces an abandon. Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. > [!IMPORTANT] -> Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. +> Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. The online startup path must validate unchanged windows too, because `--start-data-migration` no longer runs during normal startup. `Migration/ThrottlePauseMilliseconds` inserts a delay between optional-category batches, skipping the first batch of each category. Required categories are never throttled. @@ -230,6 +232,7 @@ Every migration command except the source report builds a host with `AddPersiste | Command | Class | Opens RavenDB | Reads and writes | | --- | --- | --- | --- | +| `--start-data-migration` | `MigrateCommand` | Yes | Runs the required-copy checks and engine, seeds selected optional checkpoints, and exits. On success it instructs the operator to start ServiceControl; it never starts the service itself | | `--migration-source-report` | `MigrationSourceReportCommand` | Yes | `IMigrationSource.Describe` and `Inventory` through `MigrationSourceDescriptionPrinter`; the copy never calls either. Each `MigrationSourceFact` carries its originating setting key. `Inventory` covers every collection, including uncopied ones | | `--migration-dry-run` | `MigrationDryRunCommand`, `MigrationDryRunArithmetic` | Yes | `MigrationPairIsSupportedCheck`, then `MigrationStartup.RunChecksAndOpen` (both halves), so it refuses wherever a real start would. Reads `Count`, `Read`, `ReadBody`, `ReadBodyVolume`, `Inventory`, `Describe`, `IMigrationTarget.Assess` (via `EFCoreMigrationRowAssessor`) and `BatchSizeFor`. Writes no row; the body probe is the only write. Exit code 0 even when the report shows problems; a refused check throws | | `--migration-status` | `MigrationStatusCommand`, `MigrationStatusView` | No | `IMigrationCheckpointStore.ReadAll` and `MigrationEngineOptions.FromSettings`. Read-only, safe while ServiceControl runs | @@ -292,6 +295,6 @@ There is no migration custom check: custom checks are being removed from Service - `MigrationProgressReporter` raises `MigrationStarted`, `MigrationCategoryHalted` (for both Failed states), `MigrationFinished` (carrying `CategoriesAbandoned`) and `MigrationSourceUnreachable`, mapped into the ServicePulse activity feed by their `EventLogMappingDefinition` classes. It logs and swallows publish failures but propagates cancellation. All four are raised from `OptionalCategoryCopier`, so they cover the background copy only, and a Failed required category produces no activity-feed entry; its record is the startup refusal and the log. - `MigrationSourceUnreachable` is raised once per start when `OptionalCategoryCopier` idles on a source outage, its own failed open or one the required start recorded, carrying the error and the optional categories left waiting. -- On its first pass, `OptionalCategoryCopier` logs a warning to set `Migration/Enabled` to `false` and restart when `MigrationCheckpointRules.FinishedButStillEnabled` holds: no registry category `IsOutstanding`. `--migration-status` prints the same line. `IsOutstanding` rather than `IsFinished`, so an optional category turned off with a `0` window does not hold the warning back. +- Completed checkpoints stop the online copier automatically. No enable setting or final configuration edit is needed. - `ClosedWindowProgress` logs running required categories every 30 seconds while the host is closed. - `--migration-status` reports state, progress, last error and available commands per category at any time, and whether migration can be turned off. diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs index 917700bea5..d7dde350e4 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs @@ -98,7 +98,8 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured() "MetricsReporterHostedService", "HealthCheckPublisherHostedService", // inert, no IHealthCheckPublisher is registered "ExternalIntegrationRequestsDataStore", // its drain is inert here, nothing calls Subscribe - "FinishedCopyBeforeAnIngestionNodeOpens" // keeps this node out of a database a copy has not finished filling + "FinishedCopyBeforeAnIngestionNodeOpens", // keeps this node out of a database a copy has not finished filling + "RecordHostOpenedOnTarget" // idempotent upsert of a marker only read for presence, safe on every node ]; [Test] diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index 21c76e78a6..9e8035cf09 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -82,7 +82,6 @@ public async Task SetUp() SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", Source.PrimaryDatabase); SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", Source.ThroughputDatabase); SetSourceVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); #if DEBUG // Cleared because a developer who set it to smoke test the migration would let every test past the release check, which only the test host's own AllowUnreleasedMigration registration may do. SetSourceVariable(RunCommand.AllowUnreleasedMigrationVariable, null); @@ -287,6 +286,13 @@ protected static Action AllowingAnUnreleasedMigration(Act customize?.Invoke(builder); }; + protected async Task MigrateThenRunHost(Action customize = null) + { + await StopHost(); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(customize)); + await RunHostUntilTheApiAnswers(customize); + } + protected async Task RunHostUntilTheApiAnswers(Action customize = null) { await StopHost(); @@ -320,9 +326,8 @@ public Task StartAsync(CancellationToken cancellationToken = default) public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; } - protected Settings SettingsWithMigrationEnabled(bool copyEventLog = false, Action customize = null) + protected Settings MigrationSettings(bool copyEventLog = false, Action customize = null) { - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", copyEventLog ? null : "0"); SetSourceVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "0"); customize?.Invoke(Settings); @@ -334,9 +339,9 @@ protected async Task RunRequiredCopy( bool leaveOptionalIncomplete = false, CancellationToken cancellationToken = default) { - SettingsWithMigrationEnabled(copyEventLog: leaveOptionalIncomplete); + MigrationSettings(copyEventLog: leaveOptionalIncomplete); - await RunHostUntilTheApiAnswers(builder => customize?.Invoke(builder.Services)); + await MigrateThenRunHost(builder => customize?.Invoke(builder.Services)); var copyable = MigrationStartup.CopyableCategoryIds(await SourceSupportedCategoryIds(), Target.SupportedCategoryIds); @@ -352,11 +357,12 @@ await WaitUntil( if (leaveOptionalIncomplete) { - // The guarded services read IMigrationState.AnyCategoryIncomplete, which any selected category that has not finished holds true. Halted is one of those states. - await targetCheckpointStore.Upsert( - new MigrationCheckpoint("EventLog", MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, - "Halted by the test fixture so the guarded services see an unfinished category."), - cancellationToken); + var eventLog = await targetCheckpointStore.Read(MigrationCategoryIds.EventLog, cancellationToken); + await targetCheckpointStore.Upsert(eventLog with + { + State = MigrationCategoryState.Halted, + LastError = "Halted by the test fixture." + }, cancellationToken); } } @@ -446,7 +452,7 @@ protected async Task CopyCategory(string categoryId, Cance var category = MigrationCategoryRegistry.Find(categoryId) ?? throw new ArgumentException($"'{categoryId}' is not a migration category.", nameof(categoryId)); var before = await targetCheckpointStore.Read(categoryId, cancellationToken); var target = new RecordingMigrationTarget(Target); - var options = new MigrationEngineOptions(TimeSpan.Zero, MigrationSettings.DefaultHaltThresholdPercent, MigrationSettings.DefaultHaltThresholdMinimum, []) { BodyRetryBackoff = TimeSpan.Zero }; + var options = new MigrationEngineOptions(TimeSpan.Zero, Persistence.DataMigration.MigrationSettings.DefaultHaltThresholdPercent, Persistence.DataMigration.MigrationSettings.DefaultHaltThresholdMinimum, []) { BodyRetryBackoff = TimeSpan.Zero }; await using var source = await OpenSource(cancellationToken); var engine = new MigrationEngine(source, target, targetCheckpointStore, TimeProvider.System, options, NullLogger.Instance); @@ -474,7 +480,11 @@ protected async Task SeedCheckpoint(string categoryId, MigrationCategoryState st await using var scope = targetServices.CreateAsyncScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); - await dbContext.UpsertCheckpoint(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + var existing = await targetCheckpointStore.Read(categoryId); + await dbContext.UpsertCheckpoint(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null) + { + Version = existing?.Version ?? 0 + }); } protected async Task DeleteCheckpoint(string categoryId) diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs index a55deaba69..c5fc9dfa59 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs @@ -17,19 +17,9 @@ namespace ServiceControl.Migration.AcceptanceTests; [NonParallelizable] class When_a_copy_is_killed_mid_category : MigrationAcceptanceTest { - [Test] - public void A_migrating_host_stopped_while_starting_exits_cleanly() - { - using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - - Assert.DoesNotThrowAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); - } - [Test] public void A_host_not_migrating_stopped_while_starting_still_fails() { - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); Assert.CatchAsync(async () => @@ -56,20 +46,20 @@ public async Task It_resumes_rather_than_starting_over() using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); var killed = Assert.ThrowsAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); Assert.That(killed, Is.Not.Null); Assert.That(await TargetEndpointSettingsCount(), Is.EqualTo(2), "the first batch and its cursor should have committed together"); var refusedAgain = Assert.ThrowsAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); Assert.That(refusedAgain.Message, Is.EqualTo(killed.Message), "a start re-ran a Failed category the operator had not put back"); // The row a kill part way through the category leaves: still copying, with the first batch's cursor and counts. var halted = await ReadCheckpoint("EndpointSettings"); await QueryTarget(dbContext => dbContext.UpsertCheckpoint(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null })); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var checkpoint = await ReadCheckpoint("EndpointSettings"); @@ -92,10 +82,10 @@ public async Task Without_its_checkpoint_the_second_run_re_reads_what_the_first_ using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); Assert.ThrowsAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); await DeleteCheckpoint("EndpointSettings"); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var checkpoint = await ReadCheckpoint("EndpointSettings"); @@ -112,7 +102,7 @@ public async Task A_worker_started_after_a_copy_stopped_between_categories_names using var stopping = new CancellationTokenSource(TimeSpan.FromMinutes(3)); Assert.CatchAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.StopAfterTheFirstCategorySettles(stopping)), stopping.Token)); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.StopAfterTheFirstCategorySettles(stopping)), stopping.Token)); using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); var worker = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs index 86cd583c29..ee4948759f 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs @@ -152,7 +152,7 @@ public async Task A_halted_required_category_stops_the_host_rather_than_opening_ using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); var exception = Assert.ThrowsAsync(async () => - await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.HaltTheEndpointSettingsCategory()), cancellation.Token)); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.HaltTheEndpointSettingsCategory()), cancellation.Token)); Assert.Multiple(() => { @@ -168,13 +168,13 @@ public async Task A_source_that_will_not_open_once_the_required_copy_settled_sti { await SeedSourceKnownEndpoints("Sales"); await SeedSourceEndpointSettings(("Sales", true)); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var missing = $"{Source.PrimaryDatabase}-does-not-exist"; SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var endpointSettings = await GetEndpointSettings(); var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); @@ -214,7 +214,7 @@ public async Task A_failed_optional_row_does_not_stop_the_host_opening() { await SeedSourceKnownEndpoints("Sales"); await SeedSourceEndpointSettings(("Sales", true)); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.CompleteWithErrors); SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", $"{Source.PrimaryDatabase}-does-not-exist"); @@ -240,11 +240,11 @@ async Task RefusedStartup(bool allowUnreleasedMigration = true) using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); var refusal = Assert.ThrowsAsync(async () => - await RunCommand.Run(Settings, allowUnreleasedMigration ? AllowingAnUnreleasedMigration() : null, cancellation.Token)); + await MigrateCommand.Run(Settings, allowUnreleasedMigration ? AllowingAnUnreleasedMigration() : null, cancellation.Token)); using (Assert.EnterMultipleScope()) { - Assert.That(refusal.Message, Does.Contain("this start has copied nothing")); + Assert.That(refusal.Message, Does.Contain("this run has copied nothing")); Assert.That(await TargetEndpointSettingsCount(), Is.Zero, "a check that fires after rows have moved is worse than no check"); Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing, so the abort is still free"); } diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs index 07d17190d4..539c9b12f7 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs @@ -14,13 +14,13 @@ public async Task Changing_the_windows_between_restarts_deletes_nothing() await SeedSourceEndpointSettings(("Sales", true)); SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); - await RunHostUntilTheApiAnswers(); - Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "a category no run has copied has no row, and no column anywhere says it was selected"); + await MigrateThenRunHost(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "a disabled optional category has no checkpoint"); var firstRun = await ReadCheckpoint("EndpointSettings"); SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); await RunHostUntilTheApiAnswers(); - Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "the required copy runs required categories only, so turning an optional one on starts nothing here"); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "normal startup does not select new migration categories; it uses the rows the offline command seeded"); SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); await RunHostUntilTheApiAnswers(); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs index 29d5b8b432..916b3422c5 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs @@ -29,7 +29,7 @@ public async Task They_are_readable_through_the_product_read_api() // A clock that never moves keeps HeartbeatEndpointSettingsSyncHostedService's twenty second // delay from elapsing and rewriting these rows while the assertions read them. - await RunHostUntilTheApiAnswers(builder => builder.Services.AddSingleton(new FakeTimeProvider())); + await MigrateThenRunHost(builder => builder.Services.AddSingleton(new FakeTimeProvider())); var settings = await GetEndpointSettings(); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs index a57aa10426..b850bb4523 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs @@ -9,36 +9,38 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Hosting.Commands; [TestFixture] -// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set -// process-global environment variables. One fixture added without it makes the whole suite intermittent. [NonParallelizable] class When_the_api_is_called_during_the_required_copy : MigrationAcceptanceTest { [Test] - public async Task The_api_does_not_answer_until_the_required_copy_has_finished() + public async Task The_api_opens_only_after_the_offline_command_finishes_and_the_operator_starts_ServiceControl() { await SeedSourceKnownEndpoints("Sales", "Billing", "Shipping"); await SeedSourceEndpointSettings(("Sales", true), ("Billing", true), ("Shipping", true)); var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); + var copy = MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task)), cancellation.Token); + + try + { + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf()); + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Throws.Exception.With.Message.Contains("ServiceControl will not start")); + } + finally + { + releaseTheCopy.TrySetResult(); + await copy; + } - await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); - Assert.That(releaseTheCopy.Task.IsCompleted, Is.False, "the copy must still be parked for this assertion to mean anything"); Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf(), - "Kestrel had already bound its socket while the required copy was still running"); - - releaseTheCopy.SetResult(); - var settings = await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + "the offline command must not start ServiceControl itself"); + await RunHostUntilTheApiAnswers(); + var settings = await GetEndpointSettings(); Assert.That(settings.Select(row => row.Name), Is.SupersetOf(new[] { "Sales", "Billing", "Shipping" })); - - await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); - await cancellation.CancelAsync(); - await host; } } diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs index 7620e21618..8a8523bc72 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs @@ -32,49 +32,45 @@ public async Task The_row_does_not_exist_while_the_required_copy_is_still_runnin var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); - - await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); - Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the abort is still free at this moment, and this row is what says it is not"); + var copy = MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task)), cancellation.Token); - releaseTheCopy.SetResult(); - await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); - await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); + try + { + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the abort is still free while the offline copy runs"); + } + finally + { + releaseTheCopy.TrySetResult(); + await copy; + } + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the offline command never opens the service"); + await RunHostUntilTheApiAnswers(); Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); - - await cancellation.CancelAsync(); - await host; } [Test] - public async Task A_start_with_migration_off_does_not_stamp_the_target() + public async Task A_start_detects_and_stamps_a_migration_without_an_enable_setting() { await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); - await RunHostUntilTheApiAnswers(); - - Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "only a start with the migration on may write the marker"); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); } [Test] - public async Task A_start_with_migration_off_does_not_read_the_checkpoint_table() + public async Task A_start_requires_the_checkpoint_schema_to_check_for_an_unfinished_migration() { await DropTheCheckpointTable(); - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); - - Assert.That(async () => await RunHostUntilTheApiAnswers(), Throws.Nothing, "every hosted service, StartedAsync included, ran without touching the missing table"); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), Throws.Exception); } [Test] public async Task An_ingestion_only_host_is_refused_while_a_required_category_is_failed() { Settings.MaximumConcurrencyLevel = 1; - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); await SeedRequiredCategoriesComplete(except: MigrationCategoryIds.MessageRedirects); await SeedCheckpoint(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.CompleteWithErrors); @@ -99,7 +95,6 @@ public async Task An_ingestion_only_host_is_refused_while_a_required_category_is public async Task An_ingestion_only_host_starts_while_an_optional_category_is_still_copying_or_failed() { Settings.MaximumConcurrencyLevel = 1; - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); await SeedRequiredCategoriesComplete(); await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); @@ -122,7 +117,6 @@ public async Task An_ingestion_only_host_starts_while_an_optional_category_is_st public async Task An_ingestion_only_host_is_refused_over_a_category_this_build_does_not_know() { Settings.MaximumConcurrencyLevel = 1; - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); await SeedRequiredCategoriesComplete(); await SeedCheckpoint(CategoryFromANewerBuild, MigrationCategoryState.InProgress); @@ -144,7 +138,6 @@ public async Task An_ingestion_only_host_is_refused_over_a_category_this_build_d [Test] public async Task Importing_failed_errors_is_refused_while_a_copy_is_unfinished() { - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress); using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); @@ -158,7 +151,6 @@ public async Task Importing_failed_errors_is_refused_while_a_copy_is_unfinished( [Test] public async Task Importing_failed_errors_starts_while_an_optional_category_is_still_copying_or_failed() { - SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); await SeedRequiredCategoriesComplete(); await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); @@ -180,7 +172,7 @@ public async Task The_row_does_not_exist_after_a_startup_check_refused() using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Assert.That(async () => await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), Throws.Exception.With.Message.Contains("NoSuchWindow")); Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing"); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs index f86c9ab931..135c7a213d 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs @@ -232,7 +232,7 @@ public async Task Answers_for_a_copied_message_on_the_URL_it_had_on_RavenDB() var failure = new IngestedFailure(); await SeedFailure(failure); - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var response = await HttpClient.GetAsync($"api/errors/last/{failure.UniqueMessageIdString}"); @@ -261,7 +261,7 @@ public async Task The_required_copy_completes_the_category_before_the_host_opens await SeedFailure(failure); } - await RunHostUntilTheApiAnswers(); + await MigrateThenRunHost(); var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); @@ -292,7 +292,7 @@ public async Task Records_each_skip_under_the_reason_that_caused_it() using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - var refused = Assert.ThrowsAsync(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + var refused = Assert.ThrowsAsync(async () => await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs index b545a8bea6..05db8791e5 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs @@ -68,7 +68,7 @@ public Task Run(CancellationToken cancellationToken = default) => if (holdingRows.Count > 0) { - throw new Exception($"The database already holds ServiceControl data in {string.Join(", ", holdingRows)}, so ServiceControl has already run on it, and a migration copies only into an empty database. Create a new database, run ServiceControl with --setup against it, and do not start ServiceControl on it before migrating; then start again with migration on."); + throw new Exception($"The database already holds ServiceControl data in {string.Join(", ", holdingRows)}, so ServiceControl has already run on it, and a migration copies only into an empty database. Create a new database, run ServiceControl with --setup against it, and run --start-data-migration before starting ServiceControl on it."); } }, cancellationToken); } diff --git a/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs deleted file mode 100644 index 7bf02d735b..0000000000 --- a/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace ServiceControl.Persistence.DataMigration; - -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -/// -/// Answers from the saved checkpoints, read once before the host opens rather than queried live. -/// -public sealed class CheckpointMigrationState(IMigrationCheckpointStore? checkpointStore = null) : IMigrationState -{ - volatile bool anyCategoryIncomplete; - IReadOnlyCollection selectedCategoryIds = []; - - public bool AnyCategoryIncomplete => anyCategoryIncomplete; - - /// - /// Reads every checkpoint and fixes the answer for the life of the host. - /// - /// The required categories this instance copies, which are the only ones that hold back the services that delete or overwrite copied data. They are passed in because the store holds a row only for a category some run has already been given. - public async Task Seed(IReadOnlyCollection selectedIds, CancellationToken cancellationToken = default) - { - // Only a persister that can be migrated into keeps checkpoints, so no store means no migration has run. - if (checkpointStore is null) - { - return; - } - - selectedCategoryIds = selectedIds; - - Recompute(await checkpointStore.ReadAll(cancellationToken)); - } - - /// - /// Works the answer out again from the checkpoints given, without reading the store. - /// - /// Every checkpoint the store holds. A selected category with no checkpoint among them has not started, which is not finished. - public void Recompute(IReadOnlyList checkpoints) => - anyCategoryIncomplete = selectedCategoryIds.Any(id => - checkpoints.FirstOrDefault(checkpoint => checkpoint.CategoryId == id)?.State.IsFinished() != true); -} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs index bd08c1df27..66af36395d 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs @@ -9,8 +9,8 @@ namespace ServiceControl.Persistence.DataMigration; public interface IMigrationStartupCheck { /// - /// What the check is called in the refusal the host prints, so it is phrased to finish the sentence - /// "Migration startup check '...' failed". + /// What the check is called in the refusal, so it is phrased to finish the sentence + /// "Migration check '...' failed". /// string Name { get; } diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs deleted file mode 100644 index 212e8694a5..0000000000 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace ServiceControl.Persistence.DataMigration; - -/// -/// Whether this instance is still part way through copying its old database in. -/// -public interface IMigrationState -{ - /// - /// True while any required category is unfinished. Services that delete or overwrite copied data stand down while it - /// is true. Optional categories never make it true, because they copy beside those services. - /// - bool AnyCategoryIncomplete { get; } -} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs index 6102b22058..a5306e95e5 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -19,13 +19,7 @@ public static class MigrationSettings /// retention period, and zero turns the category off. ///
public const string ArchivedAndResolvedFailedMessagesWindowKey = "Migration/ArchivedAndResolvedFailedMessagesWindow"; - /// - /// Turns the migration on: the required copy runs before the host opens. - /// - public const string EnabledKey = "Migration/Enabled"; - public const int DefaultThrottlePauseMilliseconds = 100; public const int DefaultHaltThresholdPercent = 5; public const int DefaultHaltThresholdMinimum = 100; - public const bool DefaultEnabled = false; } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index 9cabb918e0..b7303ff3a8 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,7 +63,6 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", - "MigrationEnabled": false, "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs index d3c8851ed0..9363cadd41 100644 --- a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs +++ b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs @@ -11,6 +11,14 @@ class MigrationSourceReportArgumentTests public void The_flag_selects_the_report_command() => Assert.That(new HostArguments(["--migration-source-report"]).Command, Is.EqualTo(typeof(MigrationSourceReportCommand))); + [Test] + public void The_migrate_flag_selects_the_offline_command() => + Assert.That(new HostArguments(["--start-data-migration"]).Command, Is.EqualTo(typeof(MigrateCommand))); + + [Test] + public void An_ingestion_worker_cannot_run_the_offline_command() => + Assert.That(new HostArguments(["--start-data-migration", "--error-ingestion-only"]).Help, Is.True); + [Test] public void No_flag_still_selects_the_run_command() => Assert.That(new HostArguments([]).Command, Is.EqualTo(typeof(RunCommand))); diff --git a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs deleted file mode 100644 index 9947ca3ca3..0000000000 --- a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs +++ /dev/null @@ -1,111 +0,0 @@ -#nullable enable -namespace ServiceControl.UnitTests.Migration; - -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using ServiceControl.Persistence.DataMigration; -using ServiceControl.UnitTests.Migration.Fakes; - -[TestFixture] -class CheckpointMigrationStateTests -{ - [Test] - public async Task An_instance_that_has_never_migrated_has_no_store_and_nothing_incomplete() - { - var state = new CheckpointMigrationState(); - - await state.Seed(["EndpointSettings"], CancellationToken.None); - - Assert.That(state.AnyCategoryIncomplete, Is.False); - } - - [Test] - public async Task Selected_categories_that_all_finished_leave_the_guard_off() - { - var store = new InMemoryMigrationCheckpointStore(); - await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); - await store.Upsert(Checkpoint("EndpointSettings", MigrationCategoryState.Complete)); - - var state = new CheckpointMigrationState(store); - await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); - - Assert.That(state.AnyCategoryIncomplete, Is.False); - } - - [TestCase(MigrationCategoryState.InProgress)] - [TestCase(MigrationCategoryState.NotStarted)] - [TestCase(MigrationCategoryState.Halted)] - [TestCase(MigrationCategoryState.Blocked)] - [TestCase(MigrationCategoryState.CompleteWithErrors)] - public async Task One_selected_category_short_of_finished_holds_the_guard_on(MigrationCategoryState unfinished) - { - var store = new InMemoryMigrationCheckpointStore(); - await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); - await store.Upsert(Checkpoint("EndpointSettings", unfinished)); - - var state = new CheckpointMigrationState(store); - await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); - - Assert.That(state.AnyCategoryIncomplete, Is.True); - } - - [Test] - public async Task A_selected_category_with_no_row_at_all_has_not_started() - { - var store = new InMemoryMigrationCheckpointStore(); - await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); - - var state = new CheckpointMigrationState(store); - await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); - - Assert.That(state.AnyCategoryIncomplete, Is.True); - } - - [Test] - public async Task An_unselected_category_that_never_ran_does_not_hold_the_guard_on_forever() - { - var store = new InMemoryMigrationCheckpointStore(); - await store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, null, 3, 0, 3, null, null, null, null, null)); - await store.Upsert(new MigrationCheckpoint("EventLog", MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null)); - - var state = new CheckpointMigrationState(store); - await state.Seed(["EndpointSettings"], CancellationToken.None); - - Assert.That(state.AnyCategoryIncomplete, Is.False); - } - - [Test] - public void Exactly_complete_and_abandoned_are_finished() => - Assert.That( - Enum.GetValues().Where(state => state.IsFinished()), - Is.EquivalentTo(new[] { MigrationCategoryState.Complete, MigrationCategoryState.Abandoned })); - - [Test] - public void Exactly_halted_and_complete_with_errors_are_failed() => - Assert.That( - Enum.GetValues().Where(state => state.IsFailed()), - Is.EquivalentTo(new[] { MigrationCategoryState.Halted, MigrationCategoryState.CompleteWithErrors })); - - [Test] - public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => - Assert.That( - Enum.GetValues().Where(reason => reason.IsBenign()), - Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), - "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); - - [Test] - public void Unknown_stays_the_last_skip_reason() => - Assert.That(Enum.GetValues().Last(), Is.EqualTo(MigrationSkipReason.Unknown)); - - [Test] - public void Exactly_the_reasons_no_retry_can_fix_are_permanent() => - Assert.That( - Enum.GetValues().Where(reason => reason.IsPermanent()), - Is.EquivalentTo(new[] { MigrationSkipReason.RequiredValueMissing, MigrationSkipReason.UnparseableUniqueMessageId, MigrationSkipReason.NoProcessingAttempts, MigrationSkipReason.ValueTooLong, MigrationSkipReason.MalformedValue, MigrationSkipReason.BodyMissing })); - - static MigrationCheckpoint Checkpoint(string categoryId, MigrationCategoryState state) => - new(categoryId, state, null, 0, 0, null, null, null, null, null, null); -} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs new file mode 100644 index 0000000000..9e86ba2723 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationBeforeTheHostOpensTests +{ + [Test] + public async Task A_persister_without_checkpoints_starts_without_a_migration_source() + { + using var services = new ServiceCollection().BuildServiceProvider(); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + [Test] + public async Task An_empty_checkpoint_table_starts_without_a_migration_source() + { + using var services = Services(new InMemoryMigrationCheckpointStore()); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Blocked)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task An_unfinished_required_copy_refuses_startup_without_resuming_it(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + var before = await store.Upsert(Checkpoint(MigrationCategoryIds.KnownEndpoints, state)); + using var services = Services(store); + + var exception = Assert.ThrowsAsync(() => new MigrationBeforeTheHostOpens(services).StartingAsync()); + Assert.That(exception.Message, Does.Contain("ServiceControl will not start")); + Assert.That(await store.Read(before.CategoryId), Is.EqualTo(before), "startup must never run the offline copy"); + } + + [Test] + public async Task An_unrecognized_unfinished_category_keeps_the_host_closed() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("NewRequiredCategory", MigrationCategoryState.NotStarted)); + using var services = Services(store); + var exception = Assert.ThrowsAsync(() => new MigrationBeforeTheHostOpens(services).StartingAsync()); + Assert.That(exception.Message, Does.Contain("--start-data-migration")); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task Optional_work_does_not_hold_the_host_closed(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint(MigrationCategoryIds.EventLog, state)); + using var services = Services(store); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + static ServiceProvider Services(IMigrationCheckpointStore store) => + new ServiceCollection().AddSingleton(store).BuildServiceProvider(); + + static MigrationCheckpoint Checkpoint(string id, MigrationCategoryState state) => + new(id, state, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs index 4d82cac0f8..6c67f2fb81 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs @@ -1,6 +1,5 @@ namespace ServiceControl.UnitTests.Migration; -using System; using System.Linq; using System.Reflection; using NUnit.Framework; @@ -10,30 +9,8 @@ namespace ServiceControl.UnitTests.Migration; [TestFixture] [NonParallelizable] -class MigrationEnabledSettingsTests +class MigrationOptionsSettingsTests { - static Settings NewSettings() => - new(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); - - [TearDown] - public void ClearEnvironmentVariables() - { - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", null); - } - - [Test] - public void Migration_defaults_to_off() - { - Assert.That(NewSettings().MigrationEnabled, Is.False); - } - - [Test] - public void MigrationEnabled_is_read_from_the_environment() - { - Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); - - Assert.That(NewSettings().MigrationEnabled, Is.True); - } [Test] public void Only_MigrationEngineOptions_parses_the_engine_settings_keys() diff --git a/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs index 638efaaf1b..260215cdf7 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs @@ -12,12 +12,12 @@ namespace ServiceControl.UnitTests.Migration; class MigrationIsReleasedCheckTests { [Test] - public void A_build_without_the_whole_migration_refuses_MigrationEnabled() + public void A_build_without_the_whole_migration_refuses_the_offline_command() { var exception = Assert.ThrowsAsync(() => new MigrationIsReleasedCheck().Run()); - Assert.That(exception.Message, Does.Contain("does not yet carry the whole migration").And.Contain(MigrationSettings.EnabledKey), - "the customer has to learn that nothing will be copied, and which setting to turn back off"); + Assert.That(exception.Message, Does.Contain("does not yet carry the whole migration").And.Contain("--start-data-migration"), + "the customer has to learn that the offline command is unavailable"); } // A RavenDB target fails the pair check, so this only passes while the release check runs before it. diff --git a/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs b/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs new file mode 100644 index 0000000000..b58c28d598 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class OptionalCategoryCopierTests +{ + [Test] + public async Task A_persister_without_checkpoints_needs_no_source_or_target() + { + using var services = new ServiceCollection().BuildServiceProvider(); + await Run(services); + } + + [Test] + public async Task A_database_that_never_migrated_needs_no_source_or_target() + { + using var services = new ServiceCollection() + .AddSingleton(new InMemoryMigrationCheckpointStore()) + .BuildServiceProvider(); + await Run(services); + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task A_finished_or_failed_optional_category_does_not_open_the_source(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + var checkpoint = await store.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, state, null, 0, 0, null, null, null, null, null, null)); + using var services = new ServiceCollection().AddSingleton(store).BuildServiceProvider(); + await Run(services); + Assert.That(await store.Read(checkpoint.CategoryId), Is.EqualTo(checkpoint)); + } + + static async Task Run(ServiceProvider services) + { + var logger = new ErrorLogger(); + using var copier = new OptionalCategoryCopier(services, null, new StartedLifetime(), logger); + await copier.StartAsync(CancellationToken.None); + await copier.ExecuteTask.WaitAsync(TimeSpan.FromSeconds(10)); + await copier.StopAsync(CancellationToken.None); + Assert.That(logger.Errors, Is.Empty, "an idle copier must not attempt to resolve or open a source or target"); + } + + sealed class ErrorLogger : ILogger + { + public List Errors { get; } = []; + public bool IsEnabled(LogLevel logLevel) => true; + public IDisposable BeginScope(TState state) where TState : notnull => null; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (logLevel >= LogLevel.Error) + { + Errors.Add(formatter(state, exception)); + } + } + } + + sealed class StartedLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => new(true); + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() { } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs index 4f5ff54295..17f483f0ba 100644 --- a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs +++ b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs @@ -262,7 +262,7 @@ public void A_checkpoint_saved_by_another_instance_says_which_instance_to_stop() { Assert.That(exception!.Message, Does.Contain("a second ServiceControl pointed at the same SQLServer database"), "the store's own message names a version, which tells nobody what is actually wrong"); - Assert.That(exception.Message, Does.Contain($"Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on"), + Assert.That(exception.Message, Does.Contain("Stop the other instance, then run --start-data-migration again"), "this is the only refusal in the subsystem where doing nothing makes it worse"); Assert.That(exception.Message, Does.Contain(conflict.Message).And.Contain("Nothing has opened on SQLServer yet")); Assert.That(exception.InnerException, Is.SameAs(conflict)); diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index f1f6eee178..83f0bb1afc 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -17,7 +17,6 @@ using global::ServiceControl.Operations.Metrics; using global::ServiceControl.Recoverability.Retrying.Metrics; using global::ServiceControl.Persistence; - using global::ServiceControl.Persistence.DataMigration; using global::ServiceControl.Transports; using Licensing; using Microsoft.AspNetCore.HttpLogging; @@ -109,10 +108,6 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S services.AddStartupChecks(); services.AddPersistence(settings); - // The checkpoint store is optional: EF Core registers one, RavenDB does not. - services.TryAddSingleton(provider => - new CheckpointMigrationState(provider.GetService())); - services.AddMetrics(settings.PrintMetrics); hostBuilder.AddTelemetry(settings); services.AddServiceControlHealthChecks(); diff --git a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs index 1890ba5242..bc804646db 100644 --- a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs @@ -53,10 +53,7 @@ internal static WebApplication BuildHost(Settings settings, Action(), "this error ingestion only host")); - if (settings.MigrationEnabled) - { - hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); - } + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); customize?.Invoke(hostBuilder); diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index c8970bdbac..ad520f7e4b 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -72,6 +72,7 @@ internal static IHost BuildHost(Settings settings) new FinishedCopyBeforeAnIngestionNodeOpens( provider.GetRequiredService(), "--import-failed-errors")); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); } return hostBuilder.Build(); diff --git a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs new file mode 100644 index 0000000000..feedfd181d --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Hosting.Commands; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Migration; +using ServiceControl.Persistence; + +class MigrateCommand : AbstractCommand +{ + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + Console.CancelKeyPress += cancel; + var migrationToken = cancellation.Token; + + try + { + Console.Out.WriteLine("Stop ServiceControl and all ingestion workers before running the offline migration."); + await Run(settings, cancellationToken: migrationToken); + Console.Out.WriteLine("The offline migration completed. Start the ServiceControl service or container to complete the online migration."); + } + catch (OperationCanceledException) when (migrationToken.IsCancellationRequested) + { + Console.Out.WriteLine("The offline migration stopped. Run --start-data-migration again to resume from the last committed batch."); + Environment.ExitCode = 1; + } + finally + { + Console.CancelKeyPress -= cancel; + } + } + + internal static async Task Run(Settings settings, Action customize = null, CancellationToken cancellationToken = default) + { + var builder = WebApplication.CreateBuilder(); + // Persistence also registers runtime-only services whose dependencies this command does not need. + builder.Host.UseDefaultServiceProvider(options => options.ValidateOnBuild = false); + builder.Logging.ConfigureLogging(settings.LoggingSettings.LogLevel); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddPersistence(settings); + customize?.Invoke(builder); + +#if DEBUG + RunCommand.AllowUnreleasedMigrationWhenAsked(builder.Services, Environment.GetEnvironmentVariable(RunCommand.AllowUnreleasedMigrationVariable)); +#endif + + // Never start this host: only the copier may write during the offline phase. + await using var app = builder.Build(); + await MigrationStartup.RunRequiredCopy(app.Services, settings, cancellationToken); + } +} diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs index 45cf63cc1c..0b695e5b94 100644 --- a/src/ServiceControl/Hosting/Commands/RunCommand.cs +++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs @@ -45,8 +45,7 @@ public override Task Execute(HostArguments args, Settings settings, Cancellation Run(settings, customize: null, cancellationToken); /// - /// Builds and runs the full instance. When the migration is turned on, the required copy is added as the - /// first thing the host starts, so the instance opens only once the copy has finished. + /// Builds and runs the full instance after checking that any offline migration has finished. /// internal static async Task Run(Settings settings, Action customize, CancellationToken cancellationToken = default) { @@ -70,11 +69,9 @@ internal static async Task Run(Settings settings, Action AllowUnreleasedMigrationWhenAsked(hostBuilder.Services, Environment.GetEnvironmentVariable(AllowUnreleasedMigrationVariable)); #endif - if (settings.MigrationEnabled) - { - hostBuilder.Services.AddHostedService(provider => new RequiredCopyBeforeTheHostOpens(provider, settings)); - hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); - } + hostBuilder.Services.AddHostedService(provider => new MigrationBeforeTheHostOpens(provider)); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + hostBuilder.Services.AddHostedService(); // A start that refuses would otherwise leave everything the host built undisposed. await using var app = hostBuilder.Build(); @@ -90,23 +87,7 @@ internal static async Task Run(Settings settings, Action app.Urls.Clear(); app.Urls.Add(settings.RootUrl); - // Read before RunAsync, which disposes the host before the filter below runs, so reading it there would - // throw and the catch would never fire. - var lifetime = app.Lifetime; - - // Starting the host is what marks the target as opened, through the RecordHostOpenedOnTarget hosted - // service registered above, so nothing here does it. - try - { - await app.RunAsync(cancellationToken); - } - // Stopping the service during the required copy cancels it inside host start, and that is a stop - // rather than a failure: every committed batch is durable and the next start resumes from the cursor. -#pragma warning disable PS0020 // The host cancels on its own lifetime token, not the caller's, so that is the one to filter on - catch (OperationCanceledException) when (settings.MigrationEnabled && lifetime.ApplicationStopping.IsCancellationRequested) -#pragma warning restore PS0020 - { - } + await app.RunAsync(cancellationToken); } } } diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index d616180539..633fe38180 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -23,6 +23,16 @@ This mode runs no setup, so combining it with --setup or --setup-and-run is refu Message bodies must be stored somewhere every host can read, so this mode should not be combined with file system body storage unless the path is a shared mount. +MIGRATION + + ServiceControl.exe --start-data-migration + +Copies required data from RavenDB into the configured SQL Server or PostgreSQL database. +Stop ServiceControl and all ingestion workers first. The command never starts the service. +After a successful offline copy, start ServiceControl to copy optional data in the background. +Run --start-data-migration again after an interruption to resume from saved progress. The target database records the migration state. +This build still blocks copying until the complete migration is released. + SERVICE INSTALL AND UNINSTALL AND CONFIGURATION OPTIONS As of Service Control 1.7 the command line uninstall and install switches have been removed. diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index bd6400df32..7e4571f447 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -62,6 +62,15 @@ public HostArguments(string[] args) } }; + var migrationOptions = new OptionSet + { + { + "start-data-migration", + "Copy required migration data with ServiceControl stopped", + s => Command = typeof(MigrateCommand) + } + }; + var migrationSourceReportOptions = new OptionSet { { @@ -97,6 +106,17 @@ public HostArguments(string[] args) return; } + migrationOptions.Parse(args); + + if (Command == typeof(MigrateCommand)) + { + if (ErrorIngestionOnly) + { + throw new Exception("--start-data-migration cannot run with --error-ingestion-only. Stop every instance before running the offline migration."); + } + return; + } + if (ErrorIngestionOnly) { Command = typeof(ErrorIngestionOnlyCommand); diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index 745f2bbde9..c601c07b57 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,7 +16,6 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; - using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -186,7 +185,6 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } - public bool MigrationEnabled => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.EnabledKey, MigrationSettings.DefaultEnabled); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs index 6156b0bfc5..16831e3add 100644 --- a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs +++ b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs @@ -26,7 +26,7 @@ public Task Run(CancellationToken cancellationToken = default) if (allowUnreleasedMigration is null) { throw new Exception( - $"This build of ServiceControl does not yet carry the whole migration from RavenDB, so it will not copy anything. Set {MigrationSettings.EnabledKey} back to false and start ServiceControl again."); + $"This build of ServiceControl does not yet carry the whole migration from RavenDB, so it will not copy anything. --start-data-migration is not available in this build. Keep the instance on RavenDB."); } return Task.CompletedTask; diff --git a/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs index e3ffed392a..fbdfef0d18 100644 --- a/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs +++ b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs @@ -23,7 +23,7 @@ public Task Run(CancellationToken cancellationToken = default) if (!PersistenceFactory.SqlPersistenceNames.Contains(targetName, StringComparer.OrdinalIgnoreCase)) { throw new Exception( - $"Migrating from '{PersistenceFactory.MigrationSourcePersistenceType}' to '{settings.PersistenceType}' is not supported. The only supported migration is from RavenDB to SQL Server or PostgreSQL, so set {Settings.SettingsRootNamespace}/PersistenceType to {string.Join(" or ", PersistenceFactory.SqlPersistenceNames)} before setting {MigrationSettings.EnabledKey}."); + $"Migrating from '{PersistenceFactory.MigrationSourcePersistenceType}' to '{settings.PersistenceType}' is not supported. The only supported migration is from RavenDB to SQL Server or PostgreSQL, so set {Settings.SettingsRootNamespace}/PersistenceType to {string.Join(" or ", PersistenceFactory.SqlPersistenceNames)} before running --start-data-migration."); } return Task.CompletedTask; diff --git a/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs index 00f5903bd4..8372211f84 100644 --- a/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs +++ b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs @@ -20,7 +20,7 @@ public Task Run(CancellationToken cancellationToken = default) if (retryHistoryDepth <= 0) { throw new Exception( - $"ServiceControl/RetryHistoryDepth is {retryHistoryDepth}, and at that depth the SQL persisters delete every row of HistoricRetryOperations when a retry completes. The RetryOperations category copies that table, so the first completed retry after cutover would silently discard the migrated retry history. Set ServiceControl/RetryHistoryDepth to a positive number, or clear it to use the default of 10, before setting {MigrationSettings.EnabledKey}."); + $"ServiceControl/RetryHistoryDepth is {retryHistoryDepth}, and at that depth the SQL persisters delete every row of HistoricRetryOperations when a retry completes. The RetryOperations category copies that table, so the first completed retry after cutover would silently discard the migrated retry history. Set ServiceControl/RetryHistoryDepth to a positive number, or clear it to use the default of 10, before running --start-data-migration."); } return Task.CompletedTask; diff --git a/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs index 6433658d4b..90f4f6da59 100644 --- a/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs +++ b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs @@ -8,8 +8,8 @@ namespace ServiceControl.Migration; using ServiceControl.Persistence.DataMigration; /// -/// Refuses to start a host that writes failed messages into the database, an error ingestion only host or -/// --import-failed-errors, until every required category is Done or Abandoned. An optional category, which copies +/// Refuses to start a host that writes failed messages into the database, including the main instance, +/// an error ingestion only host, or --import-failed-errors, until every required category is Done or Abandoned. An optional category, which copies /// in the background once the main host has opened, never holds it back, but a category under an id this build /// does not know does. A refusal throws, which fails the start and stops the host. Start such a host only after the /// main host has opened, because a database with no checkpoint row yet reads as one no migration has touched. @@ -20,9 +20,7 @@ sealed class FinishedCopyBeforeAnIngestionNodeOpens( IMigrationCheckpointStore checkpointStore, string hostDescription) : IHostedLifecycleService { - // It runs with the migration off too, because a node someone forgot to flag would ingest into a part-copied - // database and turn abandoning it from a clean rollback into permanent loss. A database that never migrated has - // nothing unfinished, so its nodes start as before. + // Every writer checks persisted progress, because ingesting into a partial copy ends the free abort. public async Task StartingAsync(CancellationToken cancellationToken = default) { var unfinished = (await checkpointStore.ReadAll(cancellationToken)) @@ -39,9 +37,7 @@ public async Task StartingAsync(CancellationToken cancellationToken = default) ? MigrationStartup.FailedDetail(checkpoint) : $"{checkpoint.CategoryId} is Copying ({checkpoint.State})")); - var advice = unfinished.Any(checkpoint => !checkpoint.State.IsFailed()) - ? "Let the instance running the copy finish it and start this host again." - : "Start this host again once they are settled."; + const string advice = "With ServiceControl and all ingestion workers stopped, resolve any Failed categories and run --start-data-migration again before starting this host."; throw new Exception($"A copy into this database has not finished, so {hostDescription} will not start and nothing has been lost. {detail}. {advice}"); } diff --git a/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs b/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs new file mode 100644 index 0000000000..b259b2ddcd --- /dev/null +++ b/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +sealed class MigrationBeforeTheHostOpens(IServiceProvider services) : IHostedLifecycleService +{ + public async Task StartingAsync(CancellationToken cancellationToken = default) + { + var checkpoints = services.GetService(); + if (checkpoints is null) + { + return; + } + + await new FinishedCopyBeforeAnIngestionNodeOpens(checkpoints, "ServiceControl").StartingAsync(cancellationToken); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs index f906c8d5b9..f36d874995 100644 --- a/src/ServiceControl/Migration/MigrationStartup.cs +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -24,13 +24,9 @@ internal static IReadOnlyCollection CopyableCategoryIds(IReadOnlyCollect [.. sourceSupports.Intersect(targetSupports, StringComparer.Ordinal)]; /// - /// Runs the startup checks, copies every required category this build can copy that is not already settled, - /// and then seeds the the host reads. Once every required category this build - /// copies is settled and no row under an unknown id is unfinished, a source that will not open is logged and - /// recorded on the optional categories' rows instead of refused, nothing is copied, and the host still opens. Throws when a check refuses or a category does not - /// finish, with a message telling the operator what to do and how to go back; the caller must let that stop - /// the host. Call it after the host is built and before it starts, because the copy has to finish before - /// anything else opens on the target. + /// Runs the checks and required copy without starting ServiceControl. A completed required copy can succeed + /// without its source; optional checkpoints retain the outage for the online phase. + /// Throws if a check refuses or a required category does not finish. The caller must not start the host. /// public static async Task RunRequiredCopy(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) { @@ -60,11 +56,6 @@ await MigrationStartupCheckRunner.Run( options, loggerFactory.CreateLogger()); - // Optional categories copy beside the running services, so only the required ones hold those services back. - var holdingBack = engine.SelectCategories(MigrationCategoryKind.Required) - .Select(category => category.Id) - .ToArray(); - var toCopy = engine.SelectCategories(MigrationCategoryKind.Required) .Where(category => copyable.Contains(category.Id)) .ToArray(); @@ -89,15 +80,14 @@ await MigrationStartupCheckRunner.Run( // Safe to open without the source, because every required category is settled and only the background copy of the optional ones reads it. catch (Exception exception) when (requiredCopySettled) { - // The runner's wrapper says ServiceControl will not start, which is false here and would land on the rows. + // The runner's wrapper describes a refused run, which this is not, and it would land on the rows. var cause = exception.InnerException ?? exception; logger.LogError(cause, - "The {SourcePersistenceType} migration source could not be opened, so ServiceControl opens without it: every required category this build copies is Done or Abandoned and no row under an id it does not know is unfinished, so only the background copy of the optional categories reads the source. The optional categories still copying stay as they are until a start that can open the source, and their checkpoints carry this error.", + "The {SourcePersistenceType} migration source could not be opened. Every required category this build copies is Done or Abandoned and no unknown required row is unfinished, so the offline command can complete. Outstanding optional checkpoints carry the error; start ServiceControl to continue the online migration.", PersistenceFactory.MigrationSourcePersistenceType); await RecordSourceOutage(checkpointStore, engine.SelectCategories(MigrationCategoryKind.Optional), cause, cancellationToken); - await SeedMigrationState(services, holdingBack, cancellationToken); return; } catch (Exception exception) @@ -108,9 +98,18 @@ await MigrationStartupCheckRunner.Run( await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); logger.LogInformation( - "Migration mode: copying {CopyCount} required categories before ServiceControl opens ({DeferredCount} not yet implemented: {Deferred})", + "Offline migration: copying {CopyCount} required categories with ServiceControl stopped ({DeferredCount} not yet implemented: {Deferred})", toCopy.Length, deferred.Length, string.Join(", ", deferred)); + // Seed both phases before copying so startup can detect a copy interrupted between categories. + foreach (var category in toCopy.Concat(engine.SelectCategories(MigrationCategoryKind.Optional))) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null), cancellationToken); + } + } + // Captured before the copy so the report can tell a category this run finished from one an earlier run did. var runStartedAt = timeProvider.GetUtcNow().UtcDateTime; @@ -121,16 +120,6 @@ await MigrationStartupCheckRunner.Run( ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt); RefuseIfAnyCategoryDidNotComplete(toCopy, finished, unfinishedOutsideTheCopy, settings); - - await SeedMigrationState(services, holdingBack, cancellationToken); - } - - static async Task SeedMigrationState(IServiceProvider services, IReadOnlyCollection holdingBack, CancellationToken cancellationToken) - { - if (services.GetRequiredService() is CheckpointMigrationState state) - { - await state.Seed(holdingBack, cancellationToken); - } } /// @@ -298,7 +287,7 @@ internal static async Task> CopyOrExplainWhyI catch (MigrationCheckpointConflictException exception) { throw new Exception( - $"The required copy stopped because another writer saved a migration checkpoint for this instance, which is a second ServiceControl pointed at the same {settings.PersistenceType} database. ServiceControl will not start. {exception.Message} Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on; the copy resumes from its last committed batch. " + + $"The required copy stopped because another writer saved a migration checkpoint for this instance, which is a second ServiceControl pointed at the same {settings.PersistenceType} database. The offline migration stopped. {exception.Message} Stop the other instance, then run --start-data-migration again; the copy resumes from its last committed batch. " + RollbackAdvice(settings), exception); } } @@ -329,7 +318,7 @@ internal static void RefuseIfAnyCategoryDidNotComplete( } throw new Exception( - $"The required copy did not finish, so ServiceControl will not start and nothing has been lost. {string.Join(". ", outstanding)}. " + + $"The required copy did not finish, so ServiceControl cannot start yet, and nothing has been lost. {string.Join(". ", outstanding)}. " + RollbackAdvice(settings)); } @@ -463,7 +452,7 @@ internal static string StallExplanation(string stalledCategoryId) => $"Halted: {stalledCategoryId} committed nothing for {ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes, so it was stopped. The limit is not configurable: check that the source and the target are both responding rather than looking for a setting to change."; static string RollbackAdvice(Settings settings) => - $"Nothing has opened on {settings.PersistenceType} yet, so setting {MigrationSettings.EnabledKey}=false and pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType} discards the partial copy and returns the instance to {PersistenceFactory.MigrationSourcePersistenceType} with no loss."; + $"Nothing has opened on {settings.PersistenceType} yet, so pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType} discards the partial copy and returns the instance to {PersistenceFactory.MigrationSourcePersistenceType} with no loss."; static string NoWayOutWithoutTheSource(Settings settings) => $"If the {PersistenceFactory.MigrationSourcePersistenceType} source is already gone, a required category that is Failed or has started copying can be given up with --migration-abandon , with ServiceControl stopped. " + diff --git a/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs index 4ccc0626a5..68f93d6104 100644 --- a/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs +++ b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs @@ -13,7 +13,7 @@ static class MigrationStartupCheckRunner { /// /// Runs each check in turn, and wraps whatever a failing one throws in a message naming the check and - /// saying that this start has copied nothing, because every check runs before this start's copy. + /// saying that this run has copied nothing, because every check runs before this run's copy. /// /// A check failed. The check's own message is kept, and its exception is the inner one. public static async Task Run(IReadOnlyList checks, CancellationToken cancellationToken = default) @@ -32,7 +32,7 @@ public static async Task Run(IReadOnlyList checks, Cance catch (Exception exception) { throw new Exception( - $"Migration startup check '{check.Name}' failed, so ServiceControl will not start, and this start has copied nothing. {exception.Message}", exception); + $"Migration check '{check.Name}' failed, so this run has copied nothing. {exception.Message}", exception); } } } diff --git a/src/ServiceControl/Migration/OptionalCategoryCopier.cs b/src/ServiceControl/Migration/OptionalCategoryCopier.cs new file mode 100644 index 0000000000..f7e43480b0 --- /dev/null +++ b/src/ServiceControl/Migration/OptionalCategoryCopier.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +sealed class OptionalCategoryCopier(IServiceProvider services, Settings settings, IHostApplicationLifetime lifetime, ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + try + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = lifetime.ApplicationStarted.Register(() => started.TrySetResult()); + await started.Task.WaitAsync(cancellationToken); + await Copy(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + logger.LogError(exception, "The online migration stopped. ServiceControl remains available. Check migration progress before restarting."); + } + } + + async Task Copy(CancellationToken cancellationToken) + { + var store = services.GetService(); + if (store is null) + { + return; + } + + var checkpoints = await store.ReadAll(cancellationToken); + var pending = checkpoints + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind == MigrationCategoryKind.Optional) + .Where(checkpoint => !checkpoint.State.IsFinished() && !checkpoint.State.IsFailed()) + .Select(checkpoint => checkpoint.CategoryId) + .ToHashSet(StringComparer.Ordinal); + if (pending.Count == 0) + { + return; + } + + await new MigrationIsReleasedCheck(services.GetService()).Run(cancellationToken); + var target = services.GetRequiredService(); + await using var source = PersistenceFactory.CreateMigrationSource(settings); + var copyable = MigrationStartup.CopyableCategoryIds(source.SupportedCategoryIds, target.SupportedCategoryIds); + var options = MigrationEngineOptions.FromSettings(Settings.SettingsRootNamespace, settings.EventsRetentionPeriod, settings.ErrorRetentionPeriod); + var engine = new MigrationEngine(source, target, store, services.GetRequiredService(), options, + services.GetRequiredService>()); + var categories = engine.SelectCategories(MigrationCategoryKind.Optional) + .Where(category => pending.Contains(category.Id) && copyable.Contains(category.Id)) + .ToArray(); + if (categories.Length == 0) + { + return; + } + + await target.Open(cancellationToken); + try + { + await source.Open(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + await MigrationStartup.RecordSourceOutage(store, categories, exception, cancellationToken); + logger.LogError(exception, "The online migration cannot open its source. ServiceControl remains available. Restart ServiceControl to try again, or abandon the outstanding optional categories."); + return; + } + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + await engine.RunCategories(categories, cancellationToken); + } +} diff --git a/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs index 676eb714fa..62c3c4053e 100644 --- a/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs +++ b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs @@ -8,21 +8,20 @@ namespace ServiceControl.Migration; using ServiceControl.Persistence.DataMigration; /// -/// Calls when a host started with the migration on opens +/// Calls when a host opens /// on a database a migration has already written to. /// sealed class RecordHostOpenedOnTarget(IServiceProvider services) : IHostedLifecycleService { // StartedAsync, not StartAsync: the web server binds its port during StartAsync, and a start that dies there - // served nothing. The stores are resolved here rather than injected, because a RavenDB host registers neither - // and is refused before it gets this far. + // served nothing. RavenDB registers no checkpoint store and needs no migration marker. public async Task StartedAsync(CancellationToken cancellationToken = default) { - var checkpointStore = services.GetRequiredService(); + var checkpointStore = services.GetService(); // Only a database a migration has written to gets the stamp, so the marker answers whether a host // has opened since the copy began rather than whether one ever ran on this database. - if ((await checkpointStore.ReadAll(cancellationToken)).Count > 0) + if (checkpointStore is not null && (await checkpointStore.ReadAll(cancellationToken)).Count > 0) { await services.GetRequiredService().RecordHostOpened(cancellationToken); } diff --git a/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs b/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs deleted file mode 100644 index 9ca9388246..0000000000 --- a/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace ServiceControl.Migration; - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; -using ServiceBus.Management.Infrastructure.Settings; - -/// -/// Runs the required copy as the host starts, before any hosted service of its own does. -/// A refusal throws, which fails the start and stops the host, as the copy requires. -/// -sealed class RequiredCopyBeforeTheHostOpens(IServiceProvider services, Settings settings) : IHostedLifecycleService -{ - // Inside the host rather than before it, because a Windows service reports itself started only once the host - // starts, and the Service Control Manager kills a process that has said nothing for 30 seconds. Every - // StartingAsync runs before any hosted service, so nothing has bound a port or begun ingesting. - public Task StartingAsync(CancellationToken cancellationToken = default) => - MigrationStartup.RunRequiredCopy(services, settings, cancellationToken); - - public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; - - public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; - - public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; - - public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; - - public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; -} From 1068147367b0569494a42d6cffbf07f532939577 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 10:48:35 +0800 Subject: [PATCH 20/28] Fix typo in documentation: change 'DisableAutoIndexCreation' to 'DisableAutoIndexes' --- docs/migration/ravendb-to-sql-migration-system-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index e95612915f..f35991fdc7 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -138,7 +138,7 @@ Each position in the sequence is deliberate: - waits for both databases, with a 5-minute budget per database on an embedded server. The budget is only checked when RavenDB raises its own timeout, so the wait can exceed it. > [!IMPORTANT] - > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexCreation`. + > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexes`. 7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the command skips the copy and succeeds. The operator can then start ServiceControl; its online copier handles any later source outage independently. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. 8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. 9. **Checkpoint initialization.** The command seeds `NotStarted` rows for required and selected optional categories before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. From 071a36d6e295b0f7c9126911381aa2ba08956311 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 12:07:23 +0800 Subject: [PATCH 21/28] Add CreatedDate property to licensing endpoint mappings and tests --- .../LicensingEndpointsMigrationTests.cs | 5 ++++- .../MigrationAcceptanceTest.cs | 3 ++- .../DataMigration/LicensingEndpointRowMapper.cs | 1 + .../Migration/LicensingMigrationMapperCoverageTests.cs | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs index 83e0f6a57a..614f04792d 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Migration.AcceptanceTests; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -20,7 +21,8 @@ await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) SanitizedName = "Sales_Orders", UserIndicator = "Sales team", Scope = "Sales", - EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()] + EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()], + CreatedDate = new DateTime(2026, 3, 4, 5, 6, 7, DateTimeKind.Utc) }); await CopyCategory("LicensingEndpoints"); @@ -34,6 +36,7 @@ await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) Assert.That(fromTarget, Is.Not.Null); Assert.That(fromTarget!.UserIndicator, Is.EqualTo("Sales team")); Assert.That(fromTarget.Scope, Is.EqualTo("Sales")); + Assert.That(fromTarget.CreatedDate, Is.EqualTo(new DateTime(2026, 3, 4, 5, 6, 7, DateTimeKind.Utc)), "the decommissioned endpoint sweep removes an endpoint with no throughput once it is a day old, so a lost date makes a new endpoint eligible at once"); Assert.That(row.SanitizedName, Is.EqualTo("Sales_Orders")); Assert.That(row.NormalizedSanitizedName, Is.EqualTo("sales_orders"), "GetEndpointThroughputByQueueName matches on this column, so an unnormalised value hides a migrated endpoint's throughput from the licensing report"); Assert.That(fromTarget.EndpointIndicators, Is.EqualTo(new[] { EndpointIndicator.PlatformEndpoint.ToString() }), "the indicators column holds what EF itself writes: JSON on SQL Server, a text[] on PostgreSQL"); diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index 9e8035cf09..bcc073ccfc 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -178,7 +178,8 @@ protected async Task SeedThroughputSource(Endpoint endpoint) endpoint.SanitizedName, EndpointIndicators = endpoint.EndpointIndicators ?? [], UserIndicator = endpoint.UserIndicator ?? string.Empty, - Scope = endpoint.Scope ?? string.Empty + Scope = endpoint.Scope ?? string.Empty, + endpoint.CreatedDate }; using var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs index b5772d85fa..3d868b5dc5 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs @@ -23,6 +23,7 @@ public static LicensingEndpointEntity Map(Endpoint endpoint) NormalizedSanitizedName = LicensingDataStore.Normalize(sanitizedName), UserIndicator = endpoint.UserIndicator ?? string.Empty, Scope = endpoint.Scope, + CreatedDate = endpoint.CreatedDate ?? DateTime.MinValue, EndpointIndicators = endpoint.EndpointIndicators?.ToList() ?? [] }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs index f7c2d37322..b91e3c4778 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs @@ -20,7 +20,8 @@ public async Task The_endpoint_mapper_sets_every_mapped_property() SanitizedName = "Sales.Orders", EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()], UserIndicator = "Sales team", - Scope = "Sales" + Scope = "Sales", + CreatedDate = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc) }; MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(await Model(), LicensingEndpointRowMapper.Map(endpoint)); From 49f63e642461bf7ba799f1b4ececa6d9478ec0ab Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 12:39:01 +0800 Subject: [PATCH 22/28] Remove unused ServiceControl.Persistence namespace from ImportFailedErrorsCommand --- src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index ad520f7e4b..6dddcd2645 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -18,7 +18,6 @@ using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; using ServiceControl.Migration; - using ServiceControl.Persistence; using ServiceControl.Persistence.DataMigration; class ImportFailedErrorsCommand : AbstractCommand From a338a6da9b413bfcfc3080b82f12458455a090ec Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 16 Nov 2026 21:15:20 +0800 Subject: [PATCH 23/28] Copy redirects, subscriptions, retry history and settings to SQL - Copy message redirects, subscriptions, retry operations, notification settings and the trial end date - Report rows merged by the SQL key instead of dropping them silently - Delete store-keyed rows when a category is reset for a retry --- .../MessageRedirectsMigrationTests.cs | 73 ++++++ .../MigrationAcceptanceTest.cs | 96 +++++++ .../NotificationSettingsMigrationTests.cs | 67 +++++ .../RetryOperationsMigrationTests.cs | 129 +++++++++ .../RetryResetMigrationTests.cs | 94 +++++++ .../SubscriptionsMigrationTests.cs | 62 +++++ .../TestMigrationTargets.cs | 4 + .../TrialEndDateMigrationTests.cs | 30 +++ ...e_heartbeat_sync_runs_after_a_migration.cs | 48 ++++ .../When_the_offline_migration_logs.cs | 38 +++ .../DataMigration/EFCoreMigrationTarget.cs | 37 ++- .../FailedMessageDocumentTranslator.cs | 2 +- .../IStoreKeyedCategoryWriter.cs | 17 ++ .../DataMigration/MessageRedirectRowMapper.cs | 17 ++ .../DataMigration/RetryOperationRowMapper.cs | 35 +++ .../DataMigration/SubscriptionRowMapper.cs | 19 ++ .../Writers/EndpointSettingsWriter.cs | 55 ++-- .../DataMigration/Writers/KeyMerges.cs | 63 +++++ .../Writers/LicensedEndpointDetailsWriter.cs | 19 +- .../Writers/LicensingReportMasksWriter.cs | 19 +- .../Writers/MessageRedirectsWriter.cs | 55 ++++ .../Writers/NotificationSettingsWriter.cs | 13 + .../Writers/RetryOperationsWriter.cs | 124 +++++++++ .../DataMigration/Writers/SettingRowWriter.cs | 32 +++ .../Writers/SubscriptionsWriter.cs | 55 ++++ .../Writers/TrialEndDateWriter.cs | 15 ++ .../DataMigration/RavenMigrationSource.cs | 7 +- .../Readers/LicensedEndpointDetailsReader.cs | 33 +-- .../Readers/LicensingReportMasksReader.cs | 33 +-- .../Readers/MessageRedirectsReader.cs | 22 ++ .../Readers/NotificationSettingsReader.cs | 20 ++ .../Readers/RetryOperationsReader.cs | 24 ++ .../Readers/SubscriptionsReader.cs | 27 ++ .../Readers/TrialEndDateReader.cs | 17 ++ .../DataMigration/SingleDocumentReader.cs | 54 ++++ .../Editing/NotificationsDataStore.cs | 2 +- .../Recoverability/RetryHistoryDataStore.cs | 2 +- .../TrialLicenseDataProvider.cs | 2 +- .../LicensingSettingsSourceTests.cs | 4 - .../MessageRedirectsSourceTests.cs | 31 +++ .../RavenMigrationSourceTests.cs | 33 ++- .../DataMigration/SubscriptionsSourceTests.cs | 59 +++++ .../EndpointSettingsKeyCollationTests.cs | 33 ++- ...nsedEndpointDetailsMigrationTargetTests.cs | 29 --- ...icensingReportMasksMigrationTargetTests.cs | 29 --- .../MessageRedirectRowMapperTests.cs | 28 ++ .../MessageRedirectsMigrationTargetTests.cs | 75 ++++++ .../Migration/MigrationTargetCoverageTests.cs | 50 +++- .../EFCore/Migration/PreparedSkips.cs | 11 + .../Migration/RetryOperationRowMapperTests.cs | 47 ++++ .../RetryOperationsMigrationTargetTests.cs | 244 ++++++++++++++++++ .../Migration/SettingsMigrationTargetTests.cs | 49 ++++ .../Migration/SubscriptionRowMapperTests.cs | 23 ++ .../SubscriptionsMigrationTargetTests.cs | 154 +++++++++++ .../DataMigration/IMigrationTarget.cs | 31 +-- .../DataMigration/RetryOperationRow.cs | 8 + .../DataMigration/SubscriptionRow.cs | 8 + .../Fakes/InMemoryMigrationTarget.cs | 7 + .../MigrationCategoryStateExtensionsTests.cs | 18 ++ .../MigrationEngineSkipReasonTests.cs | 4 + .../Hosting/Commands/MigrateCommand.cs | 2 + 61 files changed, 2218 insertions(+), 220 deletions(-) create mode 100644 src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs create mode 100644 src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs create mode 100644 src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs diff --git a/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs new file mode 100644 index 0000000000..207bfd706f --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +[TestFixture] +[NonParallelizable] +class MessageRedirectsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_three_redirects_from_one_document_into_three_rows() + { + var seeded = new[] + { + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = new DateTime(2026, 3, 1, 12, 0, 1, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = new DateTime(2026, 3, 1, 12, 0, 2, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old3", ToPhysicalAddress = "new3", LastModified = new DateTime(2026, 3, 1, 12, 0, 3, DateTimeKind.Utc) } + }; + await SeedSourceMessageRedirects(seeded); + + var result = await CopyCategory("MessageRedirects"); + + Assert.That(result.Copied, Is.EqualTo(3)); + + var redirects = await MessageRedirectsDataStore.GetRedirects(); + Assert.That(redirects.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified)), + Is.EquivalentTo(seeded.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified))), + "each redirect keeps its own destination and the time it was last changed"); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!), Is.EqualTo(3), "--migration-verify reads this count"); + } + + [TestCase("old1", "old1")] + [TestCase("Sales@HOST", "sales@host")] + public async Task A_repeated_address_is_copied_or_already_present_as_the_database_compares_it(string firstAddress, string secondAddress) + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = firstAddress, ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = secondAddress, ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + var stored = await MessageRedirectsDataStore.GetRedirects(); + + Assert.That(result.Skipped, Is.Zero, "a repeat loses no row the target lacks, so it is never a skip"); + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(result.Copied, Is.EqualTo(stored.Count), + "Copied is the rows the database stored: one where the key treats the addresses as the same, two where it does not"); + Assert.That(stored.Select(redirect => (redirect.FromPhysicalAddress, redirect.ToPhysicalAddress)), Does.Contain((firstAddress, "new1")), + "the first redirect in document order is copied on every database"); + } + + [Test] + public async Task A_redirect_with_no_destination_is_skipped_as_a_required_value_missing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = null!, LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = null!, ToPhysicalAddress = "new3", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "old1", "messageredirects#2" }), "a redirect with no source address is named by its place in the document"); + + var checkpoint = await ReadCheckpoint("MessageRedirects"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index bcc073ccfc..f93b77210c 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -15,8 +15,11 @@ namespace ServiceControl.Migration.AcceptanceTests; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; using NServiceBus.Extensibility; using NServiceBus.Transport; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using NUnit.Framework; using Particular.LicensingComponent.BrokerThroughput; using Particular.LicensingComponent.Contracts; @@ -29,6 +32,8 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Infrastructure.WebApi; using ServiceControl.MessageFailures; using ServiceControl.Migration.Checks; +using ServiceControl.Monitoring; +using ServiceControl.Notifications; using ServiceControl.Operations; using ServiceControl.Operations.BodyStorage; using ServiceControl.Persistence; @@ -36,10 +41,13 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Tests; using ServiceControl.Persistence.UnitOfWork; +using ServiceControl.Recoverability; using ServiceControl.Transports.BrokerThroughput; using TestHelper; // Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. @@ -224,6 +232,43 @@ protected Task SeedThroughputReportMasks(IReadOnlyList maskedStrings) => protected Task SeedThroughputLicensedEndpointDetails(LicensedEndpointDetails details) => SeedSource(Source.ThroughputDatabase, ("LicensedEndpointDetails", details)); + protected Task SeedSourceMessageRedirects(params MessageRedirect[] redirects) => + SeedSource(("messageredirects", new + { + Redirects = redirects.Select(redirect => new { redirect.FromPhysicalAddress, redirect.ToPhysicalAddress, LastModifiedTicks = redirect.LastModified.Ticks }).ToList() + })); + + // Stores the type as the string MessageTypeConverter writes, because that is what the persister reads. + protected Task SeedSourceSubscriptions(params SubscriptionRow[] subscriptions) => + SeedSource(("Subscriptions/All", new + { + All = subscriptions + .GroupBy(subscription => new MessageType(subscription.MessageType, subscription.Version).ToString()) + .ToDictionary(group => group.Key, group => new + { + Id = group.Key, + MessageType = group.Key, + Subscribers = group.Select(subscription => new { subscription.TransportAddress, subscription.Endpoint }).ToList() + }) + })); + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + protected async Task SeedSourceNotificationSettings(EmailNotifications emailNotifications, string collection = "NotificationsSettingsDocuments") + { + var document = new { Email = emailNotifications }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, "NotificationsSettings/All"); + session.Advanced.GetMetadataFor(document)["@collection"] = collection; + await session.SaveChangesAsync(); + } + + protected Task SeedSourceTrialEndDate(DateOnly endDate) => + SeedSource(("metadata/trialinformation", new TrialMetadata { TrialEndDate = endDate })); + + protected Task SeedSourceRetryHistory(RetryHistory history) => + SeedSource(("RetryOperations/History", history)); + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => SeedSourceWithBodies((document, body, contentType)); @@ -442,11 +487,60 @@ protected static async Task> CollectBatches(IMigrationSourc return batches; } + SyncPassCountingLogger syncPasses; + + protected void CountSyncPasses(IServiceCollection services) + { + syncPasses = null; + services.AddSingleton>(provider => + syncPasses = new SyncPassCountingLogger(provider.GetRequiredService().CreateLogger())); + } + + protected async Task WaitForSyncPass(FakeTimeProvider time) + { + await WaitUntil(() => Task.FromResult(syncPasses?.Started >= 1), "the heartbeat settings sync started a pass"); + + time.Advance(TimeSpan.FromHours(6) + TimeSpan.FromSeconds(1)); + + await WaitUntil(() => Task.FromResult(syncPasses.Started >= 2), "the heartbeat settings sync finished its pass and reached its next tick"); + } + + sealed class SyncPassCountingLogger(ILogger inner) : ILogger + { + int started; + + public int Started => started; + + public IDisposable BeginScope(TState state) where TState : notnull => inner.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (state is IReadOnlyList> values + && values.Any(value => value.Key == "{OriginalFormat}" && Equals(value.Value, "Performing sync for {ServiceName}"))) + { + Interlocked.Increment(ref started); + } + + inner.Log(logLevel, eventId, state, exception, formatter); + } + } + protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService(); protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService(); protected IRetryBatchStore RetryBatchStore => targetServices.GetRequiredService(); protected IRetryStagingStore RetryStagingStore => targetServices.GetRequiredService(); protected IFailedMessageRetryDataStore FailedMessageRetryDataStore => targetServices.GetRequiredService(); + protected IMessageRedirectsDataStore MessageRedirectsDataStore => targetServices.GetRequiredService(); + protected INotificationsDataStore NotificationsStore => targetServices.GetRequiredService(); + protected ITrialLicenseDataProvider TrialLicenseDataProvider => targetServices.GetRequiredService(); + protected IRetryHistoryDataStore RetryHistoryStore => targetServices.GetRequiredService(); + + // Built by hand, because its DI constructor needs NServiceBus settings the target services do not have. + // A zero cache duration makes every read go to the table. + protected ISubscriptionStorage SubscriptionStorage => + new SubscriptionStorage(targetServices.GetRequiredService(), "Particular.ServiceControl", "Particular.ServiceControl", [], TimeSpan.Zero, TimeProvider.System); protected async Task CopyCategory(string categoryId, CancellationToken cancellationToken = default) { @@ -583,6 +677,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; diff --git a/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs new file mode 100644 index 0000000000..14ba89749c --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class NotificationSettingsMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory NotificationSettingsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.NotificationSettings)!; + + // Every field differs from its default, so a field the copy drops cannot match by accident. + static EmailNotifications Seeded => new() + { + Enabled = true, + SmtpServer = "smtp.contoso.com", + SmtpPort = 587, + AuthenticationAccount = "servicecontrol", + AuthenticationPassword = "not-a-real-password", + EnableTLS = true, + To = "ops@contoso.com", + From = "servicecontrol@contoso.com" + }; + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + [TestCase("NotificationsSettingsDocuments", "NotificationsSettings")] + [TestCase("NotificationsSettings", "NotificationsSettingsDocuments")] + public async Task Copies_notification_settings_as_one_row_in_settings_from_either_collection(string collection, string otherCollection) + { + var seeded = Seeded; + await SeedSourceNotificationSettings(seeded, collection); + + var result = await CopyCategory("NotificationSettings"); + + var collections = (await SourceStore.Maintenance.ForDatabase(Source.PrimaryDatabase).SendAsync(new GetCollectionStatisticsOperation())).Collections; + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(collections, Does.ContainKey(collection).And.Not.ContainKey(otherCollection), "the test proves nothing unless the seed landed in the collection this case names"); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(seeded)), "LoadSettings reads through GetSetting, so every field must survive that read"); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } + + [Test] + public async Task A_document_with_no_email_settings_copies_nothing() + { + await SeedSourceNotificationSettings(null); + + var result = await CopyCategory("NotificationSettings"); + + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.Zero, + "a row may not carry a null document, and the product reads a null value and a missing row as the same default"); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(new EmailNotifications()))); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs new file mode 100644 index 0000000000..da6e68e86e --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +[TestFixture] +[NonParallelizable] +class RetryOperationsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_one_document_into_both_target_tables() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-10), + CompletionTime = CompletedAt.AddMinutes(-9), + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + }); + history.HistoricOperations.Add(HistoricOperation("group-3", CompletedAt.AddMinutes(-20))); + history.UnacknowledgedOperations.Add(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-5), + CompletionTime = CompletedAt.AddMinutes(-4), + Last = DateTime.MaxValue, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + }); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(3)); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)), Is.EqualTo(3), "--migration-verify counts both tables, and a count of either table alone would report 2 or 1"); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(Columns), Is.EqualTo(history.HistoricOperations.Select(Columns)), "every column of the historic operation arrives as the source holds it"); + Assert.That(stored.UnacknowledgedOperations.Select(Columns), Is.EqualTo(history.UnacknowledgedOperations.Select(Columns)), "every column of the pending operation arrives as the source holds it, even a Last of DateTime.MaxValue, which InMemoryRetry records when it has no last time"); + } + + [Test] + public async Task Duplicate_pending_operations_with_one_key_copy_once_and_count_the_rest_as_already_present() + { + var history = new RetryHistory(); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt, originator: "First in the document")); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt.AddMinutes(1), originator: "Second in the document")); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the second lands on the key the first fills, so it loses no row the target lacks and is neither a skip nor a primary-key violation"); + Assert.That(result.Skipped, Is.Zero); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.UnacknowledgedOperations.Single().Originator, Is.EqualTo("Second in the document"), "RavenDB appends a later retry of the same group, and the SQL store keeps the latest one, so the copy keeps the last in document order"); + } + + [Test] + public async Task Copies_more_operations_than_one_statement_could_carry() + { + var history = new RetryHistory(); + + foreach (var index in Enumerable.Range(0, OperationsPerList)) + { + history.HistoricOperations.Add(HistoricOperation($"historic-{index}", CompletedAt.AddMinutes(index))); + history.UnacknowledgedOperations.Add(PendingOperation($"pending-{index}", CompletedAt)); + } + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(2 * OperationsPerList)); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.HistoricOperations.Select(operation => operation.RequestId))); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.UnacknowledgedOperations.Select(operation => operation.RequestId))); + } + + // More rows than SQL Server's 2,000 parameters could carry at even one column each. + const int OperationsPerList = 2_100; + + // Whole seconds, because PostgreSQL stores microseconds and would not give back a time with more precision. + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + static (string, RetryType, DateTime, DateTime, string, bool, int) Columns(HistoricRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Originator, operation.Failed, operation.NumberOfMessagesProcessed); + + static (string, RetryType, DateTime, DateTime, DateTime, string, string, bool, int) Columns(UnacknowledgedRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Last, operation.Originator, operation.Classifier, operation.Failed, operation.NumberOfMessagesProcessed); + + internal static HistoricRetryOperation HistoricOperation(string requestId, DateTime completedAt) => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + internal static UnacknowledgedRetryOperation PendingOperation(string requestId, DateTime completedAt, string originator = "Shipping") => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Last = completedAt, + Originator = originator, + Classifier = "Shipping-Failures", + NumberOfMessagesProcessed = 1 + }; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs new file mode 100644 index 0000000000..6d07667274 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Recoverability; + +[TestFixture] +[NonParallelizable] +class RetryResetMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_retry_of_retry_operations_leaves_both_tables_as_one_copy_left_them() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var copied = await Target.Count(RetryOperations); + + var saved = await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.RetryOperations))); + var stored = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + var second = await CopyCategory(MigrationCategoryIds.RetryOperations); + + using (Assert.EnterMultipleScope()) + { + Assert.That(saved.Version, Is.EqualTo(stored.Version), "the returned checkpoint carries the version the reset saved"); + Assert.That(saved.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(copied, Is.EqualTo(4), "the first copy fills both tables"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(copied), "a historic row gets a new key on every insert, so a retry that kept them would double the history"); + Assert.That(second.AlreadyPresent, Is.Zero, "the pending table was emptied too, so the re-read finds none of its rows already present"); + } + } + + [Test] + public async Task A_retry_of_a_category_whose_keys_the_copy_supplies_deletes_nothing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = ChangedAt }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = ChangedAt }); + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.MessageRedirects); + await CopyCategory(MigrationCategoryIds.RetryOperations); + + await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.MessageRedirects))); + var second = await CopyCategory(MigrationCategoryIds.MessageRedirects); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.AlreadyPresent, Is.EqualTo(2), "rows already in SQL are left alone, so the re-read finds both"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the reset deletes only for the category it names, so another category's copied rows stay"); + } + } + + [Test] + public async Task A_stale_reset_deletes_nothing() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var stale = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(stale)); + + Assert.ThrowsAsync(() => Target.ResetForRetry(ResetForRetry(stale))); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the delete shares the reset's transaction, so a refused reset leaves every copied row"); + } + + // The same reset --migration-retry makes. It keeps StartedAt and leaves SourceTotal alone, because the engine does not write it. + static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0, + SkipReasons = null, + SettledAt = null, + LastError = null + }; + + static RetryHistory TwoHistoricAndTwoPending() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-1", ChangedAt)); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-2", ChangedAt.AddMinutes(-10))); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-3", ChangedAt)); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-4", ChangedAt.AddMinutes(-10))); + return history; + } + + static MigrationCategory RetryOperations => MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime ChangedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs new file mode 100644 index 0000000000..87455299ac --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Extensibility; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class SubscriptionsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Two_message_type_versions_for_the_same_subscriber_merge_onto_one_row_and_the_merge_is_already_present() + { + await SeedSourceSubscriptions( + new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), + new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); + + var result = await CopyCategory("Subscriptions"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a version merge loses no row the target lacks, and a skip would leave the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a merge is not a skip, so the category is Done"); + + var subscribers = await SubscriptionStorage.GetSubscriberAddressesForMessage( + [new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0))], new ContextBag(), TestContext.CurrentContext.CancellationToken); + + Assert.That(subscribers.Count(), Is.EqualTo(1)); + Assert.That(subscribers.Select(subscriber => (subscriber.TransportAddress, subscriber.Endpoint)), Is.EqualTo(new[] { ("Billing@BILLING01", "Billing") }), + "the copied row holds the subscriber's address and endpoint as the source has them"); + } + + [Test] + public async Task A_legacy_subscriber_with_no_endpoint_gets_the_endpoint_the_product_stores_for_one() + { + var copiedType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var storedType = new MessageType("MyMessages.OrderShipped", new Version(1, 0, 0)); + var token = TestContext.CurrentContext.CancellationToken; + + await SeedSourceSubscriptions(new SubscriptionRow(copiedType.TypeName, copiedType.Version, "Billing@BILLING01", Endpoint: null)); + + var result = await CopyCategory("Subscriptions"); + + // Another type, so the product's upsert cannot overwrite the copied row it is compared with. + await SubscriptionStorage.Subscribe(new Subscriber("Billing@BILLING01", null), storedType, new ContextBag(), token); + + var copied = await SubscriptionStorage.GetSubscriberAddressesForMessage([copiedType], new ContextBag(), token); + var stored = await SubscriptionStorage.GetSubscriberAddressesForMessage([storedType], new ContextBag(), token); + + Assert.That(result.Copied, Is.EqualTo(1), "Endpoint is NOT NULL on the target, so a row built without a fallback fails the batch and leaves the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(copied.Single().TransportAddress, Is.EqualTo("Billing@BILLING01")); + Assert.That(copied.Single().Endpoint, Is.EqualTo(stored.Single().Endpoint), + "the copy must store the endpoint SubscriptionStorage itself stores for the same legacy subscriber"); + Assert.That(copied.Single().Endpoint, Is.EqualTo("Billing"), "the endpoint is the part of the address before the '@'"); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs index fd07ff535b..a30e2caa57 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs @@ -67,6 +67,8 @@ public Task Write(MigrationCategory category, MigrationBat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; @@ -115,6 +117,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; diff --git a/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs new file mode 100644 index 0000000000..a280f2d58a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class TrialEndDateMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory TrialEndDateCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.TrialEndDate)!; + + [Test] + public async Task Copies_the_trial_end_date_as_one_row_in_settings() + { + var endDate = new DateOnly(2027, 3, 15); + await SeedSourceTrialEndDate(endDate); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.Null, "the target holds no trial end date before the copy, so the date read afterwards came from the copy"); + + var result = await CopyCategory("TrialEndDate"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.EqualTo(endDate), "the product's own trial licence store must read back the seeded date"); + Assert.That(await Target.Count(TrialEndDateCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs new file mode 100644 index 0000000000..6802772bce --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class When_the_heartbeat_sync_runs_after_a_migration : MigrationAcceptanceTest +{ + [Test] + public async Task Migrated_endpoint_settings_survive_the_sync_under_the_name_their_known_endpoint_uses() + { + var time = new FakeTimeProvider(); + + var billing = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = "BILLING01" }, Monitored = true }; + + await SeedSource(($"KnownEndpoints/{billing.EndpointDetails.GetDeterministicId()}", billing)); + await SeedSourceKnownEndpoints("SmokeSales"); + await SeedSourceEndpointSettings(("Billing", true), ("smokesales", true), ("SmokeSales", false)); + + await RunRequiredCopy(customize: services => + { + services.AddSingleton(time); + CountSyncPasses(services); + }); + + time.Advance(TimeSpan.FromSeconds(21)); + await WaitForSyncPass(time); + + var settings = await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(settings.Select(setting => setting.Name), Does.Contain(string.Empty), + "The heartbeat sync did not create the default row, which it does after its delete loop. The pass did not finish, so the other checks prove nothing."); + Assert.That(settings.Select(setting => setting.Name), Does.Contain("Billing"), + "The heartbeat sync deleted the migrated Billing setting. It only does that when no Billing known endpoint was copied."); + Assert.That(settings.Where(setting => setting.Name == "SmokeSales").Select(setting => setting.TrackInstances), Is.EqualTo(new[] { false }), + "SmokeSales must keep the operator's Do Not Track. On a target whose name column ignores case, the two settings become one row. If the copy keeps 'smokesales', the sync deletes it and SmokeSales gets the default."); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs new file mode 100644 index 0000000000..0e805ce94a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure; + +[TestFixture] +[NonParallelizable] +class When_the_offline_migration_logs : MigrationAcceptanceTest +{ + [Test] + public void It_logs_only_through_the_providers_the_instance_is_configured_with() + { + string[] registered = null; + + Assert.ThrowsAsync(async () => await MigrateCommand.Run(Settings, builder => + { + registered = ProviderTypes(builder.Services); + throw new ProvidersCaptured(); + })); + + var configured = ProviderTypes(new ServiceCollection().AddLogging(logging => logging.ConfigureLogging(Settings.LoggingSettings.LogLevel))); + + Assert.That(registered, Is.EquivalentTo(configured), + "an extra default provider prints every line twice and prints EF Core's SQL, which the configured providers filter out"); + } + + static string[] ProviderTypes(IServiceCollection services) => + [.. services + .Where(service => service.ServiceType == typeof(ILoggerProvider)) + .Select(service => (service.ImplementationType ?? service.ImplementationInstance?.GetType() ?? service.ImplementationFactory?.Method.ReturnType)?.FullName ?? "unknown")]; + + sealed class ProvidersCaptured : Exception; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index e0b683b1c8..066d6e8a2c 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -30,7 +30,12 @@ sealed class EFCoreMigrationTarget( new LicensingEndpointsWriter(migrationDialect), new LicensingThroughputWriter(migrationDialect), new LicensingReportMasksWriter(migrationDialect), - new LicensedEndpointDetailsWriter(migrationDialect) + new LicensedEndpointDetailsWriter(migrationDialect), + new MessageRedirectsWriter(migrationDialect), + new SubscriptionsWriter(migrationDialect), + new NotificationSettingsWriter(migrationDialect), + new TrialEndDateWriter(migrationDialect), + new RetryOperationsWriter(migrationDialect) }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => @@ -113,6 +118,36 @@ internal static int AlreadyPresentIn(MigrationCategory category, MigrationBatch public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => ExecuteWithDbContext((dbContext, token) => WriterFor(category).Count(dbContext, token), cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, resetToken) => + { + var storeKeyed = writers.GetValueOrDefault(reset.CategoryId) as IStoreKeyedCategoryWriter; + + var (deleted, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + var deletedRows = storeKeyed is null ? 0 : await storeKeyed.DeleteCopiedRows(dbContext, token); + var stored = await dbContext.UpsertCheckpoint(reset, token); + await transaction.CommitAsync(token); + + return (deletedRows, stored); + }, resetToken); + + if (storeKeyed is null) + { + logger.LogInformation("Reset category {CategoryId} for a retry and kept the rows it copied", reset.CategoryId); + } + else + { + logger.LogInformation("Reset category {CategoryId} for a retry and deleted the {DeletedRowCount} rows it copied, because a re-read would add them again under new keys", reset.CategoryId, deleted); + } + + return saved; + }, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => writers.Keys; public IReadOnlyDictionary DocumentTypes => writers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs index 1e81a7fff2..a5f241096e 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs @@ -193,7 +193,7 @@ static DateTime ReadResolvedDate(MigrationRow source, string key) => // Npgsql only writes UTC to a timestamp with time zone column, and a value whose kind is Unspecified is already UTC, so it must not be shifted. // Kept apart from the RavenDB source's copy because this project does not reference it. - static DateTime AsUtc(DateTime value) => + internal static DateTime AsUtc(DateTime value) => value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); // Uses the same length limit that ingestion applies to the MessageType column. diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs new file mode 100644 index 0000000000..726c80ad1e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; + +/// +/// A category writer whose rows get their keys from the database, so a re-read would add every row again. +/// The target deletes those rows when it resets the category for a retry. +/// The delete empties whole tables, so it is only safe for a category copied while ServiceControl is stopped. +/// +interface IStoreKeyedCategoryWriter +{ + /// + /// Deletes every row in this category's tables and returns how many it deleted. + /// Runs in the same transaction as the checkpoint reset. + /// + Task DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs new file mode 100644 index 0000000000..b0db77217d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.MessageRedirects; + +/// +/// Turns a redirect from the source into the row the product's own save would write. +/// +static class MessageRedirectRowMapper +{ + public static MessageRedirectEntity Map(MessageRedirect redirect) => new() + { + FromPhysicalAddress = redirect.FromPhysicalAddress, + ToPhysicalAddress = redirect.ToPhysicalAddress, + LastModified = redirect.LastModified + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs new file mode 100644 index 0000000000..400af82e6c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Recoverability; + +/// +/// Turns a retry operation from the source into the row the target stores. +/// Times are stored as UTC, and a time with no kind is taken as UTC already. +/// +static class RetryOperationRowMapper +{ + public static HistoricRetryOperationEntity Map(HistoricRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Originator = operation.Originator, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; + + public static UnacknowledgedRetryOperationEntity Map(UnacknowledgedRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Last = operation.Last, + Originator = operation.Originator, + Classifier = operation.Classifier, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs new file mode 100644 index 0000000000..734283aae7 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +/// +/// Turns a subscription from the source into the row the target stores. +/// The version is dropped, because the row has no column for it. +/// +static class SubscriptionRowMapper +{ + public static SubscriptionEntity Map(SubscriptionRow row) => new() + { + MessageType = row.MessageType, + TransportAddress = row.TransportAddress, + // The value SubscriptionStorage.Subscribe stores for a subscriber with no endpoint. + Endpoint = row.Endpoint ?? row.TransportAddress.Split('@').First() + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs index 72db9db495..b13a6e8e43 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -7,10 +7,9 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.DataMigration; /// -/// Writes every per-endpoint setting as stored. A setting for an endpoint the target does not know is copied too, -/// and the heartbeat settings sync removes it by its own rule once the instance opens. On a target whose name -/// column ignores case, two names that differ only in case are one row, and the writer reports each setting the -/// insert drops as a merge. +/// Copies every per-endpoint setting as stored. +/// Where the name column ignores case, names that differ only in case become one row, and the dropped ones are +/// reported as merges. The row kept is the one whose name matches a known endpoint exactly. /// sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter { @@ -24,8 +23,7 @@ public override Task Count(ServiceControlDbContext dbContext, Cancellation protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) { var insert = MigrationInsert.For(dbContext); - List rows = []; - List rowSourceIds = []; + List<(string SourceId, EndpointSettingsEntity Row)> rows = []; List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; foreach (var (row, settings) in documents) @@ -38,47 +36,32 @@ protected override async Task PrepareDocuments(ServiceControlDbCo continue; } - rows.Add(entity); - rowSourceIds.Add(row.SourceId); + rows.Add((row.SourceId, entity)); } - // The source yields document-id order, and the statement keeps the first of any keys the database treats as one. + rows = await KnownEndpointNamesFirst(dbContext, rows, cancellationToken); + + // The insert keeps the first of any names the database treats as equal. return new PreparedBatch( - async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, - await MergesIn(dbContext, rows, rowSourceIds, cancellationToken)); + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); } - // Mirrors the insert's de-duplication, so an operator can see which settings a case-insensitive name column dropped. - async Task> MergesIn(ServiceControlDbContext dbContext, List rows, List rowSourceIds, CancellationToken cancellationToken) + async Task> KnownEndpointNamesFirst(ServiceControlDbContext dbContext, List<(string SourceId, EndpointSettingsEntity Row)> rows, CancellationToken cancellationToken) { - var sameName = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); - string[] rowNames = [.. rows.Select(row => row.Name)]; - - // The database compares under the column's collation, so this also returns a stored name that differs only in case. - var keptNames = await dbContext.EndpointSettings.AsNoTracking() - .Select(stored => stored.Name) - .Where(name => rowNames.Contains(name)) - .ToListAsync(cancellationToken); - - var merges = new List<(string SourceId, string Detail)>(); + var nameComparer = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); + string[] mergingNames = [.. rows.GroupBy(row => row.Row.Name, nameComparer).Where(group => group.Count() > 1).SelectMany(group => group.Select(row => row.Row.Name))]; - for (var index = 0; index < rows.Count; index++) + if (mergingNames.Length == 0) { - var name = rows[index].Name; - var keptName = keptNames.FirstOrDefault(kept => sameName.Equals(kept, name)); - - if (keptName is null) - { - keptNames.Add(name); - } - else if (!string.Equals(keptName, name, StringComparison.Ordinal)) - { - merges.Add((rowSourceIds[index], $"the name column treats '{name}' and '{keptName}' as one endpoint, so it kept '{keptName}' and dropped the settings of '{name}'")); - } + return rows; } - return merges; + var knownNames = (await dbContext.KnownEndpoints.Where(endpoint => mergingNames.Contains(endpoint.Name)).Select(endpoint => endpoint.Name).ToListAsync(cancellationToken)).ToHashSet(StringComparer.Ordinal); + + // OrderBy is stable, so only the known names move to the front. + return [.. rows.OrderBy(row => knownNames.Contains(row.Row.Name) ? 0 : 1)]; } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs new file mode 100644 index 0000000000..8116cac289 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Infrastructure; +using Microsoft.EntityFrameworkCore; + +/// +/// Finds the rows the insert will drop because another row, earlier in the batch or already in the table, has the same +/// key under the column's collation. A row the table already holds exactly is not a merge. +/// Returns each dropped row's source id with a line naming the key that was kept. +/// +static class KeyMerges +{ + public static async Task> In(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, CancellationToken cancellationToken = default) where TEntity : class + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; + var keyProperties = entityType.FindPrimaryKey()!.Properties; + var queryColumn = keyProperties.FirstOrDefault(property => property.ClrType == typeof(string)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no text key column, so no collation can join two of its keys."); + Func[] columnMatches = [.. keyProperties.Select(property => property.ClrType == typeof(string) ? TextMatch(dialect.KeyComparer(typeof(TEntity), property.Name)) : object.Equals)]; + + object?[] KeyOf(TEntity entity) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(entity))]; + bool SameKey(object?[] first, object?[] second) => columnMatches.Select((matches, column) => matches(first[column], second[column])).All(same => same); + + var stored = new List(); + string[] queryValues = [.. rows.Select(row => (string)queryColumn.GetGetter().GetClrValue(row.Row)!).Distinct(StringComparer.Ordinal)]; + + foreach (var chunk in queryValues.Chunk(dialect.RowsPerStatementFor(dbContext))) + { + stored.AddRange(await dbContext.Set().AsNoTracking().Where(entity => chunk.Contains(EF.Property(entity, queryColumn.Name))).ToListAsync(cancellationToken)); + } + + // A stored key wins over every row of the batch, and an earlier row over a later one, as in the insert. + var kept = stored.Select(entity => (Key: KeyOf(entity), Stored: true)).ToList(); + var merges = new List<(string SourceId, string Detail)>(); + + foreach (var (sourceId, row) in rows) + { + var key = KeyOf(row); + var match = kept.FindIndex(entry => SameKey(entry.Key, key)); + + if (match < 0) + { + kept.Add((key, false)); + } + else if (kept[match].Stored && kept[match].Key.SequenceEqual(key)) + { + // Claimed by this row, so a later row of the batch with the same key is a merge. + kept[match] = (key, false); + } + else + { + merges.Add((sourceId, $"{entityType.GetTableName()} treats {Describe(key)} and {Describe(kept[match].Key)} as one key, so it kept {Describe(kept[match].Key)} and dropped the row for {Describe(key)}")); + } + } + + return merges; + } + + static Func TextMatch(IEqualityComparer comparer) => (first, second) => comparer.Equals((string?)first, (string?)second); + + static string Describe(object?[] key) => string.Join(", ", key.Select(value => $"'{value}'")); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs index 9bcf8ec909..1e9e702dc9 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs @@ -1,9 +1,6 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; -using DbContexts; -using Entities; using Infrastructure; -using Microsoft.EntityFrameworkCore; using Particular.LicensingComponent.Contracts; using ServiceControl.Persistence.DataMigration; @@ -11,21 +8,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; /// Writes the licensed endpoint details file as one row in Settings. A row the target already holds is left as it is /// and counted as already present. /// -sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.LicensedEndpointDetails) { public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; - - const string SettingKey = SettingKeys.LicensedEndpointDetails; - - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); - - protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, LicensedEndpointDetails Document)> documents, CancellationToken cancellationToken = default) - { - List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; - - return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); - } - - public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => - dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs index 18823e068c..6abd48b3d3 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs @@ -1,30 +1,13 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; -using DbContexts; -using Entities; using Infrastructure; -using Microsoft.EntityFrameworkCore; using ServiceControl.Persistence.DataMigration; /// /// Writes the report masks as one row in Settings. A row the target already holds is left as it is and counted as /// already present. /// -sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter> +sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter>(migrationDialect, SettingKeys.ReportMasks) { public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; - - const string SettingKey = SettingKeys.ReportMasks; - - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); - - protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, List Document)> documents, CancellationToken cancellationToken = default) - { - List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; - - return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); - } - - public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => - dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs new file mode 100644 index 0000000000..9544efd9d9 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +/// +/// Copies the message redirects, one row per redirect. A redirect whose source address is already taken is reported as a merge. +/// A redirect that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class MessageRedirectsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.MessageRedirects.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, MessageRedirect Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var rows = new List<(string SourceId, MessageRedirectEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, redirect) in documents) + { + if (redirect.FromPhysicalAddress is null || redirect.ToPhysicalAddress is null) + { + var column = redirect.FromPhysicalAddress is null ? nameof(MessageRedirectEntity.FromPhysicalAddress) : nameof(MessageRedirectEntity.ToPhysicalAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"MessageRedirects.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = MessageRedirectRowMapper.Map(redirect); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs new file mode 100644 index 0000000000..85c0842e91 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Infrastructure; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the notification email settings as one Settings row. +/// +sealed class NotificationSettingsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.NotificationEmails) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs new file mode 100644 index 0000000000..e84dd7d149 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs @@ -0,0 +1,124 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +/// +/// Copies the retry history into two tables. A pending operation whose key is already taken is reported as a merge. +/// Historic operations get new keys, so a retry's reset empties both tables before the history is read again. +/// An operation that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class RetryOperationsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override async Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.LongCountAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.LongCountAsync(cancellationToken); + + // Whole tables, because the target is refused unless it held no ServiceControl data before the first copy. + public async Task DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.ExecuteDeleteAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.ExecuteDeleteAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, RetryOperationRow Document)> documents, CancellationToken cancellationToken = default) + { + var pendingInsert = MigrationInsert.For(dbContext); + var historicText = HistoricTextColumns(dbContext); + List historic = []; + List<(string SourceId, UnacknowledgedRetryOperationEntity Row)> unacknowledged = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, operation) in documents) + { + if (operation.Historic is { } historicOperation) + { + if (historicOperation.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it")); + } + else if (UnstorableHistoricValue(historicText, row, RetryOperationRowMapper.Map(historicOperation)) is { } unstorable) + { + skips.Add(unstorable); + } + else + { + historic.Add(historicOperation); + } + + continue; + } + + var pending = operation.Unacknowledged!; + + if (pending.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it")); + continue; + } + + var entity = RetryOperationRowMapper.Map(pending); + + if (UnstorableValueSkip(pendingInsert, migrationDialect, row, entity) is { } unstorablePending) + { + skips.Add(unstorablePending); + continue; + } + + unacknowledged.Add((row.SourceId, entity)); + } + + // Reversed, because the insert keeps the first of a repeated key and the latest retry must win. + unacknowledged.Reverse(); + + return new PreparedBatch( + async (context, token) => + { + var inserted = await migrationDialect.InsertMissing(context, [.. unacknowledged.Select(pending => pending.Row)], token); + + // Fresh entities per attempt, because a failed save can leave generated ids on the previous ones. + // Reversed, so on a tie in completion time the first in the document still wins, as in RavenDB. + context.HistoricRetryOperations.AddRange(Enumerable.Reverse(historic).Select(RetryOperationRowMapper.Map)); + await context.SaveChangesAsync(token); + + return historic.Count + inserted.Count; + }, + historic.Count + unacknowledged.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, unacknowledged, cancellationToken)); + } + + // Read from the model, because MigrationInsert.For refuses a table whose key the store generates. + static (IProperty Property, string TableAndColumn)[] HistoricTextColumns(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(HistoricRetryOperationEntity))!; + + return [.. entityType.GetProperties().Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{entityType.GetTableName()}.{property.GetColumnName()}"))]; + } + + (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableHistoricValue((IProperty Property, string TableAndColumn)[] text, MigrationRow row, HistoricRetryOperationEntity entity) + { + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(entity) is string { Length: var units } value && property.GetMaxLength() is { } maxLength && units > maxLength && migrationDialect.TextLength(value) is var length && length > maxLength) + { + return (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}, on document '{row.SourceId}'"); + } + } + + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(entity) is string value && migrationDialect.RejectedCharacterIn(value) is { } rejected) + { + return (row.SourceId, MigrationSkipReason.MalformedValue, $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains, on document '{row.SourceId}'"); + } + } + + return null; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs new file mode 100644 index 0000000000..8d285689bb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes a category that the target stores as one JSON row in Settings. +/// A row already under that key is left alone and counted as already present. +/// +abstract class SettingRowWriter(IMigrationSqlDialect migrationDialect, string settingKey) : MigrationCategoryWriter where TDocument : class +{ + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Settings.LongCountAsync(setting => setting.Key == settingKey, cancellationToken); + + /// + /// Builds the Settings row for one document, storing the whole document by default. + /// Override it when the document holds the value rather than being it. + /// + protected virtual SettingEntity RowFor(string key, TDocument document) => SettingRowMapper.Map(key, document); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => RowFor(settingKey, document.Document))]; + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs new file mode 100644 index 0000000000..a748509255 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Copies the subscriptions. The key leaves out the message type's version, so two versions of one type for one +/// subscriber become one row, and the second is reported as a merge. +/// A subscription that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class SubscriptionsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Subscriptions.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, SubscriptionRow Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var rows = new List<(string SourceId, SubscriptionEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, subscription) in documents) + { + if (subscription.MessageType is null || subscription.TransportAddress is null) + { + var column = subscription.MessageType is null ? nameof(SubscriptionEntity.MessageType) : nameof(SubscriptionEntity.TransportAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"Subscriptions.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = SubscriptionRowMapper.Map(subscription); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs new file mode 100644 index 0000000000..5e0ace65ad --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Entities; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the trial end date as one Settings row, storing only the date, as the product does. +/// +sealed class TrialEndDateWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.TrialEndDate) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override SettingEntity RowFor(string key, TrialMetadata document) => SettingRowMapper.Map(key, document.TrialEndDate); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index cfd20d1582..18266881e0 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -31,7 +31,12 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig new LicensingEndpointsReader(lifecycle), new LicensingThroughputReader(lifecycle), new LicensingReportMasksReader(lifecycle), - new LicensedEndpointDetailsReader(lifecycle) + new LicensedEndpointDetailsReader(lifecycle), + new MessageRedirectsReader(lifecycle), + new SubscriptionsReader(lifecycle), + new NotificationSettingsReader(lifecycle), + new TrialEndDateReader(lifecycle), + new RetryOperationsReader(lifecycle) }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs index 477edb9b3b..f5b8d3f66e 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs @@ -2,11 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; -using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Runtime.CompilerServices; -using System.Threading; using Particular.LicensingComponent.Contracts; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB.Throughput; @@ -16,32 +12,11 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; /// whole document, or no rows when the document does not exist. The batch's cursor is the document's id, and a read /// resumed after that id gives nothing. /// -sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.LicencedEndpointDetailsDocumentId) { - const string DocumentId = LicensingDataStore.LicencedEndpointDetailsDocumentId; - public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; - public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - ReadLicensedEndpointDetails(resumeAfter, cancellationToken); - - async IAsyncEnumerable ReadLicensedEndpointDetails(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (resumeAfter == DocumentId) - { - yield break; - } - - if (resumeAfter is not null) - { - throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); - } - - using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); - var document = await session.LoadAsync(DocumentId, cancellationToken); - - yield return new MigrationBatch( - document is null ? [] : [new MigrationRow(DocumentId, document, ReadOnlyDictionary.Empty)], - DocumentId); - } + protected override IEnumerable<(string SourceId, LicensedEndpointDetails Document)> RowsFrom(LicensedEndpointDetails document) => + [(DocumentId, document)]; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs index 6b74a4e20a..47597f08cd 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs @@ -2,11 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; -using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Runtime.CompilerServices; -using System.Threading; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB.Throughput; using ServiceControl.Persistence.RavenDB.Throughput.Models; @@ -16,32 +12,11 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; /// holding the list, or no rows when the document does not exist, and a list stored as null reads as an empty one. /// The batch's cursor is the document's id, and a read resumed after that id gives nothing. /// -sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader>(lifecycle) +sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader>(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.ReportMasksDocumentId) { - const string DocumentId = LicensingDataStore.ReportMasksDocumentId; - public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; - public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - ReadReportMasks(resumeAfter, cancellationToken); - - async IAsyncEnumerable ReadReportMasks(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (resumeAfter == DocumentId) - { - yield break; - } - - if (resumeAfter is not null) - { - throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); - } - - using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); - var document = await session.LoadAsync(DocumentId, cancellationToken); - - yield return new MigrationBatch( - document is null ? [] : [new MigrationRow(DocumentId, document.MaskedStrings ?? [], ReadOnlyDictionary.Empty)], - DocumentId); - } + protected override IEnumerable<(string SourceId, List Document)> RowsFrom(ReportConfigurationDocument document) => + [(DocumentId, document.MaskedStrings ?? [])]; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs new file mode 100644 index 0000000000..12795ddbba --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs @@ -0,0 +1,22 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.RavenDB.MessageRedirects; + +/// +/// Reads the message redirects document as one row per redirect, in the document's order. +/// +sealed class MessageRedirectsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, MessageRedirectsDataStore.CollectionId) +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + // The writer skips a redirect with no source address, and that skip still needs an id to name. + protected override IEnumerable<(string SourceId, MessageRedirect Document)> RowsFrom(MessageRedirectsCollection document) => + document.ToRedirects().Select((redirect, index) => (redirect.FromPhysicalAddress ?? $"{DocumentId}#{index}", redirect)); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs new file mode 100644 index 0000000000..9a0ae42aea --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Editing; + +/// +/// Reads the notification settings document, whose email settings become one Settings row. +/// +sealed class NotificationSettingsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, NotificationsDataStore.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; + + protected override IEnumerable<(string SourceId, EmailNotifications Document)> RowsFrom(NotificationsSettingsDocument document) => + document.Email is { } email ? [(DocumentId, email)] : []; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs new file mode 100644 index 0000000000..72280d9697 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs @@ -0,0 +1,24 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; +using ServiceControl.Recoverability; + +/// +/// Reads the retry history document as rows, historic operations first, then unacknowledged ones. +/// +sealed class RetryOperationsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, RetryHistoryDataStore.DocumentId) +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + protected override IEnumerable<(string SourceId, RetryOperationRow Document)> RowsFrom(RetryHistory document) => + [ + .. document.HistoricOperations.Select(operation => ($"historic|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(operation, null))), + .. document.UnacknowledgedOperations.Select(operation => ($"unacknowledged|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(null, operation))) + ]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs new file mode 100644 index 0000000000..7f61ffa043 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs @@ -0,0 +1,27 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the subscriptions document as one row per subscriber of each message type and version. +/// +sealed class SubscriptionsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, Subscriptions.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + protected override IEnumerable<(string SourceId, SubscriptionRow Document)> RowsFrom(Subscriptions document) + { + foreach (var subscription in document.All.Values) + { + foreach (var subscriber in subscription.Subscribers) + { + yield return ($"{subscription.Id}|{subscriber.TransportAddress}", new SubscriptionRow(subscription.MessageType.TypeName, subscription.MessageType.Version, subscriber.TransportAddress, subscriber.Endpoint)); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs new file mode 100644 index 0000000000..3724963f67 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the trial metadata document, whose end date becomes one Settings row. +/// +sealed class TrialEndDateReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, TrialLicenseDataProvider.TrialMetadataId) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override IEnumerable<(string SourceId, TrialMetadata Document)> RowsFrom(TrialMetadata document) => [(DocumentId, document)]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs new file mode 100644 index 0000000000..aac5d66c70 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs @@ -0,0 +1,54 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads a category that RavenDB keeps in one document with a fixed id. +/// The document is one batch, and its id is the cursor. A missing document gives a batch with no rows. +/// +abstract class SingleDocumentReader(RavenReadOnlySourceLifecycle lifecycle, string databaseName, string documentId) : MigrationCategoryReader(lifecycle) + where TStored : class + where TDocument : class +{ + /// + /// The id of the one document this category reads, and the cursor of its only batch. + /// + protected string DocumentId { get; } = documentId; + + public sealed override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadDocument(resumeAfter, cancellationToken); + + /// + /// Turns the loaded document into rows, in the order the writer gets them. Called only when the document exists. + /// Leave out a missing value rather than return a null document. + /// + protected abstract IEnumerable<(string SourceId, TDocument Document)> RowsFrom(TStored document); + + async IAsyncEnumerable ReadDocument(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (resumeAfter == DocumentId) + { + yield break; + } + + if (resumeAfter is not null) + { + throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); + } + + using var session = Lifecycle.OpenSession(databaseName); + var document = await session.LoadAsync(DocumentId, cancellationToken); + + yield return new MigrationBatch( + document is null ? [] : [.. RowsFrom(document).Select(row => new MigrationRow(row.SourceId, row.Document, ReadOnlyDictionary.Empty))], + DocumentId); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs index c801d834fa..1aefbe8e67 100644 --- a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Persistence.RavenDB.Editing class NotificationsDataStore(IRavenSessionProvider sessionProvider) : INotificationsDataStore { - const string SingleDocumentId = "NotificationsSettings/All"; + internal const string SingleDocumentId = "NotificationsSettings/All"; public async Task LoadSettings(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs index 93f55c7ef1..2df9d1af09 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs @@ -9,7 +9,7 @@ class RetryHistoryDataStore(IRavenSessionProvider sessionProvider) : IRetryHistoryDataStore { - const string DocumentId = "RetryOperations/History"; + internal const string DocumentId = "RetryOperations/History"; public async Task> GetRetryHistory(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs index f0feaa2aaf..a3d51910a1 100644 --- a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs @@ -23,6 +23,6 @@ public async Task StoreTrialEndDate(DateOnly trialEndDate, CancellationToken can await session.SaveChangesAsync(cancellationToken); } - static string TrialMetadataId = "metadata/trialinformation"; + internal const string TrialMetadataId = "metadata/trialinformation"; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs index a2a7b1c7e3..fa228033bb 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs @@ -59,11 +59,9 @@ public async Task An_absent_document_gives_one_empty_batch_that_still_records_th } [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] - [TestCase(MigrationCategoryIds.LicensedEndpointDetails, "LicensedEndpointDetails")] public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string categoryId, string documentId) { await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); - await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); var batches = await Read(categoryId, documentId); @@ -71,11 +69,9 @@ public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string } [TestCase(MigrationCategoryIds.LicensingReportMasks)] - [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] public async Task A_read_resumed_after_a_cursor_the_reader_never_issued_is_refused(string categoryId) { await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); - await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); var exception = Assert.ThrowsAsync(() => Read(categoryId, "Sales.Orders/Broker")); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs new file mode 100644 index 0000000000..ebd380e1bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using RavenMessageRedirectsDataStore = Persistence.RavenDB.MessageRedirects.MessageRedirectsDataStore; + +class MessageRedirectsSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects); + + [Test] + public async Task A_document_whose_last_redirect_was_removed_gives_one_empty_batch() + { + var redirect = new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }; + await MessageRedirectsDataStore.AddRedirect(redirect); + await MessageRedirectsDataStore.RemoveRedirect(redirect); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { RavenMessageRedirectsDataStore.CollectionId })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index 463c008a2f..67a81ea63f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -4,11 +4,16 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.Unicast.Subscriptions; using NUnit.Framework; using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Notifications; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.RavenDB.Throughput.Models; using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; @@ -133,13 +138,22 @@ public async Task Reading_every_category_creates_no_index() [MigrationCategoryIds.LicensingEndpoints] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingThroughput] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingReportMasks] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso"] }), - [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }) + [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }), + [MigrationCategoryIds.MessageRedirects] = tests => tests.MessageRedirectsDataStore.AddRedirect(new MessageRedirect { FromPhysicalAddress = "Sales@OLDHOST", ToPhysicalAddress = "Sales@NEWHOST", LastModified = DateTime.UtcNow }), + [MigrationCategoryIds.Subscriptions] = tests => tests.SeedSubscription(), + [MigrationCategoryIds.NotificationSettings] = tests => tests.NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + [MigrationCategoryIds.TrialEndDate] = tests => tests.ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + [MigrationCategoryIds.RetryOperations] = tests => tests.RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10) }; string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; + // Both databases, because the licensing readers read the throughput database. async Task IndexNames() => - await DocumentStore.Maintenance.SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); + [.. await IndexNamesIn(DatabaseName), .. await IndexNamesIn(ThroughputDatabaseName)]; + + Task IndexNamesIn(string database) => + DocumentStore.Maintenance.ForDatabase(database).SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); const int IndexNamePageSize = 1024; @@ -152,6 +166,21 @@ async Task SeedThroughputDocument(string id, object document) await session.SaveChangesAsync(); } + async Task SeedSubscription() + { + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)), + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using var session = await SessionProvider.OpenSession(); + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + async Task SeedEndpointSettings(int count) { for (var index = 0; index < count; index++) diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs new file mode 100644 index 0000000000..a45824203a --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Unicast.Subscriptions; +using NUnit.Framework; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +class SubscriptionsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Flattens_the_dictionary_of_subscriptions_into_one_row_per_subscriber() + { + var eventV1 = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var eventV2 = new MessageType("MyMessages.OrderPlaced", new Version(2, 0, 0)); + + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = eventV1, + Subscribers = [ + new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }, + new SubscriptionClient { Endpoint = "Shipping", TransportAddress = "Shipping@SHIP01" } + ] + }; + document.All["v2"] = new Subscription + { + Id = "v2", + MessageType = eventV2, + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + + var batches = await Read(); + + Assert.That(batches.Last().Cursor, Is.EqualTo(Subscriptions.SingleDocumentId)); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => (row.SourceId, (SubscriptionRow)row.Document)), Is.EquivalentTo(new[] + { + ("v1|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("v1|Shipping@SHIP01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping")), + ("v2|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")) + }), "each row carries every value its subscriber holds, under an id naming the subscription and the address"); + } + + async Task> Read() + { + await using var source = await OpenMigrationSource(); + return await CollectBatches(source, MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions)); + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs index 8a90e4965c..9a081a0b2b 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -30,7 +30,12 @@ public async Task A_case_merge_inside_one_batch_logs_both_names_and_the_one_kept await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); - Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + using (Assert.EnterMultipleScope()) + { + Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/2 ").And.Contains("kept 'Sales'"), "the operator finds the dropped row by this source id"); + Assert.That(targetWarnings.Messages, Has.None.StartsWith("Merged EndpointSettings/1 "), "the kept row was not dropped"); + } } [Test] @@ -44,6 +49,26 @@ public async Task A_case_merge_with_a_row_an_earlier_batch_wrote_is_logged_and_a Assert.That(targetWarnings.Messages, Has.Exactly(1).Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); } + [Test] + public async Task A_case_merge_keeps_the_setting_whose_name_a_known_endpoint_uses_even_when_it_comes_second() + { + await OpenACaseInsensitiveTarget("Sales"); + + await Write(("EndpointSettings/1", "sales", true), ("EndpointSettings/2", "Sales", false)); + + List stored; + using (var scope = ServiceProvider.CreateScope()) + { + stored = await scope.ServiceProvider.GetRequiredService().EndpointSettings.AsNoTracking().ToListAsync(); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Select(setting => (setting.Name, setting.TrackInstances)), Is.EqualTo(new[] { ("Sales", false) }), "the heartbeat sync deletes a setting no known endpoint has the exact name of, so the orphan must be the one dropped"); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/1 ").And.Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + } + async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) { using (var scope = ServiceProvider.CreateScope()) @@ -70,11 +95,13 @@ async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) await ServiceProvider.GetRequiredService().Open(); } - async Task Write(params (string SourceId, string Name)[] rows) + Task Write(params (string SourceId, string Name)[] rows) => Write([.. rows.Select(row => (row.SourceId, row.Name, true))]); + + async Task Write(params (string SourceId, string Name, bool TrackInstances)[] rows) { var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); var batch = new MigrationBatch( - [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = true }, new Dictionary()))], + [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = row.TrackInstances }, new Dictionary()))], rows[^1].SourceId); var checkpoint = await ServiceProvider.GetRequiredService().Read(category.Id) ?? new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs deleted file mode 100644 index 1c6cba5d3d..0000000000 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ServiceControl.Persistence.Tests; - -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using NUnit.Framework; -using Particular.LicensingComponent.Contracts; -using ServiceControl.Persistence.DataMigration; - -class LicensedEndpointDetailsMigrationTargetTests : PersistenceTestBase -{ - [SetUp] - public Task OpenTarget() => Target.Open(); - - IMigrationTarget Target => ServiceProvider.GetRequiredService(); - - static readonly MigrationCategory LicensedEndpointDetailsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensedEndpointDetails); - - [Test] - public async Task Count_is_the_licensed_endpoint_details_row_alone() - { - await LicensingDataStore.SaveReportMasks(["contoso"]); - Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.Zero, "the report masks row is not a licensed endpoint details row"); - - await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); - - Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the report masks row and every other row in the table"); - } -} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs deleted file mode 100644 index 45faa38e2e..0000000000 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ServiceControl.Persistence.Tests; - -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using NUnit.Framework; -using Particular.LicensingComponent.Contracts; -using ServiceControl.Persistence.DataMigration; - -class LicensingReportMasksMigrationTargetTests : PersistenceTestBase -{ - [SetUp] - public Task OpenTarget() => Target.Open(); - - IMigrationTarget Target => ServiceProvider.GetRequiredService(); - - static readonly MigrationCategory LicensingReportMasksCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingReportMasks); - - [Test] - public async Task Count_is_the_report_masks_row_alone() - { - await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); - Assert.That(await Target.Count(LicensingReportMasksCategory), Is.Zero, "the licensed endpoint details row is not a report masks row"); - - await LicensingDataStore.SaveReportMasks(["contoso"]); - - Assert.That(await Target.Count(LicensingReportMasksCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the licensed endpoint details row and every other row in the table"); - } -} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs new file mode 100644 index 0000000000..a6e99bb5bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var entity = MessageRedirectRowMapper.Map(new MessageRedirect + { + FromPhysicalAddress = "Sales@OLDHOST", + ToPhysicalAddress = "Sales@NEWHOST", + LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) + }); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs new file mode 100644 index 0000000000..467cedecc8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory MessageRedirectsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + + [Test] + public async Task A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var batch = BatchOf( + ("Redirect/1", Redirect(new string('f', 450), new string('t', 450))), + ("Redirect/2", Redirect(new string('f', 451), "Sales@NEWHOST")), + ("Redirect/3", Redirect("Billing@OLDHOST", new string('t', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new MessageRedirectsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(MessageRedirectsCategory, batch, CheckpointAfter("Redirect/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "Redirect/2", "Redirect/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await MessageRedirectsDataStore.GetRedirects()).Select(redirect => redirect.FromPhysicalAddress), Is.EqualTo(new[] { new string('f', 450) })); + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details[0], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/2'")); + Assert.That(details[1], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/3'")); + } + } + + [Test] + public async Task A_redirect_whose_address_the_key_treats_as_an_earlier_ones_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("Redirect/1", Redirect("Sales@HOST", "Sales@NEWHOST")), + ("Redirect/2", Redirect("Sales@HOST", "Sales@OTHERHOST")), + ("Redirect/3", Redirect("sales@host", "Sales@NEWHOST"))); + var keyIgnoresCase = dialect.KeyComparer(typeof(MessageRedirectEntity), nameof(MessageRedirectEntity.FromPhysicalAddress)).Equals("Sales@HOST", "sales@host"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new MessageRedirectsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresCase ? new[] { "Redirect/2", "Redirect/3" } : new[] { "Redirect/2" }), + "a second copy of an address is dropped on every database, and a case-only variant only where the key column ignores case"); + Assert.That(merges[0].Detail, Does.Contain("kept 'Sales@HOST'"), "the target logs this line, so it has to name the address it kept"); + } + } + + static MessageRedirect Redirect(string from, string to) => + new() { FromPhysicalAddress = from, ToPhysicalAddress = to, LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) }; + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index 0cf8345618..3237b5700e 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -1,13 +1,17 @@ namespace ServiceControl.Persistence.Tests; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; @@ -47,8 +51,30 @@ public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_ Assert.That(model.FindEntityType(typeof(SettingEntity))!.FindProperty(nameof(SettingEntity.Value))!.GetMaxLength(), Is.Null, "NoValueCanBeTooLong exempts the settings categories because of this; give them a value-too-long test if the column gains a limit"); } + [Test] + public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_them_for_a_retry_and_no_other_writer_does() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)); + + Assert.That(writers, Is.EquivalentTo(EntitiesEachWriterWrites.Keys), "list the entities a new writer writes here, so this test can tell whether a retry must delete its rows"); + + using (Assert.EnterMultipleScope()) + { + foreach (var (writer, entities) in EntitiesEachWriterWrites) + { + var storeKeyed = entities.Any(entity => model.FindEntityType(entity)!.FindPrimaryKey()!.Properties.Any(property => property.ValueGenerated != ValueGenerated.Never)); + + Assert.That(typeof(IStoreKeyedCategoryWriter).IsAssignableFrom(writer), Is.EqualTo(storeKeyed), + $"{writer.Name}: a retry keeps the rows of a writer that is not an IStoreKeyedCategoryWriter, so a re-read adds rows with store keys a second time, and a writer whose keys the copy supplies must not delete"); + } + } + } + // Each is one Settings row under a constant key whose value column has no maximum length, so no source value can be too long for it. - static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails]; + static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. static readonly Dictionary ValueTooLongTests = new() @@ -57,6 +83,26 @@ public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_ [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), [MigrationCategoryIds.LicensingEndpoints] = nameof(LicensingEndpointsMigrationTargetTests.A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), - [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.MessageRedirects] = nameof(MessageRedirectsMigrationTargetTests.A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.Subscriptions] = nameof(SubscriptionsMigrationTargetTests.A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy), + [MigrationCategoryIds.RetryOperations] = nameof(RetryOperationsMigrationTargetTests.A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + }; + + // The writers do not say which entities they write, so this list does. + static readonly Dictionary EntitiesEachWriterWrites = new() + { + [typeof(KnownEndpointsWriter)] = [typeof(KnownEndpointEntity)], + [typeof(EndpointSettingsWriter)] = [typeof(EndpointSettingsEntity)], + [typeof(FailedMessagesWriter)] = [typeof(FailedMessageEntity), typeof(FailedMessageGroupEntity)], + [typeof(LicensingEndpointsWriter)] = [typeof(LicensingEndpointEntity)], + [typeof(LicensingThroughputWriter)] = [typeof(LicensingEndpointThroughputEntity)], + [typeof(LicensingReportMasksWriter)] = [typeof(SettingEntity)], + [typeof(LicensedEndpointDetailsWriter)] = [typeof(SettingEntity)], + [typeof(MessageRedirectsWriter)] = [typeof(MessageRedirectEntity)], + [typeof(SubscriptionsWriter)] = [typeof(SubscriptionEntity)], + [typeof(NotificationSettingsWriter)] = [typeof(SettingEntity)], + [typeof(TrialEndDateWriter)] = [typeof(SettingEntity)], + [typeof(RetryOperationsWriter)] = [typeof(HistoricRetryOperationEntity), typeof(UnacknowledgedRetryOperationEntity)] }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs index e04ebaee93..8e43f3c55f 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs @@ -23,6 +23,17 @@ public static async Task> DetailsFrom(IServiceProvider ser return [.. prepared.Skips.Select(skip => skip.Detail)]; } + /// + /// Prepares a batch without writing it, and returns the rows the writer reports as merges with their log detail. + /// + public static async Task> MergesFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService(), batch); + + return prepared.Merges; + } + /// /// The table and column a property is stored in, as this provider names them. /// diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs new file mode 100644 index 0000000000..6763d6d562 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Recoverability; + +class RetryOperationRowMapperTests : PersistenceTestBase +{ + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task Sets_every_mapped_property_of_a_historic_and_an_unacknowledged_operation() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + using (Assert.EnterMultipleScope()) + { + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + })); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs new file mode 100644 index 0000000000..58adff0d22 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs @@ -0,0 +1,244 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Recoverability; + +class RetryOperationsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory RetryOperationsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var dialect = ServiceProvider.GetRequiredService(); + var fourHundredCodePoints = char.ConvertFromUtf32(0x1F600) + new string('r', 399); + var countsCodePoints = dialect.TextLength(fourHundredCodePoints) < fourHundredCodePoints.Length; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(new string('r', 400)), null)), + ("historic/2", new RetryOperationRow(Historic(new string('r', 401)), null)), + ("historic/3", new RetryOperationRow(Historic(fourHundredCodePoints), null)), + ("pending/1", new RetryOperationRow(null, Pending(new string('p', 401), "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", new string('c', 451)))), + ("pending/3", new RetryOperationRow(null, Pending(new string('p', 400), new string('c', 450))))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/3")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + var historicRequestId = PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(HistoricRetryOperationEntity.RequestId)); + string[] expectedDetails = + [ + $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/2'", + .. countsCodePoints ? Array.Empty() : new[] { $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/3'" }, + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.RequestId))} takes at most 400 characters and the value holds 401, on document 'pending/1'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Classifier))} takes at most 450 characters and the value holds 451, on document 'pending/2'" + ]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(fourHundredCodePoints, Has.Length.EqualTo(401), "the emoji request id is one UTF-16 unit over the column, and 400 code points"); + Assert.That(result.Copied, Is.EqualTo(countsCodePoints ? 3 : 2), "a value exactly as long as its column fits, in either table"); + Assert.That(result.SkippedIds, Is.EqualTo(countsCodePoints ? new[] { "historic/2", "pending/1", "pending/2" } : new[] { "historic/2", "historic/3", "pending/1", "pending/2" }), + "400 code points fit a column that counts code points, and a skip there would lose a retry operation the database could hold"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = countsCodePoints ? 3 : 4 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(countsCodePoints ? new[] { new string('r', 400), fourHundredCodePoints } : new[] { new string('r', 400) })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.Classifier)), Is.EqualTo(new[] { (new string('p', 400), new string('c', 450)) })); + Assert.That(details, Is.EqualTo(expectedDetails)); + } + } + + [Test] + public async Task A_retry_operation_holding_a_character_the_database_refuses_is_skipped_as_malformed_value_where_the_database_refuses_it() + { + var dialect = ServiceProvider.GetRequiredService(); + var refused = dialect.RejectedCharacterIn("\0") is not null; + var historic = Historic("group-1"); + historic.Originator = "Sales\0Orders"; + var pending = Pending("group-2", "Shipping-Failures"); + pending.Originator = "Ship\0ping"; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("historic/2", new RetryOperationRow(Historic("group-3"), null)), + ("pending/1", new RetryOperationRow(null, pending)), + ("pending/2", new RetryOperationRow(null, Pending("group-4", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.SkippedIds, Is.EqualTo(refused ? new[] { "historic/1", "pending/1" } : Array.Empty()), "only a database that refuses the character fails the statement over it"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-3" } : new[] { "group-1", "group-3" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-4" } : new[] { "group-2", "group-4" })); + + if (refused) + { + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.MalformedValue] = 2 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(HistoricRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'historic/1'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'pending/1'" + })); + } + } + } + + [Test] + public async Task A_retry_operation_with_no_request_id_is_skipped_naming_the_table_and_the_rest_copy() + { + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(null!), null)), + ("historic/2", new RetryOperationRow(Historic("group-1"), null)), + ("pending/1", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it", + "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the table the operation was bound for"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "historic/1", "pending/1" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-1" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_second_pending_operation_with_the_same_key_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic("group-2"), null)), + ("pending/skipped", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Billing-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress)))); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2", "pending/1" } : new[] { "pending/1" }), + "the last pending operation with one key is kept and the earlier ones are dropped on every database, a case-only request id counts as the same key only where the column ignores case, one request retried as another type never merges, and a historic or skipped row never merges"); + Assert.That(merges[0].Detail, Does.Contain(requestIdIgnoresCase ? "kept 'GROUP-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'" : "kept 'group-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'"), + "the target logs this line, so it has to name the key it kept and the key it dropped without swapping them"); + } + } + + [Test] + public async Task A_pending_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService(); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + var first = await Target.Write(RetryOperationsCategory, BatchOf(("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("pending/2", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress))), + ("historic/1", new RetryOperationRow(Historic("group-2"), null))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), second); + var result = await Target.Write(RetryOperationsCategory, second, first.Saved with { Cursor = second.Cursor }); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2" } : Array.Empty()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another retry type never"); + + if (requestIdIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'group-2', 'FailureGroup' and dropped the row for 'GROUP-2', 'FailureGroup'")); + } + + Assert.That(result.Copied, Is.EqualTo(requestIdIgnoresCase ? 2 : 3), "the historic row and the other retry type are copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(requestIdIgnoresCase ? 2 : 1)); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.RetryType)), Is.EquivalentTo(requestIdIgnoresCase + ? new[] { ("group-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) } + : new[] { ("group-2", RetryType.FailureGroup), ("GROUP-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) })); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_retry_operation_time_of_local_or_unspecified_kind_is_copied_as_the_same_instant() + { + var historic = Historic("group-1"); + historic.StartTime = CompletedAt.AddMinutes(-1).ToLocalTime(); + historic.CompletionTime = DateTime.SpecifyKind(CompletedAt, DateTimeKind.Unspecified); + var pending = Pending("group-2", "Shipping-Failures"); + pending.StartTime = DateTime.SpecifyKind(CompletedAt.AddMinutes(-1), DateTimeKind.Unspecified); + pending.CompletionTime = CompletedAt.ToLocalTime(); + pending.Last = CompletedAt.AddMinutes(1).ToLocalTime(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("pending/1", new RetryOperationRow(null, pending))); + + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/1")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "Npgsql refuses a time that is not UTC for a timestamptz column, and that would fail the whole retry history"); + Assert.That(stored.HistoricOperations.Select(operation => (operation.StartTime, operation.CompletionTime)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt) }), + "an Unspecified time is already UTC and a Local time is converted, so both keep their instant"); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.StartTime, operation.CompletionTime, operation.Last)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt, CompletedAt.AddMinutes(1)) })); + } + } + + static HistoricRetryOperation Historic(string requestId) => new() + { + RequestId = requestId, + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + static UnacknowledgedRetryOperation Pending(string requestId, string classifier, RetryType retryType = RetryType.FailureGroup) => new() + { + RequestId = requestId, + RetryType = retryType, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = classifier, + NumberOfMessagesProcessed = 1 + }; + + static MigrationBatch BatchOf(params (string Id, RetryOperationRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.RetryOperations, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..98c6ec145d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +class SettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly string[] SettingsCategoryIds = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; + + [TestCase(MigrationCategoryIds.LicensingReportMasks)] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] + [TestCase(MigrationCategoryIds.NotificationSettings)] + [TestCase(MigrationCategoryIds.TrialEndDate)] + public async Task Count_is_the_row_under_the_categorys_own_key_alone(string categoryId) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == categoryId); + + foreach (var otherCategoryId in SettingsCategoryIds.Where(id => id != categoryId)) + { + await StoreSettingFor(otherCategoryId); + } + + Assert.That(await Target.Count(category), Is.Zero, "the other settings rows are not rows of this category"); + + await StoreSettingFor(categoryId); + + Assert.That(await Target.Count(category), Is.EqualTo(1), "counted as the whole Settings table, this would also count the other settings rows"); + } + + Task StoreSettingFor(string categoryId) => categoryId switch + { + MigrationCategoryIds.LicensingReportMasks => LicensingDataStore.SaveReportMasks(["contoso"]), + MigrationCategoryIds.LicensedEndpointDetails => LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }), + MigrationCategoryIds.NotificationSettings => NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + MigrationCategoryIds.TrialEndDate => ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + _ => throw new ArgumentOutOfRangeException(nameof(categoryId), categoryId, "This test stores no settings row for that category.") + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs new file mode 100644 index 0000000000..5334eef99d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class SubscriptionRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var row = SubscriptionRowMapper.Map(new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, row); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs new file mode 100644 index 0000000000..0b437aece1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs @@ -0,0 +1,154 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SubscriptionsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory SubscriptionsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions); + + [Test] + public async Task A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow(new string('t', 200), version, new string('a', 200), new string('e', 450))), + ("s/2", new SubscriptionRow(new string('t', 201), version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, new string('a', 201), "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", new string('e', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/4")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/2", "s/3", "s/4" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 3 }), + "an over-long value would fail the whole insert and halt the category"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.MessageType))} takes at most 200 characters and the value holds 201, on document 's/2'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.TransportAddress))} takes at most 200 characters and the value holds 201, on document 's/3'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.Endpoint))} takes at most 450 characters and the value holds 451, on document 's/4'" + })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { (new string('t', 200), new string('a', 200), new string('e', 450)) }), "the row whose values all fit is stored as it was read"); + Assert.That(await Target.Count(SubscriptionsCategory), Is.EqualTo(1), "Count is the rows the Subscriptions table holds"); + } + } + + [Test] + public async Task A_second_version_of_a_type_for_one_subscriber_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderShipped", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2", "s/3" } : new[] { "s/2" }), + "the key has no version, so a second version is dropped on every database, a case-only address only where the column ignores case, and another type never"); + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01'"), "the target logs this line, so it has to name the key it kept"); + + if (addressIgnoresCase) + { + Assert.That(merges[1].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'"), + "the line must not swap the key kept and the key dropped"); + } + } + } + + [Test] + public async Task A_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService(); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + var first = await Target.Write(SubscriptionsCategory, BatchOf(("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))), CheckpointAfter("s/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), second); + var result = await Target.Write(SubscriptionsCategory, second, first.Saved with { Cursor = second.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2" } : Array.Empty()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another address never"); + + if (addressIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'")); + } + + Assert.That(result.Copied, Is.EqualTo(addressIgnoresCase ? 1 : 2), "the other address is copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(addressIgnoresCase ? 2 : 1)); + Assert.That(await StoredSubscriptions(), Is.EquivalentTo(addressIgnoresCase + ? new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") } + : new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "billing@billing01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + [Test] + public async Task A_subscription_missing_a_key_column_is_skipped_naming_the_column_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", version, TransportAddress: null!, "Billing")), + ("s/2", new SubscriptionRow(MessageType: null!, version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", "Shipping"))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "Subscriptions.TransportAddress is NOT NULL and the document has no value for it", + "Subscriptions.MessageType is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the column that is missing"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/1", "s/2" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + async Task> StoredSubscriptions() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var stored = await scope.ServiceProvider.GetRequiredService().Subscriptions.AsNoTracking().ToListAsync(); + + return [.. stored.Select(subscription => (subscription.MessageType, subscription.TransportAddress, subscription.Endpoint))]; + } + + static MigrationBatch BatchOf(params (string Id, SubscriptionRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.Subscriptions, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index e2b93060ab..449c4cf85d 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -11,22 +11,19 @@ namespace ServiceControl.Persistence.DataMigration; public interface IMigrationTarget { /// - /// Makes the target ready to write. The host calls it before the copy and after the target's own checks have passed. + /// Makes the target ready to write. Called once, before the copy starts. /// Task Open(CancellationToken cancellationToken = default); /// - /// The most rows a source may return in one batch for this category. - /// The target picks it because its own database sets the limit, and a bigger batch fails the write. + /// The most rows this target can write in one batch for the category. A bigger batch fails. /// Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default); /// /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. - /// Add this batch's own outcome with and save that, so the stored counts are never a guess. - /// For a category that carries bodies, every row must have its body. - /// Bodies that go to external storage are written first, outside the transaction. So a failed write can leave a - /// stored body that no row points to, but never a row that points to a body that was not stored. + /// Build the checkpoint with . + /// Bodies kept outside the database are written first, so a failure can leave an unused body but never a row without one. /// Task Write( MigrationCategory category, @@ -35,26 +32,30 @@ Task Write( CancellationToken cancellationToken = default); /// - /// How many rows the target holds for one category, for progress and verify. - /// Counts only that category, even where two categories share a table. + /// How many rows the target holds for one category, even where categories share a table. /// Task Count(MigrationCategory category, CancellationToken cancellationToken = default); /// - /// The categories this target can write. A category outside this set is never handed to the engine. - /// Answers before Open and does not change across it. + /// Saves a reset checkpoint so its category can be copied again, and returns it with its new version. + /// Where the database makes the row keys, it also deletes the rows already copied, so none are copied twice. + /// Throws if another writer saved the checkpoint first. + /// + Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default); + + /// + /// The categories this target can write. Known before Open and never changes. /// IReadOnlyCollection SupportedCategoryIds { get; } /// - /// The type of this target takes, for each category in - /// . A row holding any other type fails its batch with an - /// . Answers before Open and does not change across it. + /// The type this target expects for each supported category. + /// A row of any other type fails its batch. Known before Open and never changes. /// IReadOnlyDictionary DocumentTypes { get; } } /// -/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +/// What the target did with one batch, and the checkpoint it saved with it. Every skipped row has a reason in SkipReasons. /// public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs new file mode 100644 index 0000000000..aa5939175e --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Recoverability; + +/// +/// One retry operation from the retry history document. Exactly one of Historic and Unacknowledged is set. +/// +public sealed record RetryOperationRow(HistoricRetryOperation? Historic, UnacknowledgedRetryOperation? Unacknowledged); diff --git a/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs new file mode 100644 index 0000000000..3441fd2fe6 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// +/// One subscriber of one message type, on its way from source to target. +/// +public sealed record SubscriptionRow(string MessageType, Version Version, string TransportAddress, string? Endpoint); diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index b6ef99b0a1..927b8be477 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -176,6 +176,13 @@ async Task WriteNow( public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + /// + /// Saves the reset through the checkpoint store, so a stale Version throws as on the real target. + /// It deletes no rows, because no unit test needs that. + /// + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs new file mode 100644 index 0000000000..8843923d12 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryStateExtensionsTests +{ + [Test] + public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsBenign()), + Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), + "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 4053d7d71b..7726a173f1 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -204,6 +204,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); @@ -223,6 +225,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); diff --git a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs index feedfd181d..8681aa14e9 100644 --- a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs +++ b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs @@ -6,6 +6,7 @@ namespace ServiceControl.Hosting.Commands; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Particular.ServiceControl.Hosting; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; @@ -47,6 +48,7 @@ internal static async Task Run(Settings settings, Action var builder = WebApplication.CreateBuilder(); // Persistence also registers runtime-only services whose dependencies this command does not need. builder.Host.UseDefaultServiceProvider(options => options.ValidateOnBuild = false); + builder.Logging.ClearProviders(); builder.Logging.ConfigureLogging(settings.LoggingSettings.LogLevel); builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddPersistence(settings); From eda1aa67db72e0aa4d4a62522d465cb2fe9b09d3 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Mon, 16 Nov 2026 21:19:53 +0800 Subject: [PATCH 24/28] Refactor migration documentation for clarity and accuracy --- .../ravendb-to-sql-migration-instructions.md | 30 +-- .../ravendb-to-sql-migration-overview.md | 198 ++++++++---------- 2 files changed, 96 insertions(+), 132 deletions(-) diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index d1879cf889..dbcf16c25e 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -25,12 +25,11 @@ flowchart TD S2["2. Create an empty SQL database and run --setup"] S3["3. Choose the optional data windows"] S4["4. Run --migration-dry-run and book the outage"] - S5["5. Back up RavenDB"] - S1 --> S2 --> S3 --> S4 --> S5 + S1 --> S2 --> S3 --> S4 end subgraph outage["ServiceControl stopped: the outage"] - S6["6. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] + S6["5. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] Q1{"Did --start-data-migration succeed?"} Fix["Correct the reported cause"] RetryAbandon1["Run --migration-retry or --migration-abandon for the Failed category"] @@ -42,17 +41,17 @@ flowchart TD end subgraph online["ServiceControl open on SQL: no way back to RavenDB"] - S7["7. Start ServiceControl"] - S8["8. Watch the background copy with --migration-status"] + S7["6. Start ServiceControl"] + S8["7. Watch the background copy with --migration-status"] Q2{"Is an optional category Failed?"} RetryAbandon2["Stop ServiceControl, run --migration-retry or --migration-abandon, then start it"] - S9["9. Stop ServiceControl, run --migration-verify, then start it"] + S9["8. Stop ServiceControl, run --migration-verify, then start it"] S7 --> S8 --> Q2 Q2 -->|Yes| RetryAbandon2 --> S8 Q2 -->|"No: every category is Done or Abandoned"| S9 end - S5 --> S6 + S4 --> S6 Q1 -->|Yes| S7 ``` @@ -99,7 +98,7 @@ In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You > [!IMPORTANT] > > - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. -> - Do not start ServiceControl again until step 7. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. +> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: @@ -153,14 +152,7 @@ The dry run reads RavenDB and tells you what the migration will do, including an Book the outage from the estimate in the report, and add a margin. It is an estimate, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. -## Step 5: Back up RavenDB - -> [!IMPORTANT] -> Back up both RavenDB databases, primary and throughput, before you start the copy. Or turn off RavenDB expiration on them. - -RavenDB keeps deleting old failed messages and event log items during the migration and after it. Without a backup, the RavenDB database that you keep to go back to loses data every day. - -## Step 6: Run the offline command +## Step 5: Run the offline command 1. Stop ServiceControl and every ingestion worker. 2. Run `ServiceControl.exe --start-data-migration` from the instance installation folder. In a container deployment, run the command in a one-off container against the same databases. @@ -169,7 +161,7 @@ RavenDB keeps deleting old failed messages and event log items during the migrat The command exits without starting ServiceControl. -## Step 7: Start ServiceControl +## Step 6: Start ServiceControl After the offline command reports success, start the ServiceControl service or container. ServiceControl detects the migration from SQL checkpoints and copies optional categories in the background. @@ -180,7 +172,7 @@ Normal startup refuses an unfinished required copy and tells you to run `--start If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). -## Step 8: Watch the background copy +## Step 7: Watch the background copy ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: @@ -195,7 +187,7 @@ If the copy slows down your production work, raise `ServiceControl/Migration/Thr Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. -## Step 9: Finish the migration +## Step 8: Finish the migration 1. Stop ServiceControl. 2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 1b3c015e16..0bb31d6c1d 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -3,7 +3,7 @@ > [!NOTE] > This build does not carry the whole migration yet. This page describes it as it will be when it ships. -This page describes the migration as designed, not what is built today. [The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. +[The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. ## Overview @@ -20,8 +20,8 @@ This page describes the migration as designed, not what is built today. [The ins - **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. - **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. - **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. -- **The main thing that makes the outage long is a big backlog of unresolved failures.** They almost never age out (see the [6.18 exception](#rows-removed-after-the-copy)), they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. -- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds the unresolved failures through a RavenDB index, so while ServiceControl is closed it reads only them. If the index is still catching up, the copy waits for it first. If the index is broken, or leaves out some failed messages, the copy reads every failed message instead, archived and resolved ones included, and logs a warning saying so. Then a big archive does make the outage longer. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They are required, rarely age out (see the [6.18 exception](#rows-removed-after-the-copy)), and can be months old. Keep them low by resolving and archiving. The *dry run* tells you where you stand. +- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds unresolved failures through a RavenDB index, and waits for it to catch up first. If the index is broken or incomplete, the copy reads every failed message instead and logs a warning. Then a big archive does make the outage longer. ```mermaid flowchart LR @@ -59,7 +59,7 @@ stateDiagram-v2 - **Short outage.** Only the required data copies while ServiceControl is closed. - **Support every RavenDB hosting shape.** Embedded, self-hosted or RavenDB Cloud, on one code path. -- **Never write to RavenDB.** But RavenDB's own expiration keeps deleting old failed messages and event log items, during the migration and afterwards. So the old database is a fallback that degrades from the moment you start, which is why [the instructions](ravendb-to-sql-migration-instructions.md#step-5-back-up-ravendb) have you back it up first. +- **Never write to RavenDB.** RavenDB's own expiration still deletes old failed messages and event log items. - **No duplicates and no gaps.** The copied rows and the *checkpoint* commit together, so a crash needs no clean-up. - **Every identifier anything points at is kept.** Only the event log, historic retry operations and pending integration events get new ids, because nothing points at theirs. - **Refuses rather than half-migrates.** Every check runs before the first row moves. @@ -72,7 +72,7 @@ stateDiagram-v2 - **Zero downtime.** There is a real, if short, outage. - **Rollback once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. - **Pausing or steering a running copy.** There is no pause, resume or abort. Decisions are two commands run with ServiceControl stopped, and every other change is a configuration edit and a restart. -- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine was design to be expandable in the future. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine is designed so it can be expanded in the future. ### The moving parts @@ -87,7 +87,7 @@ flowchart LR ``` - **Both persisters load into one ServiceControl process**, each in its own `AssemblyLoadContext`, and both are live during the copy. -- **The engine knows neither database.** It knows only `IMigrationSource` and `IMigrationTarget`, and deals in categories, *cursors* and counts. This allows it to be tested, and expanded in the future. +- **The engine knows neither database.** It works only through `IMigrationSource` and `IMigrationTarget`, in categories, *cursors* and counts, so it can be tested on its own and expanded in the future. - The source maps each category to what it reads, and describes itself as labelled facts. - The target maps each category to where it writes and how to count it. - Each side adds its own startup checks. @@ -100,13 +100,13 @@ flowchart LR [The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. - **The [free abort](#going-back-to-ravendb) window runs from the first `--start-data-migration` invocation until ServiceControl opens.** A successful command does not end this window; starting ServiceControl does. -- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command builds a persistence-only host and never starts it. Normal startup refuses unfinished required checkpoints before any hosted service starts. Optional categories never hold them back. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command never starts the host, and normal startup refuses while a required category is unfinished. ## Design ### Supported locations -The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no database-to-database transfer. RavenDB holds documents and SQL holds tables, so the rows have to be reshaped, and ServiceControl's own persisters already know how to read one and write the other. So what matters is whether the host can reach both ends, and how far the data travels. +The copy goes from RavenDB, through the ServiceControl host, to SQL, because documents must be reshaped into tables and ServiceControl's persisters already know both sides. So the host must reach both ends, and distance matters. | | Supported locations | | --- | --- | @@ -126,12 +126,8 @@ The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no - The host reaches the RavenDB source, the SQL target and the body store at the same time. - The SQL database is new and empty: only the schema `--setup` created. - Both RavenDB databases, primary and throughput, are on one server or cluster. The migration opens both through one `IDocumentStore`, and so does the licensing component. -- A SQL Server target has Full-Text Search installed, because message search needs it. - - The `AddFullTextSearch` EF Core migration checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent. - - An EF Core migration runs once, so this is checked the first time `--setup` brings a database up to that migration, not on every `--setup`. - - A stock SQL Server container image does not include Full-Text Search. - - PostgreSQL needs nothing extra, because its search index is a GIN index over `to_tsvector`. -- Transient failures on a managed target are already survivable. Retry on failure is on by default, and no setting turns it off. +- A SQL Server target has Full-Text Search installed, because message search needs it. `--setup` fails without it, and the stock SQL Server container image does not include it. PostgreSQL needs nothing extra. +- Transient failures on a managed target are retried by default. **Not supported:** @@ -171,14 +167,14 @@ There are eighteen *categories*: sixteen required and two optional. - RavenDB index definitions. - In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations` and `UnarchiveOperations`. - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. -- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. +- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Nothing has read it since 6.2. - Broker and audit service version details, which refill on the next throughput collection. - Failed message edit locks, which stop one failed message being edited twice. See [Open decisions](#values-that-change-or-merge-in-sql). -- Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. +- Heartbeat state, which neither persister stores. It is rebuilt in memory, so monitoring after the move behaves as after any restart. **Written new by the copy rather than copied:** -- Retry claims in `FailedMessageRetries` are not copied. A claim ties a message to the retry that sent it, and while it exists, retrying the message from the failed messages list sends nothing. The copy writes a new claim for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). As on RavenDB, the claim goes when the retried message succeeds, when it fails again, or when it is resent from Pending Retries. +- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). ### Optional data windows @@ -203,7 +199,7 @@ Every skipped row is counted under a reason, and its id is written to the log. A skip is one of three kinds: - **Fault, a retry can fix it.** Something outside the data went wrong. Fix the cause and retry. -- **Fault, no retry can fix it.** The RavenDB document itself cannot become a SQL row. A retry reads the same document and skips it again, so only [abandoning](#retry-and-abandon) gets past it. It is a fault because the row is lost, not because anything is broken. +- **Fault, no retry can fix it.** The RavenDB document cannot become a SQL row, so a retry skips it again. Only [abandoning](#retry-and-abandon) gets past it. - **Harmless.** A row SQL would have removed anyway. A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Open decisions](#values-that-change-or-merge-in-sql). @@ -219,12 +215,11 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied | Reason | | --- | -| A message body that RavenDB does not hold at all, counted as `BodyMissing`. The whole message is skipped, for the same reason. The one exception is a message that expired during the copy (see below), which a retry does get past | +| A message body that RavenDB does not hold, counted as `BodyMissing`. The whole message is skipped. A retry does get past a message that expired during the copy | | A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | | A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | -| A subscription whose message type or address is over 200 characters, the SQL key limit | | A row missing a value SQL requires, such as a known endpoint with no name, or a failed message with no failing endpoint address to retry to | -| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters. Cutting it would change a key or send retries to the wrong address, so the row is skipped. The skip names the column and the length, counted the way the database counts (PostgreSQL by characters, SQL Server by UTF-16 units) | +| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters, or a subscription's message type or address. Cutting it would change a key or send retries to the wrong address. The skip names the column and the length | | A value of the wrong type, such as a `TimeSent` that is not a date. On PostgreSQL, also a text value holding the character U+0000, which PostgreSQL cannot store | **Harmless skips:** @@ -235,16 +230,13 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied | An archived or resolved message, or event log item, already past its retention period | - **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. -- **A malformed document is skipped and counted, not a reason to stop the copy on its own.** The rest of the category carries on, so abandoning a category that read to the end and ended Failed this way gives up only those rows. But if the skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then also gives up every row it had not yet read. In that case raise the threshold or fix the cause, and retry. -- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them shortly after it opens, by the same rule it always applies, so the migration does not repeat that rule. +- **A malformed document does not stop the copy.** It is skipped and counted, and the rest of the category carries on. If skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then gives up every unread row too. Raise the threshold or fix the cause, and retry. +- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them itself shortly after it opens. - **Rows outside a window are not skips.** They are simply not part of the copy. -- **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. - - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). - - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. - - The exception is an unresolved message from version 6.18 or earlier, which can still expire during the copy. See [Open decisions](#rows-removed-after-the-copy). - - A failed message that expires after the copy reads it, but before the copy reads its body, is a skip. It counts as a body RavenDB does not hold, so it is a fault skip and the category ends Failed. For unresolved messages this happens only with a source last written by version 6.18 or earlier. A retry gets past it, because the expired message is no longer there to read. - - If the copy stops, and the message it stopped at expires before the next start, that start cannot carry on. The category stops with an error saying RavenDB holds no document with that id. If RavenDB is the right database, `--migration-retry` gets past it, because a retry reads the category from the start. - - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. +- **Rows RavenDB expires during the copy are not skips either.** Only archived or resolved failed messages and event log items expire, plus some unresolved messages from 6.18 or earlier (see [Open decisions](#rows-removed-after-the-copy)). + - A message that expires between the copy reading it and reading its body is a `BodyMissing` fault skip. A retry gets past it. + - If the message a stopped copy was at expires before the next start, the category stops with an error naming the missing id. `--migration-retry` gets past it. + - A window shorter than the retention period keeps the background copy clear of the expiry sweep. ### Startup checks @@ -252,7 +244,7 @@ In this order, cheapest first: 1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. `--start-data-migration` refuses a RavenDB target, because RavenDB is only ever the source. 2. The optional category windows are valid time spans of zero or more. -3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history, and no row count would ever show it. +3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history. 4. The SQL schema is current. This also refuses a database upgraded without `--setup`. 5. Until a required category has started copying: the SQL database holds no ServiceControl data. `--setup` writes none, but even one plain start of ServiceControl writes rows. Copying into a database that already served on SQL would mix the two. 6. Message body storage is writable. The probe body it writes is deleted again. @@ -265,7 +257,7 @@ In this order, cheapest first: If every required category is already Done or Abandoned, the offline command can succeed without opening the source. Normal startup does not need the source; only the online copier opens it. -Checks stop at the first one that refuses, and the refusal names it. Once open, the source can add checks of its own. The RavenDB source adds none. +Checks stop at the first refusal, which names the check. ```mermaid flowchart LR @@ -285,24 +277,17 @@ flowchart LR - **The source opens RavenDB read-only**: connect, check the version, stop. It never runs RavenDB's database setup, and it refuses any write request. - **Both source databases are opened through one `IDocumentStore`**, which is why they must be on the same server or cluster. -- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. A RavenDB start rewrites no stored document, so the copy reads exactly what this build's RavenDB persister would read. The only version check compares the RavenDB server to the RavenDB client, and only for an external source. -- **Distance to the source sets the pace.** The copier already holds each document from the stream, so each body costs one round trip rather than two. But it is one per message, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. +- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. +- **Distance to the source sets the pace.** Each message body is one round trip, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. ### Writing to SQL - **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). -- **A failed message keeps its failure groups and its last-modified time.** `LastModified` is RavenDB's own `@last-modified`, copied as it is, so failure groups, and lists filtered by when a message last changed, show the same messages as before. -- **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. -- **`StatusChangedAt` is rebuilt.** - - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. `@expires` is the status-change time plus `ServiceControl/ErrorRetentionPeriod`, so the copy subtracts the current value. A changed retention period makes this time wrong, see [Open decisions](#values-that-change-or-merge-in-sql). - - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. -- **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. -- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier decides inline or external storage with the same rule ingestion uses, because `IBodyStoragePersistence` does not apply it. The size limit for inline storage defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. -- **Throughput rows are written directly, not through the collector, and each day's count is set, not added.** - - Throughput is required, so it copies while ServiceControl is closed, before any collector has written to SQL. - - Setting makes the category safe to resume after a crash, where adding would double-count. The collector's own path does add (`LicensingDataStore.cs:198`). - - Copying the rows stops the audit and broker collectors gathering the same days again, because `LastCollectedDate` is worked out from the newest throughput row (`LicensingDataStore.cs:45`). - - The checkpoint stops a second pass overwriting days the collectors have written since. +- **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. +- **`UniqueMessageId` keeps its value** but becomes a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry key and the body key. +- **`StatusChangedAt` is rebuilt.** For archived and resolved messages it is `@expires` minus the current `ServiceControl/ErrorRetentionPeriod` (wrong if that changed, see [Open decisions](#values-that-change-or-merge-in-sql)). For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL`. +- **Bodies go through `IBodyStoragePersistence`**, which handles compression and the body store. The copier keeps a body inline when it is within `ServiceControl/MaxBodySizeToStore` (102,400 bytes by default), as ingestion does. +- **Throughput rows are written directly, and each day's count is set, not added**, so a resumed copy cannot double-count. The copied rows also stop the collectors gathering the same days again. ### Batch size and throttling @@ -313,7 +298,7 @@ flowchart LR ### Progress and checkpoints -The checkpoint is one row per category, kept in the SQL database and created by `--setup`. It holds the category's *state*, its cursor, its copied, skipped and already-present counts, how many of the already-present rows were merges, the count per skip reason, how many rows SQL held for the category when it settled, timings, the last error, the window it copies with, and a version number to catch a second writer. +The checkpoint is one row per category in the SQL database, created by `--setup`. It holds the category's *state*, cursor, counts, timings, last error and window, plus a version number that catches a second writer. ```mermaid sequenceDiagram @@ -331,21 +316,20 @@ sequenceDiagram - **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. - **A crash costs only the batch in flight.** Rerun `--start-data-migration` for required data, or restart ServiceControl for optional data. The copier resumes from the committed cursor. -- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. Unreadable bodies stay off the checkpoint until the write commits. -- **Every skip has a reason, or the save is refused.** The per-reason counts must add up exactly to the skipped count, and the target's counts must match what it committed. -- **The stored counts always describe the rows in SQL.** The target adds its own outcome and saves the result beside the rows, so nothing is saved later or separately. -- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts, or a command and a host, pointed at one database cannot interleave. -- **A copy stopped between two categories still shows the ones it never reached.** Before copying anything, a start saves a *not started* row for each category it is about to copy, so every gate sees them as unfinished. +- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. +- **Every skip has a reason, or the save is refused.** +- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts pointed at one database cannot interleave. +- **A copy stopped between two categories still shows the ones it never reached.** Each start first saves a *not started* row for every category it will copy. - **Only the copier and the two commands write the checkpoint.** The status and verify commands only read it. ### Failed categories -A category is *Failed* when something went wrong. Everything already copied stays committed. Nothing is retried in the background and nothing waits for a timer: the category stays Failed, with the reason written on it, until you run a [command](#retry-and-abandon). +A category is *Failed* when something went wrong. What it copied stays committed, and it stays Failed, with the reason, until you run a [command](#retry-and-abandon). **What makes a category Failed:** - **Any fault skip by the end of the category**, even one. -- **Too many fault skips in this run, which stops it early.** More than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Needing both means a tiny category does not stop on one bad row, and a huge one does not stop on its 101st. Under the threshold a category runs to the end, so you get the whole list of bad rows in one pass. +- **Too many fault skips in this run, which stops it early:** more than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Below that, a category runs to the end, so you see every bad row in one pass. - **The category's own counts not balancing:** every row read must be copied, skipped or already present. - **In a required category only, an exception or a stall of 30 minutes with nothing committed**, such as the body store unreachable, a certificate expired, the source or target down, or the disk full. The error and the cursor are recorded. @@ -353,21 +337,19 @@ A category is *Failed* when something went wrong. Everything already copied stay - **Harmless skips.** - **A shutdown or a crash.** The category stays Copying. The next `--start-data-migration` invocation resumes required data; the next ServiceControl start resumes optional data. -- **An exception or a stall in an optional category.** The error is written on the category, shown in status, and the next start carries on from the cursor. A RavenDB outage after opening is handled the same way. So a network blip never forces anything to be abandoned. If the error keeps coming back, retry or abandon the category. +- **An exception or a stall in an optional category**, including a RavenDB outage. The error shows in status, and the next start carries on from the cursor. If it keeps coming back, retry or abandon the category. **What a Failed category costs:** -- **The other categories still run.** Every required category gets its go in one start, so one fix-and-retry covers them all. A category that must follow a Failed one waits. +- **The other categories still run**, so one fix-and-retry covers them all. A category that depends on a Failed one waits. - **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. - **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. -**The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. - Some losses leave a category Done without any check noticing. They are listed under [Open decisions](#what-the-checks-do-not-catch). ### Retry and abandon -Stop ServiceControl before running either command. Rerun `--start-data-migration` for unfinished required data before starting the service again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. +Stop ServiceControl before either command. In a container, run it as a one-off `docker run --rm ` against the same database. | Command | Works on | What it does | | --- | --- | --- | @@ -383,36 +365,34 @@ Stop ServiceControl before running either command. Rerun `--start-data-migration **What a retry does:** -- **It re-reads the whole category from the start**, every time, which recovers from anything. Rows already in SQL are left alone, so only the rows that failed can change. Every message body is read again, so a retry takes about as long as the first copy. -- **Historic retry operations and pending integration events** get new keys from SQL on every insert, so a re-read could not recognise rows already copied. A retry of either first deletes what that category copied, in the same save as the reset. That is safe because both are required, so ServiceControl has not opened and the target started empty. +- **It re-reads the whole category from the start**, leaving rows already in SQL alone. Every body is read again, so a retry takes about as long as the first copy. +- **Historic retry operations and pending integration events** get new keys on every insert, so a retry first deletes what the category copied, in the same save as the reset. This is safe because both are required, so the target started empty. - **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. **To abandon:** -- **A required category can be abandoned once it is Failed or has started copying.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone before it starts, point back or start over: ServiceControl has not opened, so nothing being served is lost. +- **A required category can be abandoned once it has started.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone first, point back or start over. - **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. -- **Abandoning cascades to categories whose rows hang off it**, and the command names each: throughput history (it points at licensing endpoint records) and endpoint settings (it belongs to known endpoints). Group comments only waits for unresolved failed messages, so it is not abandoned with them. -- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a value too long for its column, a value of the wrong type, a missing required value, a body RavenDB does not hold. The refusal and the dry run say which, so abandon after the first failure rather than retrying. The exception is a category that stopped early at the threshold: abandoning it also gives up every row it had not yet read, so raise the threshold and retry first. +- **Abandoning cascades** to throughput history (from licensing endpoint records) and endpoint settings (from known endpoints), and the command names each. Group comments are not abandoned with unresolved failed messages. +- **A category that failed only on [skips no retry can fix](#skipped-rows) should be abandoned, not retried.** If it stopped early at the threshold, raise the threshold and retry first, because abandoning gives up every unread row too. > [!IMPORTANT] > Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. -**Nothing happens on a restart that you did not ask for.** A container or service restart policy restarts a refused host without anyone looking, and gets the same refusal each time, without reading anything. That is loud and loses nothing. - ### Ingestion workers -- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, as before, because only SQL can scale out. +- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, because only SQL can scale out. - **On SQL it starts only when every required category is Done or Abandoned**, based on SQL checkpoints. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. - **Every SQL writer runs that check**, because ingesting into a part-copied database would make going back a loss. - **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. -- **A worker or import that gets past the check marks the target as opened**, without an enable setting. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. +- **A worker or import that passes the check marks the target as opened**, which ends the free abort, because going back would lose what it wrote. > [!IMPORTANT] > **Stop workers before running `--start-data-migration`, and restart them only after the main host has opened.** A worker cannot detect a migration before its first checkpoint exists, see [Open decisions](#gaps-nothing-enforces). ### Going back to RavenDB -**While ServiceControl is closed**, only the copier has written to SQL and the migration has written nothing to RavenDB, which is still the source of truth. This is the *free abort*. RavenDB's own expiration still runs, though, unless you disabled it. +**While ServiceControl is closed**, RavenDB is still the source of truth, because the migration writes nothing to it. This is the *free abort*. RavenDB's own expiration still runs unless you disabled it. - **To go back:** point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. - **To try again later:** start from a new, empty SQL database (`--setup`). @@ -422,15 +402,9 @@ Stop ServiceControl before running either command. Rerun `--start-data-migration > > **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. -- **Every SQL start checks the checkpoint table.** A RavenDB instance has no checkpoint store and skips the check. -- **An unfinished required category keeps the host closed.** Run `--start-data-migration` again, after retrying or abandoning Failed categories. Startup never runs the required copy. -- **Outstanding optional checkpoints activate the online copier.** They do not prevent ServiceControl from opening. Failed optional categories remain Failed until an operator acts. -- **With no outstanding categories, ServiceControl does not open RavenDB.** Completed checkpoints remain as the migration record. -- **Before starting the copy, the offline command seeds required and selected optional checkpoints.** The online phase does not depend on a setting that an operator must remember to retain. - -Before ServiceControl opens, the other exits remain pointing back at RavenDB or starting over against a new, empty SQL database. +- **Every SQL start checks the checkpoint table.** An unfinished required category keeps the host closed until `--start-data-migration` finishes it. Outstanding optional categories start the online copier but do not stop ServiceControl opening. With nothing outstanding, ServiceControl does not open RavenDB. -**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl has opened, nothing can start without it. After, ServiceControl still opens, and the optional categories wait for RavenDB to come back, or can be abandoned. +**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl opens, nothing can start without it. After that, the optional categories wait for it, or can be abandoned. ### The dry run @@ -442,8 +416,8 @@ The dry run never writes to RavenDB, and can run again later against whatever is - An estimate of how long ServiceControl will be closed. - **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. - The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. -- How many failed error imports there are, with advice to run `--import-failed-errors` on RavenDB first. They are copied either way, but ones imported first arrive as ordinary failed messages. -- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure, unless that failure stopped it early at the threshold. +- How many failed error imports there are. They are copied either way, but running `--import-failed-errors` on RavenDB first is better. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. It says nothing about load on the source. @@ -453,22 +427,21 @@ It says nothing about load on the source. | --- | --- | --- | | `--migration-source-report` | Source facts and a document count per collection | Yes | | `--migration-dry-run` | The report above | Yes | -| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read, with skips and merges broken out. Exits 0 only when every category is Done or Abandoned, so a script can ask whether the migration is finished. Rows the copy never read are shown, not failed | Yes | +| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read. Exits 0 only when every category is Done or Abandoned | Yes | | `--migration-status` | Each category's state, progress, last error and the commands it can take | No, so it runs any time | - **On an embedded source, stop the ServiceControl service before any command that opens RavenDB.** A second RavenDB process cannot use a data directory the first one holds, so the dry run is part of the outage. -- **Verify reads every selected category in full**, though not the message bodies. On a big instance that takes a while, and on an embedded source that time is part of the outage too. +- **Verify reads every selected category in full**, though not the bodies, so it takes a while on a big instance. - **On an external source** they all run against a live instance. - **In a container**, run each as a one-off `docker run` of the same image against an external RavenDB server. **What verify compares:** -- **RavenDB against the rows the copy read, never against SQL as it is now.** Once ServiceControl opens, normal use changes most categories' SQL rows within minutes: ingestion, heartbeats, retries, archiving, the licensing collectors, ServiceControl's own custom checks and the retention sweep. A SQL count then measures that use, not the copy. -- **RavenDB holding more rows than the copy read means rows were never read.** Nothing writes to RavenDB once the copy starts, so it can only shrink. Verify flags these rows. They are not in SQL, so keep RavenDB. The same flag shows if something other than this migration wrote to RavenDB since, such as an old instance still running on it. -- **RavenDB holding fewer is shown, not judged.** RavenDB's expiry keeps deleting archived and resolved messages, event log items and a few unresolved messages from 6.18 or earlier, after the copy has read them. -- **The same expiry can hide rows that were never read**, see [Open decisions](#what-the-checks-do-not-catch). -- **A windowed category is counted inside the window it started with**, on RavenDB only. -- **Verify also prints how many rows SQL held for each category when it settled.** Before ServiceControl opens nothing else writes, so for a required category that is exactly what the copy left in SQL. For the two optional categories it includes what ServiceControl wrote while they copied. +- **RavenDB against the rows the copy read, never against SQL now**, because normal use changes SQL rows within minutes of opening. +- **RavenDB holding more rows than the copy read means rows were never read.** Keep RavenDB. The same flag shows if something else wrote to RavenDB, such as an old instance still running on it. +- **RavenDB holding fewer is shown, not judged**, because expiry keeps deleting rows after the copy reads them. Expiry can also hide rows that were never read, see [Open decisions](#what-the-checks-do-not-catch). +- **A windowed category is counted inside the window it started with.** +- **Verify also prints how many rows SQL held for each category when it settled.** ### Monitoring and settings @@ -482,30 +455,29 @@ It says nothing about load on the source. - **The source needs no new settings.** It reads the instance's existing RavenDB settings. - **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. -- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. Finished checkpoints deactivate copying automatically. There is no migration custom check, because custom checks are being removed from ServiceControl. -- **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. -- **Stopping a copy takes a restart.** +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** There is no migration custom check, because custom checks are being removed from ServiceControl. +- **There is no HTTP API, pause, resume or abort.** Stopping a copy takes a restart. ## Open decisions -The design currently accepts each case below. Each one changes, merges or drops data, or lets a loss pass without failing a category. None is settled: each needs a decision before the migration ships. Where the design gives a reason for accepting a case, it is stated. +Each case below changes, merges or drops data, or lets a loss pass without failing a category. The design accepts each one for now, but each needs a decision before the migration ships. ### Values that change or merge in SQL -- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so it keeps the newest attempt and the number of attempts. A message that failed five times still shows five attempts and still shows as a repeated failure, but only the newest attempt keeps its headers and exception. This applies to every failed message, whatever its status. -- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. -- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. - - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. - - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. - - The copy logs a warning naming both endpoints and the one it kept. - - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **Attempt history collapses to the newest attempt.** SQL keeps the newest attempt and the attempt count, so only the newest attempt keeps its headers and exception. +- **Message redirects, subscriptions and pending retry operations merge when their keys match in SQL.** The copy keeps one row, counts the other as already present, and logs a warning naming both. + - Redirects merge on source addresses the key column treats as equal. On SQL Server, by default, that includes addresses that differ only in case. + - Subscriptions also merge when they differ only in message-type version, because the SQL key leaves the version out. + - Pending retry operations with the same retry type and request id merge, keeping the latest. +- **On SQL Server, endpoint settings whose names differ only in case merge into one row**, because the name column compares without case by default. PostgreSQL keeps both. + - The copy keeps the setting whose name exactly matches a known endpoint, because ServiceControl deletes the other. Otherwise it keeps the first in RavenDB order, which is effectively arbitrary. + - The choice is made within one batch of up to a thousand. Across batches, the first one met wins. + - The copy logs a warning, and the dry run lists each pair. - **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. -- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row.** SQL Server ignores trailing spaces when it compares the key. PostgreSQL keeps both. - - The copy keeps the endpoint whose RavenDB document id sorts first in ordinal order. - - A day that both endpoints counted holds the sum of their counts, as it would if SQL Server had recorded both endpoints from the start. - - This is not counted as a merge. The copy logs no warning, and the dry run does not list them. The second endpoint and its share of each shared day count as already present. - - The copy adds counts only for names that differ in case and trailing spaces. A key column given a collation that also ignores, for example, accents still makes one row of other names, and that row keeps one endpoint's count. The other endpoint's counts are lost. -- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** The copy subtracts the current value from `@expires`, so the rebuilt time is off by the difference. The message still ages out of SQL when RavenDB would have deleted it, because SQL adds the same current value back when it checks the row against the retention period. +- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row**, and a day both counted holds the sum. PostgreSQL keeps both. + - This is not reported as a merge: the copy logs no warning, and the dry run does not list it. + - A key column whose collation also ignores, for example, accents still makes one row, but keeps only one endpoint's counts. The other's counts are lost. +- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** It is off by the difference, but the message still ages out of SQL when RavenDB would have deleted it. - **Every retry-issued message gets a retry claim, even one whose claim RavenDB had already released.** While the claim exists, retrying the message from the failed messages list sends nothing. - **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. @@ -513,21 +485,21 @@ The dry run counts the subscription and endpoint settings merges before anything ### Rows removed after the copy -- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens.** ServiceControl's usual clean-up removes it, because that group has no failed messages in SQL until the background copy brings the archived and resolved ones. -- **Unresolved messages from version 6.18 or earlier can expire during the copy.** A message that failed again after being archived or resolved on 6.18 or earlier kept its old expiry, and upgrading does not remove it. One that RavenDB deletes before the copy reads it is not copied and not counted as a skip. +- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. +- **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. ### What the checks do not catch -- **Nothing counts RavenDB during the copy.** A read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read. -- **Rows never read on a Done category are reported, not fixed.** Verify prints them, but its exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before ServiceControl opens. The current reasoning: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. -- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Only tests on each reader cover those. -- **Expiry can hide rows that were never read.** In verify, a deleted row and a missed row cancel out, so a flagged number is a floor. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. -- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. -- **A crash during a batch that carries bodies leaves orphan bodies in the body store.** External bodies are written before the transaction opens, so a crash leaves bodies no row points at, never rows pointing at missing bodies. +- **Nothing counts RavenDB during the copy.** A read that ends early still balances and ends Done. `--migration-verify` finds it afterwards. +- **Rows never read on a Done category are reported, not fixed.** Verify's exit code ignores them, and no command can recopy a Done category. Keeping RavenDB is the remedy. +- **Verify uses the same reader as the copy**, so it cannot catch a reader that always misses the same rows, or a source pointed at the wrong database. +- **Expiry can hide rows that were never read**, because in verify a deleted row and a missed row cancel out. +- **A write that silently stores fewer rows still ends Done**, because already present is what is left after copied and skipped. +- **A crash during a batch with bodies leaves orphan bodies** in the body store, never rows pointing at missing bodies. ### Gaps nothing enforces -- **An ingestion worker started at the same moment as the first migration start can slip in before the copy writes its first checkpoint.** The ingestion gate cannot see a copy with no checkpoint row, so the worker ingests into a part-copied database. Only the instructions prevent this. +- **An ingestion worker started at the same moment as the first migration start can slip in before the first checkpoint exists**, and ingest into a part-copied database. Only the instructions prevent this. - **Reusing an old copy after going back to RavenDB cannot be detected**, because it looks exactly like a normal resume. The copy skips every category already finished and misses anything RavenDB received since. ## Glossary From 33b1b0f5abdc64a58f1c18f942449bfdc98e789f Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 08:41:10 +0800 Subject: [PATCH 25/28] Update migration overview to clarify custom ServicePulse UI for migration status --- docs/migration/ravendb-to-sql-migration-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 0bb31d6c1d..43c8f20f73 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -455,7 +455,7 @@ It says nothing about load on the source. - **The source needs no new settings.** It reads the instance's existing RavenDB settings. - **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. -- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** There is no migration custom check, because custom checks are being removed from ServiceControl. +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** There is no migration custom check instead migration status will be surfaced with a custom ServicePulse UI. - **There is no HTTP API, pause, resume or abort.** Stopping a copy takes a restart. ## Open decisions From 5f62c89cc8d9bff6bef031c7555cee6d44900883 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 08:51:16 +0800 Subject: [PATCH 26/28] Trigger CI From 731848e3482ddce0c5e4891083654c38a3aa2559 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Tue, 17 Nov 2026 23:06:43 +0800 Subject: [PATCH 27/28] Copy the last four required categories from RavenDB to SQL: pending integration events, custom checks, failed error imports with their bodies, and group comments. With them, the offline copy covers all sixteen required categories. --- .../ravendb-to-sql-migration-instructions.md | 2 +- .../ravendb-to-sql-migration-overview.md | 60 +++-- .../ravendb-to-sql-migration-system-design.md | 34 +-- .../CustomCheckMigrationTests.cs | 97 +++++++ .../FailedErrorImportMigrationTests.cs | 162 +++++++++++ .../GroupCommentsMigrationTests.cs | 51 ++++ .../LicensingEndpointsMigrationTests.cs | 2 +- .../LicensingThroughputMigrationTests.cs | 46 ++-- .../MigrationAcceptanceTest.cs | 53 ++++ .../PendingIntegrationEventsMigrationTests.cs | 37 +++ .../RetryResetMigrationTests.cs | 25 +- .../When_the_copy_merges_or_changes_rows.cs | 63 +++++ ...unresolved_failed_messages_are_migrated.cs | 9 +- .../PostgreSqlMigrationSqlDialect.cs | 14 +- .../SqlServerMigrationSqlDialect.cs | 52 ++-- .../DataMigration/CustomCheckRowMapper.cs | 21 ++ .../DataMigration/EFCoreMigrationTarget.cs | 29 +- .../FailedErrorImportRowMapper.cs | 25 ++ .../FailedMessageDocumentTranslation.cs | 3 +- .../FailedMessageDocumentTranslator.cs | 28 +- .../DataMigration/GroupCommentRowMapper.cs | 16 ++ .../IStoreKeyedCategoryWriter.cs | 3 +- .../PendingIntegrationEventRowMapper.cs | 19 ++ .../DataMigration/PreparedBatch.cs | 19 +- .../Writers/CustomChecksWriter.cs | 71 +++++ .../Writers/EndpointSettingsWriter.cs | 2 +- .../Writers/FailedErrorImportsWriter.cs | 146 ++++++++++ .../Writers/FailedMessagesWriter.cs | 85 ++++-- .../Writers/GroupCommentsWriter.cs | 79 ++++++ .../DataMigration/Writers/KeyMerges.cs | 27 +- .../Writers/LicensingEndpointsWriter.cs | 13 +- .../Writers/LicensingThroughputWriter.cs | 39 ++- .../Writers/PendingIntegrationEventsWriter.cs | 59 ++++ .../Writers/RetryOperationsWriter.cs | 44 +-- .../DataMigration/Writers/StoreKeyedInsert.cs | 66 +++++ .../Writers/SubscriptionsWriter.cs | 6 +- .../Infrastructure/IMigrationSqlDialect.cs | 8 +- .../DataMigration/FailedMessageStream.cs | 35 ++- .../DataMigration/RavenMigrationSource.cs | 6 +- .../Readers/CustomChecksReader.cs | 38 +++ .../Readers/FailedErrorImportsReader.cs | 30 +++ .../Readers/FailedMessagesReader.cs | 5 +- .../Readers/GroupCommentsReader.cs | 23 ++ .../Readers/LicensingThroughputReader.cs | 13 +- .../Readers/PendingIntegrationEventsReader.cs | 19 ++ .../Readers/RavenStorageCheckIds.cs | 22 ++ .../DataMigration/ThroughputDocuments.cs | 32 --- .../ExternalIntegrationRequestsDataStore.cs | 2 +- .../RavenCustomCheckDataStore.cs | 4 +- .../Recoverability/GroupsDataStore.cs | 4 +- .../MigrationValueLengthTests.cs | 82 ++++++ .../DataMigration/CustomCheckSourceTests.cs | 76 ++++++ .../FailedErrorImportSourceTests.cs | 48 ++++ .../DataMigration/GroupCommentsSourceTests.cs | 31 +++ .../LicensingThroughputSourceTests.cs | 8 +- .../PendingIntegrationEventsSourceTests.cs | 38 +++ .../RavenMigrationSourceTests.cs | 13 +- .../RavenStorageCheckIdsTests.cs | 15 ++ .../UnresolvedFailedMessageSourceTests.cs | 40 ++- .../EndpointSettingsKeyCollationTests.cs | 30 --- .../LicensingThroughputKeyCollationTests.cs | 138 ++++++++++ .../Migration/CustomCheckRowMapperTests.cs | 30 +++ .../CustomChecksMigrationTargetTests.cs | 142 ++++++++++ .../FailedErrorImportRowMapperTests.cs | 33 +++ .../FailedErrorImportsMigrationTargetTests.cs | 133 +++++++++ .../FailedMessageMigrationCoverageTests.cs | 7 +- .../FailedMessagesMigrationTargetTests.cs | 252 +++++++++++++++++- .../Migration/GroupCommentRowMapperTests.cs | 22 ++ .../GroupCommentsMigrationTargetTests.cs | 115 ++++++++ .../LicensingEndpointsMigrationTargetTests.cs | 84 ++++++ ...LicensingThroughputMigrationTargetTests.cs | 124 ++++++++- .../Migration/MigrationTargetCoverageTests.cs | 36 ++- .../PendingIntegrationEventRowMapperTests.cs | 44 +++ ...ngIntegrationEventsMigrationTargetTests.cs | 120 +++++++++ .../EFCore/Migration/PreparedSkips.cs | 11 + .../RetryOperationsMigrationTargetTests.cs | 41 +++ .../SubscriptionsMigrationTargetTests.cs | 39 +++ .../EFCore/Migration/TargetWarnings.cs | 49 ++++ .../DataMigration/HaltThreshold.cs | 11 +- .../DataMigration/IMigrationTarget.cs | 4 +- .../DataMigration/MigrationCategoryIds.cs | 1 + .../MigrationCategoryRegistry.cs | 1 + .../MigrationCategoryStateExtensions.cs | 12 +- .../DataMigration/MigrationEngine.cs | 12 +- .../DataMigration/MigrationMetadataKeys.cs | 18 ++ .../DataMigration/MigrationSkipReason.cs | 6 + ...ategory_runs_in_a_fixed_order.approved.txt | 1 + .../Migration/CustomCheckStatusTests.cs | 14 + .../MigrationCategoryRegistryTests.cs | 8 +- .../MigrationCategoryStateExtensionsTests.cs | 41 ++- .../MigrationEngineCategorySelectionTests.cs | 2 +- .../Migration/MigrationEngineCopyTests.cs | 40 +++ .../Migration/RequiredCopyGateTests.cs | 139 +++++++++- .../Migration/MigrationStartup.cs | 47 +++- 94 files changed, 3545 insertions(+), 361 deletions(-) create mode 100644 src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs create mode 100644 src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs create mode 100644 src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index dbcf16c25e..27e456c9c0 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -190,7 +190,7 @@ Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes ## Step 8: Finish the migration 1. Stop ServiceControl. -2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. +2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped rows. Merged and changed rows are not shown here. Each one is logged as a warning by the copy run that merged or changed it, and that run's summary counts them. It exits with 0 only when every category is Done or Abandoned. 3. Start ServiceControl. Finished checkpoints remain in SQL, but no background copy runs and RavenDB is not opened. Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 43c8f20f73..36c4d99b4d 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -155,8 +155,8 @@ There are eighteen *categories*: sixteen required and two optional. | Licensing report masks | Required | | | Uploaded licensed endpoint details file | Required | Nothing recomputes it. Losing it means downloading it again from the licence portal and uploading it again | | Subscriptions | Required | | -| Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been. RavenDB sends them in no particular order anyway, so no ordering is lost | -| Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts | +| Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been, except a `MessageFailed` event for a message archived or resolved before the move (see [Rows removed after the copy](#rows-removed-after-the-copy)). RavenDB sends them in no particular order anyway, so no ordering is lost | +| Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts. Results of the six checks the RavenDB persister runs about its own database, matched on id and category, are left out, as harmless skips | | Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | | Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. Some comments are removed after ServiceControl opens, see [Open decisions](#rows-removed-after-the-copy) | | Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | @@ -174,7 +174,7 @@ There are eighteen *categories*: sixteen required and two optional. **Written new by the copy rather than copied:** -- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). +- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts that RavenDB still held a claim for, so a retry by id sends or holds back a message just as it did before the move. ### Optional data windows @@ -186,8 +186,13 @@ There are eighteen *categories*: sixteen required and two optional. ### Data that changes shape -- **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. -- **Other values change or merge on the way into SQL.** Each is listed under [Open decisions](#values-that-change-or-merge-in-sql). +- **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. For historic retry operations and pending integration events, the log says so once per category, in an information line. +- **Other values change or merge on the way into SQL.** Each is listed under [Open decisions](#values-that-change-or-merge-in-sql), with the line the log shows for it. + - A row copied with a changed value logs a warning starting `Changed in category on the way into SQL:`, followed by what changed and what was kept. + - A row merged into another logs a warning starting `Merged in category into a row with the same key:`, naming both keys, the source row kept, or the row SQL already held, and the source row dropped. + - The category's summary at the end of the copy counts both, apart from the rows already present, for example `Subscriptions: 14 copied (0 changed), 0 already present, 1 merged, nothing skipped`. The summary is a warning when anything was changed or merged. The counts cover only the current run: a category resumed after a crash counts only the changes and merges of the run that finished it, and counts earlier merges as already present. + - The summary's already present count leaves out the merged rows. If the copy reports more merged rows than SQL found already present in total, the summary adds a warning naming both counts, because the copy's prediction of which keys SQL treats as equal then disagrees with SQL. + - The per-row lines are written after each batch commits. If the process stops between a batch's commit and its log lines, those lines are lost. A resumed copy does not repeat them, because it finds those rows already present. ### Skipped rows @@ -200,7 +205,7 @@ A skip is one of three kinds: - **Fault, a retry can fix it.** Something outside the data went wrong. Fix the cause and retry. - **Fault, no retry can fix it.** The RavenDB document cannot become a SQL row, so a retry skips it again. Only [abandoning](#retry-and-abandon) gets past it. -- **Harmless.** A row SQL would have removed anyway. +- **Harmless.** A row SQL would have removed anyway, or a result of a check the source's own storage runs about itself (the RavenDB source flags its own checks, and the target skips flagged rows). A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Open decisions](#values-that-change-or-merge-in-sql). @@ -228,6 +233,7 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied | --- | | A blank group comment, which SQL never stores | | An archived or resolved message, or event log item, already past its retention period | +| A result of one of the six checks the RavenDB persister runs about its own database, such as index lag or free disk space, matched on the check's id and category. Only a RavenDB primary instance reports these checks, so nothing updates them after the move. A check that reuses one of these ids under another category is copied, such as a RavenDB audit instance's dirty memory check | - **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. - **A malformed document does not stop the copy.** It is skipped and counted, and the rest of the category carries on. If skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then gives up every unread row too. Raise the threshold or fix the cause, and retry. @@ -283,11 +289,11 @@ flowchart LR ### Writing to SQL - **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). -- **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. +- **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. A failure group with no id is the exception, see [Open decisions](#values-that-change-or-merge-in-sql). - **`UniqueMessageId` keeps its value** but becomes a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry key and the body key. - **`StatusChangedAt` is rebuilt.** For archived and resolved messages it is `@expires` minus the current `ServiceControl/ErrorRetentionPeriod` (wrong if that changed, see [Open decisions](#values-that-change-or-merge-in-sql)). For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL`. - **Bodies go through `IBodyStoragePersistence`**, which handles compression and the body store. The copier keeps a body inline when it is within `ServiceControl/MaxBodySizeToStore` (102,400 bytes by default), as ingestion does. -- **Throughput rows are written directly, and each day's count is set, not added**, so a resumed copy cannot double-count. The copied rows also stop the collectors gathering the same days again. +- **Throughput rows are written directly, and each day's count is added to the stored count**, as ServiceControl's own throughput write adds it. Endpoints whose names the SQL key treats as one have their counts for a day summed into one row: SQL Server compares names under the key column's own collation, and PostgreSQL compares them as its deterministic collations do, character for character. A resumed copy cannot double-count, because each batch and its checkpoint commit together. The copied rows also stop the collectors gathering the same days again. ### Batch size and throttling @@ -366,7 +372,7 @@ Stop ServiceControl before either command. In a container, run it as a one-off ` **What a retry does:** - **It re-reads the whole category from the start**, leaving rows already in SQL alone. Every body is read again, so a retry takes about as long as the first copy. -- **Historic retry operations and pending integration events** get new keys on every insert, so a retry first deletes what the category copied, in the same save as the reset. This is safe because both are required, so the target started empty. +- **Historic retry operations, pending integration events and throughput history** are deleted first. The first two get new keys on every insert, and throughput history adds each day's count to the stored one, so a re-read would copy them a second time. The retry deletes what the category copied, in the same save as the reset. This is safe because all three are required, so the target started empty and ServiceControl has not opened on it. Once ServiceControl has opened on the database, a retry of any of them is refused and deletes nothing. - **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. **To abandon:** @@ -464,29 +470,35 @@ Each case below changes, merges or drops data, or lets a loss pass without faili ### Values that change or merge in SQL -- **Attempt history collapses to the newest attempt.** SQL keeps the newest attempt and the attempt count, so only the newest attempt keeps its headers and exception. -- **Message redirects, subscriptions and pending retry operations merge when their keys match in SQL.** The copy keeps one row, counts the other as already present, and logs a warning naming both. +- **Attempt history collapses to the newest attempt.** SQL keeps the newest attempt and the attempt count, so only the newest attempt keeps its headers and exception. This is not logged and not counted as a change, because SQL keeps only the newest attempt for every message that failed more than once, by design. +- **A failure group with no id is dropped from its message, and a group with no title or type is stored with an empty one.** SQL keys a group by its id, so the message is missing from that group in ServicePulse, and a group retry or archive leaves it out. ServiceControl's classifiers always set an id, so this needs hand-made data. The log names each message, in a warning whose detail starts `Dropped the failure groups with no id` or `Stored an empty title or type for the failure groups that had none`. +- **Two failure groups of one message whose ids SQL treats as one key become one**, keeping the first group's title and type. ServiceControl never gives a message the same group twice. The log names each message, in a warning whose detail starts `Kept the failure group '' titled '' and dropped`. +- **Message redirects, subscriptions, pending retry operations, group comments and failed error imports merge when their keys match in SQL.** The copy keeps one row, counts the other as already present, and logs a warning starting `Merged <source id> in category <category> into a row with the same key:` naming both. On SQL Server the copy predicts these merges from the key column's collation settings for case, accents, kana type and width, plus SQL Server's trailing-space rule, and reports the merges those settings make. It compares text with .NET's culture rules rather than SQL Server's own tables, so an unusual character the two treat differently can merge without a line. - Redirects merge on source addresses the key column treats as equal. On SQL Server, by default, that includes addresses that differ only in case. - - Subscriptions also merge when they differ only in message-type version, because the SQL key leaves the version out. - - Pending retry operations with the same retry type and request id merge, keeping the latest. -- **On SQL Server, endpoint settings whose names differ only in case merge into one row**, because the name column compares without case by default. PostgreSQL keeps both. + - Subscriptions also merge when they differ only in message-type version, because the SQL key leaves the version out. The warning names the version and endpoint of the row kept and of the row dropped. + - Pending retry operations with the same retry type and request id merge, keeping the latest. The warning names the source row kept and the one dropped, and each source id ends with that operation's completion time. + - On SQL Server, group comments merge when their group ids differ only in trailing spaces. ServiceControl makes every group id from a GUID, so only a comment written on a hand-made group id can merge. PostgreSQL keeps both. + - Failed error imports of the same message merge. Version 6.18 and earlier stored each failed import attempt as its own document, and SQL keeps one row per message. The copy keeps the first it reads, and for those documents RavenDB order is effectively random, so it can keep an older attempt's exception. The warning names both documents, or the stored row's failure time and this one's. +- **On SQL Server, endpoint settings whose names differ only in case, kana type or width merge into one row**, because the name column's default collation ignores all three. A column whose collation also ignores accents merges those too. PostgreSQL keeps both. - The copy keeps the setting whose name exactly matches a known endpoint, because ServiceControl deletes the other. Otherwise it keeps the first in RavenDB order, which is effectively arbitrary. - The choice is made within one batch of up to a thousand. Across batches, the first one met wins. - - The copy logs a warning, and the dry run lists each pair. -- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. + - The copy logs a warning starting `Merged EndpointSettings/<name>` that names both names and the one kept, and the dry run lists each pair. +- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. The log names each message, in a warning whose detail starts `Cut the message type from <length> characters to its last 450`. - **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row**, and a day both counted holds the sum. PostgreSQL keeps both. - - This is not reported as a merge: the copy logs no warning, and the dry run does not list it. - - A key column whose collation also ignores, for example, accents still makes one row, but keeps only one endpoint's counts. The other's counts are lost. -- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** It is off by the difference, but the message still ages out of SQL when RavenDB would have deleted it. -- **Every retry-issued message gets a retry claim, even one whose claim RavenDB had already released.** While the claim exists, retrying the message from the failed messages list sends nothing. -- **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. + - The copy keeps the endpoint it meets first, counts the other as already present and logs a warning starting `Merged <source id> in category LicensingEndpoints` naming both. Within one batch the endpoint met first is the one whose source id sorts first, and an endpoint SQL already holds is met before any in the batch. The dry run does not list it. + - A key column whose collation also ignores, for example, accents, kana type or width makes one row in the same way, a day holds the sum of every endpoint the column treats as one, and the copy logs the same warning for it. +- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** It is off by the difference, but the message still ages out of SQL when RavenDB would have deleted it. Archived and resolved messages are not copied yet, so no log line flags these. +- **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. No log line lists these messages. +- **A failed message's sending and receiving endpoints are rebuilt from its headers** when RavenDB hands back the stored endpoint without its type, following ingestion's own rules on the same headers. A message whose headers name no endpoint gets none. No log line names these messages. +- **A failed error import's failure time is when RavenDB last modified its document**, because RavenDB stores no failure time for an import. ServiceControl never changes an import after storing it, so this is when the import was stored. No log line names imports for this. A missing exception or body is stored empty, and the log names each such import, in a warning whose detail starts `Stored an empty exception`, `Stored an empty body` or `Stored an empty exception and an empty body`. The dry run counts the subscription and endpoint settings merges before anything moves. ### Rows removed after the copy -- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. -- **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. +- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. The same happens when the group's messages are gone from RavenDB or were skipped by the copy. The copy names each such comment, in a warning starting `Changed GroupComment/<group id> in category GroupComments on the way into SQL: The group '<group id>' has no copied failed message`, which also quotes the whole comment. +- **A `MessageFailed` event still waiting to be sent for a message archived or resolved before the move is never sent.** ServiceControl looks the message up when it sends the event. Archived and resolved messages arrive only with the background copy, so the lookup finds nothing, and the event is deleted with no log line. RavenDB would have sent it. +- **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. No log line names these messages yet. ### What the checks do not catch @@ -518,7 +530,7 @@ The dry run counts the subscription and endpoint settings merges before anything - **Abandoned:** you gave up on what a category had not copied, or on one it depends on. Final. - **Already present:** a row the copier found in SQL already, so it left it alone. - **Fault skip:** a row lost because something went wrong. -- **Harmless skip:** a row left behind because SQL would have removed it anyway. +- **Harmless skip:** a row left behind because SQL would have removed it anyway, or because it is a result of a check the source's own storage runs about itself (the RavenDB source flags its own checks, and the target skips flagged rows). - **Free abort:** going back to RavenDB at no cost, possible only while ServiceControl is still closed. - **Dry run:** a read-only rehearsal that reports what would move, what would be skipped and how long the outage would be. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index f35991fdc7..d066366a1b 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -58,7 +58,7 @@ Dependencies point inward: each persister knows only its own store, the neutral Supporting assemblies: -- `SqlServerMigrationSqlDialect` and `PostgreSqlMigrationSqlDialect` implement `IMigrationSqlDialect` and are the only provider-specific migration code. SQL Server reads key-column collations in `Open` and inserts with `MERGE ... WITH (HOLDLOCK)`, de-duplicating the batch with `ROW_NUMBER() OVER (PARTITION BY <key> COLLATE <column collation>)`. PostgreSQL uses `INSERT ... ON CONFLICT DO NOTHING RETURNING`. Both implement `SetLicensingEndpointThroughput`. Each provider assembly carries its own `AddMigrationCheckpoints` EF migration and model snapshot, so a checkpoint schema change needs a migration in both. +- `SqlServerMigrationSqlDialect` and `PostgreSqlMigrationSqlDialect` implement `IMigrationSqlDialect` and are the only provider-specific migration code. SQL Server reads key-column collations in `Open`, builds each key column's `KeyComparer` from the collation's comparison style (case, accent, kana type and width, plus SQL Server's trailing-space padding), and inserts with `MERGE ... WITH (HOLDLOCK)`, de-duplicating the batch with `ROW_NUMBER() OVER (PARTITION BY <key> COLLATE <column collation>)`. PostgreSQL uses `INSERT ... ON CONFLICT DO NOTHING RETURNING`. Both implement `AddLicensingEndpointThroughput`, which groups each statement's rows by key and sums their counts before adding them to the stored day, so the database decides which names are one day. SQL Server groups under the key column's collation; PostgreSQL's deterministic collations make a plain `GROUP BY` the same equality. Each provider assembly carries its own `AddMigrationCheckpoints` EF migration and model snapshot, so a checkpoint schema change needs a migration in both. - `ServiceControl.DomainEvents` holds `MigrationStarted`, `MigrationCategoryHalted`, `MigrationFinished` and `MigrationSourceUnreachable`. The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadiness` or `IMigrationCheckpointStore`, so the guards below resolve `null` on RavenDB and return. `MigrationPairIsSupportedCheck` is what rejects RavenDB as a target. @@ -144,8 +144,8 @@ Each position in the sequence is deliberate: 9. **Checkpoint initialization.** The command seeds `NotStarted` rows for required and selected optional categories before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. 10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. 11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. -12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row as harmless. -13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. +12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row with the explanation `HarmlessSkipExplanations` holds for each harmless reason. A test fails unless that map covers exactly the `IsBenign` reasons. +13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`, and harmless reasons flagged `, harmless` through `IsBenign`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. 14. **`RecordHostOpenedOnTarget`** is registered on normal startup, never on the offline command. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. ## The background copy @@ -177,18 +177,19 @@ Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader | Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; the two licensing endpoint readers use `ThroughputDocuments.Endpoints`; single-document readers load by id | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | | Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyMissing` after one attempt, a permanent fault no retry fixes, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | | Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold. A crash leaves orphan bodies, see [open decisions](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch) | -| Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `SetLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips and merges are logged after commit, and `Extend` adds the batch's `PreparedBatch.Merges` to `MergedCount`. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | +| Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `AddLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips, merges and changes are logged after commit. `MigrationWriteResult.Merged` and `Changed` carry the batch's counts of each, which the engine adds up per category for the current run in `MigrationEngine.MergedAndChangedThisRun`; the checkpoint stores neither. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | | Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | | Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | | Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | -Body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. +Failed message body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. `FailedErrorImportsWriter` places a failed error import's body by the SQL import store's own rule, inline up to `MaxBodySizeToStore` and otherwise external only, under `FailedErrorImportEntity.ExternalBodyId`, also before the transaction opens. Invariants enforced across class boundaries: - `MigrationCheckpoint.Extend` throws unless the per-`MigrationSkipReason` counts sum exactly to the skipped count. `EFCoreMigrationTarget.AccountForEveryRow` throws unless prepared rows plus skips equal the batch size, catching rows dropped during preparation. `AlreadyPresentIn` throws when a writer over-reports. Every row is copied, skipped or already present. -- `MigrationSkipReasonExtensions.IsBenign` is fixed: `PastRetention`, `BlankGroupComment`. Everything else is a fault, `Unknown` included. -- `PreparedBatch.Merges` reports rows whose key collides with another under the target's collation. `EndpointSettingsWriter` detects them with `IMigrationSqlDialect.KeyComparer` against the batch and the existing table, and `EFCoreMigrationTarget` logs one warning per merge naming both keys and the one kept. The dropped row counts as already present, and also in `MergedCount`, the part of already present that merges explain. Every writer whose key can fold two source rows into one reports them here, or `MergedCount` understates. +- `MigrationSkipReasonExtensions.IsBenign` is fixed: `PastRetention`, `BlankGroupComment`, `SourceStorageCheck`. Everything else is a fault, `Unknown` included. +- `PreparedBatch.Merges` reports rows whose key collides with another under the target's collation. Writers with a text key detect them with `KeyMerges.In` or `KeyMerges.Split`, which compare with `IMigrationSqlDialect.KeyComparer` against the batch and the existing table. `FailedErrorImportsWriter`, whose key is a GUID, reports a second import of the same message in the batch, and one the table holds with another failure time. `EFCoreMigrationTarget` logs one warning per merge naming both keys, the source row kept, or the stored row, and the source row dropped. The dropped row counts as already present, and also in `MigrationWriteResult.Merged`, so the category summary `MigrationStartup.ReportWhatTheCopyLeftBehind` logs shows merges apart from the rest of already present. Every writer whose key can fold two source rows into one reports them here, or the summary understates merges. +- `PreparedBatch.Changes` reports rows the insert adds whose stored value differs from the source's on purpose: a failed message's cut message type, dropped or blank failure groups, and failure groups one key folds together; a failed error import's missing exception or body stored empty; and a group comment on a group with no copied failed message, which the retention sweep deletes. A failed message keeping only its newest processing attempt is not a change, because every row does that by design. Writers report only rows the target does not already hold, so a re-read reports nothing twice. `EFCoreMigrationTarget` logs one warning per change after the commit, and `MigrationWriteResult.Changed` counts them. A process that stops between the commit and the log loses that batch's lines for good. `MigrationStartup.ReportWhatTheCopyLeftBehind` warns, naming both counts, when a category's merged count exceeds its already-present count, because the key comparer then disagrees with the database. `PreparedBatch.RenumberedRows` names rows the database gives new ids, and the target logs it once per copy of a category, on the batch whose checkpoint has copied nothing yet. - `TargetCountAtSettle` is read with `IMigrationTarget.Count` just before the save that settles a category at the end of the source, or that abandons it, and is never updated afterwards. For a required category it is exactly what the copy left in SQL: the host is closed, `TargetHoldsNoServiceControlDataCheck` started the target empty, and a second copier would move the row's version so the settle save is refused. That is why the count needs no shared transaction with the save. For an optional category it also includes what ServiceControl wrote while the copy ran. ## Category states @@ -224,7 +225,7 @@ Settlement at end of source, in order: - `CanBeRetried`: an `IsFailed` row, or an optional `NotStarted` or `InProgress` row with `LastError` (which covers `RecordSourceOutage` rows). Never `Blocked` or a missing row. `MigrationOperatorCommands.Retry` rejects `EventLog` before consulting it. - `CanBeAbandoned`: a required row that `IsFailed` or is `InProgress` with copied plus skipped plus already-present above 0, treating an id unknown to the registry as required; an optional category in any state except `Complete` or `Abandoned`, including no row. -Checkpoint schema: one row per category in `MigrationCheckpoints` (`migration_checkpoints` on PostgreSQL), primary key `CategoryId`. Columns: `State`, `Cursor`, `CopiedCount`, `SkippedCount`, `AlreadyPresentCount`, `MergedCount`, `SkipReasons` (JSON keyed by enum name; unknown names deserialize to `Unknown`, a fault), `StartedAt`, `LastProgressAt`, `SettledAt`, `LastError`, `StartedWindowSeconds`, `TargetCountAtSettle` (null until the category first settles) and `Version`, the optimistic concurrency token. No attempt counter and no decision record. `AddMigrationCheckpoints` creates the table in both providers and has not been released, so `TargetCountAtSettle` replaces the never-written `SourceTotal` column inside it, and `MergedCount` is added there too, rather than in a second migration. Run-level facts are `Settings` rows under `Migration/`, such as `Migration/HostOpenedOnTarget`. +Checkpoint schema: one row per category in `MigrationCheckpoints` (`migration_checkpoints` on PostgreSQL), primary key `CategoryId`. Columns: `State`, `Cursor`, `CopiedCount`, `SkippedCount`, `AlreadyPresentCount`, `SkipReasons` (JSON keyed by enum name; unknown names deserialize to `Unknown`, a fault), `StartedAt`, `LastProgressAt`, `SettledAt`, `LastError`, `StartedWindowSeconds`, `TargetCountAtSettle` (null until the category first settles) and `Version`, the optimistic concurrency token. No attempt counter and no decision record. `AddMigrationCheckpoints` creates the table in both providers and has not been released, so `TargetCountAtSettle` replaces the never-written `SourceTotal` column inside it rather than in a second migration. This build does not store merged or changed counts: `MigrationEngine.MergedAndChangedThisRun` holds them for the current run only, so the summary of a resumed category counts only that run's. Phase 13 adds them to the checkpoint so that verify can show them. Run-level facts are `Settings` rows under `Migration/`, such as `Migration/HostOpenedOnTarget`. ## Commands @@ -240,12 +241,13 @@ Every migration command except the source report builds a host with `AddPersiste | `--migration-retry <category>` | `MigrationRetryCommand`, `MigrationOperatorCommands` | No | `Retry` checks `CanBeRetried` and builds the reset row; `IMigrationTarget.ResetForRetry` persists it | | `--migration-abandon <category>` | `MigrationAbandonCommand`, `MigrationOperatorCommands` | No | Checks `CanBeAbandoned`, then writes `Abandoned` rows through the checkpoint store only | -`MigrationOperatorCommands.Retry` builds the reset row without I/O: state `NotStarted`; `Cursor`, the four counts, `SkipReasons`, `SettledAt`, `TargetCountAtSettle` and `LastError` cleared; `StartedAt` and `StartedWindowSeconds` kept. Change retry semantics there. `ResetForRetry` only persists the row it receives, in one transaction under the execution strategy. Store-keyed categories cannot be re-read idempotently, so their writers implement `IStoreKeyedCategoryWriter.DeleteCopiedRows` and the reset deletes their rows in the same transaction: +`MigrationOperatorCommands.Retry` builds the reset row without I/O: state `NotStarted`; `Cursor`, the four counts, `SkipReasons`, `SettledAt`, `TargetCountAtSettle` and `LastError` cleared; `StartedAt` and `StartedWindowSeconds` kept. Change retry semantics there. `ResetForRetry` only persists the row it receives, in one transaction under the execution strategy. A category whose rows a re-read cannot recognise as already copied cannot be re-read idempotently: store-keyed rows come back under new keys, and throughput counts are added again. Its writer implements `IStoreKeyedCategoryWriter.DeleteCopiedRows` and the reset deletes its rows in the same transaction: - `RetryOperationsWriter`: `HistoricRetryOperations`, then `UnacknowledgedRetryOperations`. - `PendingIntegrationEventsWriter`: `ExternalIntegrationDispatchRequests`. +- `LicensingThroughputWriter`: `LicensingEndpointThroughput`. -Both are required categories, so the host has not opened and the target started empty. `EventLog` is also store-keyed but copies after the host opens, when its rows are indistinguishable from live ones, so it is abandon-only. +All three are required categories, and `CanBeRetried` takes a required category only when it is Failed, which keeps the host closed, so the host has not opened and the target started empty. `ResetForRetry` also checks this itself: inside the reset transaction it reads the `Migration/HostOpenedOnTarget` setting, and when it is set it refuses a reset that would delete, naming the category, and deletes and saves nothing. A reset that deletes nothing, such as an optional category's, is not refused. `EventLog` is also store-keyed but copies after the host opens, when its rows are indistinguishable from live ones, so it is abandon-only. `--migration-abandon` first re-saves the named row unchanged, when it exists, to bump its version and fence off a concurrent copy. It then abandons dependants along `MigrationCategory.MustFollowIsDataLink` (`EndpointSettings` after `KnownEndpoints`, `LicensingThroughput` after `LicensingEndpoints`), deepest first, skipping `Complete` and `Abandoned` dependants and inserting `Abandoned` rows for dependants without one, and saves the named row last. `GroupComments` follows the unresolved failed messages for ordering only and is not cascaded. An optional category with no row gets a new `Abandoned` row. Every row it saves `Abandoned` carries `TargetCountAtSettle`, read just before the save, as a settle does. No domain event is raised, because ServiceControl is stopped. @@ -257,7 +259,7 @@ Both operator commands rely on `MigrationCheckpointConflictException` to detect - **RavenDB lower than the sum** is rows RavenDB removed after the copy read them, normal for the event log, the archive and a few unresolved messages with a stale expiry. Printed, not judged. - **Expiry can hide the same number of unread rows** in the categories it touches, so a flagged number is a floor. - **The count streams the same reader the copy used** (`RavenMigrationSource.Count`), so verify finds rows one run missed, such as a resume that skipped a range or a stream that ended early. It cannot find a reader that always misses the same rows, or a source pointed at the wrong database. -- **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `MergedCount` and `TargetCountAtSettle` are printed beside the sum instead. +- **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `TargetCountAtSettle` is printed beside the sum instead. Merged rows are printed there only once phase 13 stores their count on the checkpoint; until then they appear only in the log of the run that made them. - **An abandoned category's sum covers its last pass only.** Rows an earlier pass copied before a retry are in `TargetCountAtSettle` but not in the sum, so RavenDB minus the sum overstates what was never copied. For rows never read on a Done category, verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, and neither operator command takes a Done row. Whether to fix them is an [open decision](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch). @@ -277,15 +279,15 @@ Every reader derives from `MigrationCategoryReader<TDocument>` and every writer | 5 | `NotificationSettings` | Required | `NotificationSettingsReader`, `NotificationSettingsWriter` | One `Settings` row via `SettingRowMapper` | | 6 | `TrialEndDate` | Required | `TrialEndDateReader`, `TrialEndDateWriter` | One `Settings` row via `SettingRowMapper` | | 7 | `RetryOperations` | Required | `RetryOperationsReader`, `RetryOperationsWriter` | Two tables: historic rows store-keyed via `StoreKeyedInsert`, unacknowledged rows via `InsertMissing`, which sizes the batch. Deletes its rows on retry | -| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database. Reads only the `EndpointDocuments` collection, through `ThroughputDocuments.Endpoint`, because the throughput database gives its documents no id prefix and also holds the settings documents. With no prefix RavenDB streams in reverse etag order and ignores `startAfter`, so `ThroughputDocuments.Endpoints` reads every endpoint, sorts the ids in ordinal order and skips past the cursor itself | -| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. The reader gives one batch per group from `ThroughputDocuments.EndpointGroups` and ignores the batch size, because one endpoint can hold a year or more of days. A group is the endpoints of one source whose names match ignoring case and trailing spaces, and its cursor is its last id in ordinal order, so a resume never splits it. The writer adds the counts of a group's endpoints whose names the key column treats as the same. `SetLicensingEndpointThroughput` splits the batch into statements and sets each day's count rather than incrementing, so re-reads are idempotent | +| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database. Reads only the `EndpointDocuments` collection, through `ThroughputDocuments.Endpoint`, because the throughput database gives its documents no id prefix and also holds the settings documents. With no prefix RavenDB streams in reverse etag order and ignores `startAfter`, so `ThroughputDocuments.Endpoints` reads every endpoint, sorts the ids in ordinal order and skips past the cursor itself. Reports through `KeyMerges.In` each endpoint folded into another's key. `LicensingThroughputWriter` reports no merge for the days, because the fold is reported here | +| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. The reader gives one batch per endpoint from `ThroughputDocuments.Endpoints` with a batch size of one, and ignores the requested batch size, because one endpoint can hold a year or more of days. Each batch's cursor is its endpoint's id. The writer keeps the last row of a day repeated for one source endpoint, compared ordinally. `AddLicensingEndpointThroughput` splits the batch into statements and adds each day's count to the stored one, summing endpoints whose names the key column treats as the same, in one batch or across batches. A re-read is not idempotent, so a resume relies on each batch committing with its checkpoint, and a retry deletes the copied rows first | | 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | | 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | -| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator` | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator`. `FailedMessageStream.ResolveRetryClaims` looks up each batch's RavenDB retry claims in one request and gives every retry-issued row the `RetryClaimHeld` metadata key. The writer writes a claim only for a retry-issued message it inserts whose flag is true, so the claims mirror RavenDB's, and fails the batch when the flag is missing or not a bool | | 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | -| 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | | +| 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | The RavenDB reader flags a check whose id and category match one of the RavenDB persister's own six checks with the `SourceStorageCheck` metadata key, and the writer skips a flagged row as `SourceStorageCheck` without knowing any id. A RavenDB audit instance's dirty memory result reports under another category, so it is copied | | 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | -| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment` | +| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment`. Reports group ids that differ only in trailing spaces as merges on SQL Server. Reports each copied comment on a group with no copied failed message as a change, because the retention sweep deletes it | | 1 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | | 2 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | diff --git a/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs new file mode 100644 index 0000000000..920c6e2cce --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs @@ -0,0 +1,97 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Infrastructure; + +[TestFixture] +[NonParallelizable] +class CustomCheckMigrationTests : MigrationAcceptanceTest +{ + static readonly EndpointDetails Endpoint = new() { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + + static Guid IdOf(EndpointDetails endpoint, string customCheckId) => + new CustomCheckDetail { OriginatingEndpoint = endpoint, CustomCheckId = customCheckId, Category = "Ping" }.GetDeterministicId(); + + async Task<IList<CustomCheckView>> CopiedChecks() => (await CustomChecks.GetStats(new PagingInfo())).Results!; + + // One of each status, because Status.Fail is the enum's zero and a status that never deserializes would read as Fail. + [Test] + public async Task Copies_a_failing_and_a_passing_custom_check_with_their_deterministic_ids_and_reported_status() + { + await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await SeedSourceCustomCheck(Endpoint, "Sales.DiskCheck", "Disk", failed: false); + + var result = await CopyCategory("CustomChecks"); + + var copied = await CopiedChecks(); + var failing = copied.Single(check => check.CustomCheckId == "Sales.HealthCheck"); + var passing = copied.Single(check => check.CustomCheckId == "Sales.DiskCheck"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!), Is.EqualTo(2), "Count is what --migration-verify reads"); + Assert.That(Guid.Parse(failing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.HealthCheck")), "a live report of the same check upserts on this id, so it replaces the copied row rather than adding a second"); + Assert.That(failing.Status, Is.EqualTo(Status.Fail)); + Assert.That(failing.Category, Is.EqualTo("Ping")); + Assert.That(failing.FailureReason, Is.EqualTo("timed out")); + Assert.That(failing.ReportedAt, Is.EqualTo(SeededReportedAt)); + Assert.That((failing.OriginatingEndpoint!.Name, failing.OriginatingEndpoint.Host, failing.OriginatingEndpoint.HostId), Is.EqualTo((Endpoint.Name, Endpoint.Host, Endpoint.HostId))); + Assert.That(Guid.Parse(passing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.DiskCheck"))); + Assert.That(passing.Status, Is.EqualTo(Status.Pass), "a status read back as the enum's zero would show every copied check red"); + Assert.That(passing.Category, Is.EqualTo("Disk")); + Assert.That(passing.FailureReason, Is.Null); + } + } + + [Test] + public async Task A_check_with_no_originating_endpoint_is_skipped_as_a_required_value_missing() + { + await SeedSource(($"CustomChecks/{Guid.NewGuid()}", new { CustomCheckId = "Sales.HealthCheck", Category = "Ping", Status = (int)Status.Pass, ReportedAt = DateTime.UtcNow })); + + var result = await CopyCategory("CustomChecks"); + var checkpoint = await ReadCheckpoint("CustomChecks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + Assert.That(checkpoint.CopiedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons?.GetValueOrDefault(MigrationSkipReason.RequiredValueMissing), Is.EqualTo(1), "CustomChecks.OriginatingEndpointName is NOT NULL, and a throw would stop the batch rather than count the row"); + } + } + + [Test] + public async Task Only_a_check_whose_id_and_category_match_a_RavenDB_persister_check_is_skipped_as_harmless_and_the_category_ends_done() + { + var serviceControl = new EndpointDetails { Name = "Particular.ServiceControl", Host = "MACHINE", HostId = Guid.NewGuid() }; + var audit = new EndpointDetails { Name = "Particular.ServiceControl.Audit", Host = "MACHINE", HostId = Guid.NewGuid() }; + await SeedSourceCustomCheck(serviceControl, "Error Database Index Lag", "ServiceControl Health", failed: true, reason: "index lag"); + await SeedSourceCustomCheck(audit, "Audit Database Index Lag", "ServiceControl.Audit Health", failed: true, reason: "index lag"); + await SeedSourceCustomCheck(audit, "RavenDB dirty memory", "ServiceControl.Audit Health", failed: true, reason: "dirty memory"); + await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await SeedSourceCustomCheck(Endpoint, "Message Ingestion Process", "Sales", failed: true, reason: "stuck"); + + var result = await CopyCategory("CustomChecks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a harmless skip leaves a required category Done, so ServiceControl can open"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.SourceStorageCheck] = 1 }), "only the primary's index lag check is the RavenDB persister's own"); + Assert.That((await CopiedChecks()).Select(check => (check.CustomCheckId, check.Category)), Is.EquivalentTo(new[] + { + ("Audit Database Index Lag", "ServiceControl.Audit Health"), + ("RavenDB dirty memory", "ServiceControl.Audit Health"), + ("Sales.HealthCheck", "Ping"), + ("Message Ingestion Process", "Sales") + }), "the skip matches the id and the category, so the audit instance's checks and a customer check with a colliding id keep their failing results"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs new file mode 100644 index 0000000000..cef7c85604 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs @@ -0,0 +1,162 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[NonParallelizable] +class FailedErrorImportMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_a_small_body_inline_with_the_time_RavenDB_stored_the_import_as_its_failure_time() + { + await SeedSourceFailedErrorImport("native-1", [1, 2, 3], "Deserialization failed"); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + var import = await GetImport("native-1"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await Target.Count(Category), Is.EqualTo(1), "Count is what --migration-verify reads"); + Assert.That(import.Body, Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That(import.BodyStoredExternally, Is.False); + Assert.That(import.ExceptionInfo, Is.EqualTo("Deserialization failed")); + Assert.That(import.FailedAt, Is.EqualTo(await SourceLastModified("native-1")).Within(TimeSpan.FromMilliseconds(1)), "replay runs oldest first by this time, so the time of the copy would reorder a customer's stuck imports"); + Assert.That(RecordedBodies.Written, Is.Empty); + } + } + + [Test] + public async Task Migrated_bodies_replay_through_the_products_own_import_path() + { + byte[] small = [1, 2, 3]; + var large = LargeBody(); + await SeedSourceFailedErrorImport("native-3", small, "Deserialization failed"); + await SeedSourceFailedErrorImport("native-4", large, "Deserialization failed"); + + await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + var external = await GetImport("native-4"); + Assert.That(external.BodyStoredExternally, Is.True, "otherwise this proves nothing about the external placement"); + Assert.That(external.Body, Is.Empty, "the product keeps no inline copy of a body it stores in body storage"); + + var replayed = new Dictionary<string, FailedTransportMessage>(); + await FailedImportStore.ProcessFailedErrorImports((message, _) => + { + replayed[message.Id] = message; + return Task.CompletedTask; + }, TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(replayed.GetValueOrDefault("native-3")?.Body, Is.EqualTo(small), "an inline body is read from the row"); + Assert.That(replayed.GetValueOrDefault("native-4")?.Body, Is.EqualTo(large), "an external body is read from body storage under the body id the SQL import store uses for that key"); + Assert.That(replayed.GetValueOrDefault("native-3")?.Headers, Is.EqualTo(HeadersFor("native-3")), "the replay ingests the message under the headers the copy stored"); + Assert.That(replayed.GetValueOrDefault("native-4")?.Headers, Is.EqualTo(HeadersFor("native-4"))); + } + } + + [Test] + public async Task A_retry_finds_every_import_already_present_and_writes_no_body_a_second_time() + { + await SeedSourceFailedErrorImport("native-2", [9], "Deserialization failed"); + await SeedSourceFailedErrorImport("native-7", LargeBody(), "Deserialization failed"); + await CopyCategory(MigrationCategoryIds.FailedErrorImports); + var bodiesWritten = RecordedBodies.Written.Count; + + await Target.ResetForRetry(RetryResetMigrationTests.ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports))); + var second = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(bodiesWritten, Is.EqualTo(1), "the large body is the only one the first copy writes to body storage"); + Assert.That(second.AlreadyPresent, Is.EqualTo(2), "--migration-retry re-reads from the start, and the rows the first copy wrote are left alone"); + Assert.That(await Target.Count(Category), Is.EqualTo(2)); + Assert.That(RecordedBodies.Written, Has.Count.EqualTo(bodiesWritten), "a body is written only for an import the target does not hold yet"); + } + } + + [Test] + public async Task Only_the_first_import_of_a_key_in_a_batch_has_its_body_written() + { + var first = LargeBody(); + var second = LargeBody(); + await SeedSource((DocumentId("native-8-a"), ImportOf("native-8", first)), (DocumentId("native-8-b"), ImportOf("native-8", second))); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "both documents derive one key, and the insert keeps the first row of a key"); + Assert.That(RecordedBodies.Written.Select(written => written.Body), Is.EqualTo(new[] { first }), "a second body under the same id would replace the body of the row the insert kept"); + } + } + + [Test] + public async Task An_import_with_no_message_is_skipped_as_a_required_value_missing() + { + await SeedSource((DocumentId("native-5"), new FailedErrorImport { Id = "native-5", Message = null, ExceptionInfo = "Deserialization failed" })); + await SeedSourceFailedErrorImport("native-6", [6], "Deserialization failed"); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + + var checkpoint = await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 }), + "FailedErrorImports.MessageId is NOT NULL; the source attaches an empty body, so nothing is skipped as BodyMissing or BodyUnreadable"); + } + } + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + static string DocumentId(string nativeMessageId) => $"FailedErrorImports/{nativeMessageId}"; + + static Dictionary<string, string> HeadersFor(string nativeMessageId) => new() { ["NServiceBus.MessageId"] = nativeMessageId }; + + // One byte over the limit, because the store moves a body out only when it is larger than the limit. + byte[] LargeBody() + { + var body = new byte[EFSettings.BodyStorage.MaxBodySizeToStore + 1]; + Random.Shared.NextBytes(body); + return body; + } + + Task SeedSourceFailedErrorImport(string nativeMessageId, byte[] body, string exceptionInfo) => + SeedSource((DocumentId(nativeMessageId), ImportOf(nativeMessageId, body, exceptionInfo))); + + static FailedErrorImport ImportOf(string nativeMessageId, byte[] body, string exceptionInfo = "Deserialization failed") => new() + { + Id = nativeMessageId, + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = HeadersFor(nativeMessageId), Body = body }, + ExceptionInfo = exceptionInfo + }; + + async Task<DateTime> SourceLastModified(string nativeMessageId) + { + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + + return session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedErrorImport>(DocumentId(nativeMessageId)))!.Value.ToUniversalTime(); + } + + Task<FailedErrorImportEntity> GetImport(string nativeMessageId) + { + var key = FailedErrorImport.DeriveKey(HeadersFor(nativeMessageId), nativeMessageId); + + return QueryTarget(dbContext => dbContext.FailedErrorImports.AsNoTracking().SingleAsync(row => row.UniqueMessageId == key)); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs new file mode 100644 index 0000000000..0d5d7751b2 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class GroupCommentsMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory GroupCommentsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)!; + + [Test] + public async Task Copies_each_comment_onto_its_group_and_an_orphaned_one_too_and_skips_a_null_or_blank_one_as_harmless() + { + var kept = new IngestedFailure(); + var cleared = new IngestedFailure(); + var neverWritten = new IngestedFailure(); + var spaces = new IngestedFailure(); + await SeedSourceWithBodies([.. new[] { kept, cleared, neverWritten, spaces }.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]); + await SeedSourceGroupComments( + (kept.Groups.Single().Id, "Waiting on the payments team"), + (cleared.Groups.Single().Id, ""), + (neverWritten.Groups.Single().Id, null), + (spaces.Groups.Single().Id, " "), + ("group-with-no-copied-message", "Raised with the shipping team")); + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var result = await CopyCategory(MigrationCategoryIds.GroupComments); + var comments = (await GroupsStore.GetUnresolvedGroupsByClassifier(kept.Groups.Single().Type, null, new PagingInfo())).ToDictionary(view => view.Id, view => view.Comment); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "SQL never stores a blank comment, so leaving one behind loses nothing and must not keep ServiceControl closed"); + Assert.That(result.Copied, Is.EqualTo(2), "a comment whose group has no copied message is copied too, and left to the retention sweeper"); + Assert.That(await Target.Count(GroupCommentsCategory), Is.EqualTo(2), "--migration-verify reads this count"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BlankGroupComment] = 3 }), "a harmless skip is still counted, so status and verify can show it"); + Assert.That(comments[kept.Groups.Single().Id], Is.EqualTo("Waiting on the payments team"), "ServicePulse's group list shows the comment on the group it was written for"); + Assert.That(new[] { cleared, neverWritten, spaces }.Select(failure => comments[failure.Groups.Single().Id]), Is.All.Null, "the product shows no comment for a group whose comment was blank"); + } + } + + // The prefix is retyped because this project cannot reference the RavenDB persister that declares it. + Task SeedSourceGroupComments(params (string GroupId, string Comment)[] comments) => + SeedSource([.. comments.Select(comment => ($"GroupComment/{comment.GroupId}", (object)new GroupComment { Id = $"GroupComment/{comment.GroupId}", Comment = comment.Comment }))]); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs index 614f04792d..6089eb2ea6 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs @@ -89,7 +89,7 @@ public async Task Two_names_differing_only_in_a_trailing_space_never_fail_the_ba using (Assert.EnterMultipleScope()) { Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space and keeps the first row; PostgreSQL's does not and keeps both"); - Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is already present, not lost"); + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is counted as already present"); Assert.That(result.Skipped, Is.Zero); Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs index a00b710498..b6f2048f0c 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs @@ -102,14 +102,15 @@ public async Task The_registry_order_blocks_the_counts_until_their_endpoints_are } [Test] - public async Task Running_the_category_twice_produces_the_same_count() + public async Task A_retry_copies_the_category_again_with_the_same_count() { await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); await CopyCategory("LicensingEndpoints"); await CopyCategory("LicensingThroughput"); - await DeleteCheckpoint("LicensingThroughput"); + await Target.ResetForRetry(RetryResetMigrationTests.ResetForRetry(await ReadCheckpoint("LicensingThroughput"))); + var afterReset = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()); var second = await CopyCategory("LicensingThroughput"); var total = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput @@ -117,33 +118,26 @@ public async Task Running_the_category_twice_produces_the_same_count() .Select(row => row.MessageCount) .SingleAsync()); - Assert.That(second.Copied, Is.EqualTo(1), "the second run wrote nothing, so the count below cannot tell setting from adding"); - Assert.That(total, Is.EqualTo(100), "running the category twice must set the day's count, not add to it"); + using (Assert.EnterMultipleScope()) + { + Assert.That(afterReset, Is.Zero, "the reset deletes the copied days before the category is read again"); + Assert.That(second.Copied, Is.EqualTo(1), "the second run writes the one day once"); + Assert.That(total, Is.EqualTo(100), "the reset deletes the copied days, so copying again does not add the count a second time"); + } } [Test] - public async Task Two_names_differing_only_in_a_trailing_space_add_their_counts_into_one_day_without_failing_the_batch() + public async Task Two_names_differing_only_in_a_trailing_space_add_their_counts_into_one_day_without_failing_the_copy() { - var endpoints = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); - var endpointBatch = new MigrationBatch( - [ - new MigrationRow("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary<string, object>()), - new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary<string, object>()) - ], - "Sales.Orders/Broker"); - await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); - - var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); var day = new DateOnly(2026, 1, 1); - var batch = new MigrationBatch( - [ - new MigrationRow("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100), new Dictionary<string, object>()), - new MigrationRow("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250), new Dictionary<string, object>()) - ], - "Sales.Orders/Broker"); - var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + await SeedThroughputSource(new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders /Broker", day, 100); + await SeedThroughputTimeSeries("Sales.Orders/Broker", day, 250); + await CopyCategory("LicensingEndpoints"); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); - var result = await Target.Write(category, batch, checkpointToExtend); + var result = await CopyCategory("LicensingThroughput"); var keyIgnoresTrailingSpace = (await QueryTarget(dbContext => Task.FromResult(dbContext.Database.ProviderName))).Contains("SqlServer"); var count = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput @@ -153,10 +147,10 @@ public async Task Two_names_differing_only_in_a_trailing_space_add_their_counts_ using (Assert.EnterMultipleScope()) { - Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space, so the two days are one row; PostgreSQL's does not, so they are two"); - Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the day SQL Server's key folds into the other is already present, not lost or failed"); + Assert.That(result.Copied, Is.EqualTo(2), "the reader sends each endpoint in its own batch, and a day added to the row an earlier batch stored counts as copied"); + Assert.That(result.AlreadyPresent, Is.Zero); Assert.That(result.Skipped, Is.Zero); - Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "in a single batch, every row reported as copied is a row in the table"); + Assert.That(await Target.Count(category), Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space, so the two days are one row; PostgreSQL's does not, so they are two"); Assert.That(count, Is.EqualTo(keyIgnoresTrailingSpace ? 350 : 250), "where the key makes the two endpoints one, their counts for the day are added, as the product's own throughput write adds them"); } } diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index f93b77210c..35590023e8 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -28,6 +28,7 @@ namespace ServiceControl.Migration.AcceptanceTests; using Particular.ServiceControl.Hosting; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.AcceptanceTesting.InfrastructureConfig; +using ServiceControl.Contracts.CustomChecks; using ServiceControl.Hosting.Commands; using ServiceControl.Infrastructure.WebApi; using ServiceControl.MessageFailures; @@ -118,6 +119,7 @@ public async Task SetUp() targetServiceCollection.AddLogging(); targetServiceCollection.AddPersistence(Settings); targetServiceCollection.AddSingleton<IBodyStoragePersistence>(RecordedBodies); + targetServiceCollection.AddSingleton(new NServiceBus.CriticalError((_, _) => Task.CompletedTask)); targetServices = targetServiceCollection.BuildServiceProvider(); Target = targetServices.GetRequiredService<IMigrationTarget>(); @@ -269,6 +271,17 @@ protected Task SeedSourceTrialEndDate(DateOnly endDate) => protected Task SeedSourceRetryHistory(RetryHistory history) => SeedSource(("RetryOperations/History", history)); + // The id is the one the RavenDB persister gives a claim, retyped because this project cannot reference that persister. + protected async Task SeedSourceRetryClaim(string uniqueMessageId) + { + var document = new { FailedMessageId = $"FailedMessages/{uniqueMessageId}", RetryBatchId = "RetryBatches/forwarded" }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, $"FailedMessageRetries/{uniqueMessageId}"); + session.Advanced.GetMetadataFor(document)["@collection"] = "FailedMessageRetries"; + await session.SaveChangesAsync(); + } + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => SeedSourceWithBodies((document, body, contentType)); @@ -305,6 +318,42 @@ protected Task SeedSourceKnownEndpoints(params string[] names) => .Select(name => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" } }) .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint))]); + // A whole second in UTC, because PostgreSQL refuses an unspecified kind and keeps only microseconds. + protected static readonly DateTime SeededReportedAt = new(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + // Written out by shape, because the RavenDB persister's CustomCheck type is in an assembly this project does not reference. + protected async Task SeedSourceCustomCheck( + EndpointDetails endpoint, string customCheckId, string category = "Ping", bool failed = false, string reason = null) + { + var detail = new CustomCheckDetail + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = category, + ReportedAt = SeededReportedAt, + HasFailed = failed, + FailureReason = reason + }; + + var documentId = $"CustomChecks/{detail.GetDeterministicId()}"; + var document = new + { + Id = documentId, + detail.CustomCheckId, + detail.Category, + // The integer, so the document reads back the same whether or not the seeding store saves enums as integers. + Status = (int)(failed ? Status.Fail : Status.Pass), + detail.ReportedAt, + FailureReason = reason, + OriginatingEndpoint = endpoint + }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, documentId); + session.Advanced.GetMetadataFor(document)["@collection"] = "CustomChecks"; + await session.SaveChangesAsync(); + } + protected Task<IMigrationSource> OpenSource(CancellationToken cancellationToken = default) => PersistenceFactory.OpenMigrationSource(Settings, cancellationToken); @@ -536,6 +585,10 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except protected INotificationsDataStore NotificationsStore => targetServices.GetRequiredService<INotificationsDataStore>(); protected ITrialLicenseDataProvider TrialLicenseDataProvider => targetServices.GetRequiredService<ITrialLicenseDataProvider>(); protected IRetryHistoryDataStore RetryHistoryStore => targetServices.GetRequiredService<IRetryHistoryDataStore>(); + protected ExternalIntegrationRequestsDataStore ExternalIntegrationStore => targetServices.GetRequiredService<ExternalIntegrationRequestsDataStore>(); + protected ICustomChecksDataStore CustomChecks => targetServices.GetRequiredService<ICustomChecksDataStore>(); + protected IFailedErrorImportDataStore FailedImportStore => targetServices.GetRequiredService<IFailedErrorImportDataStore>(); + protected IGroupsDataStore GroupsStore => targetServices.GetRequiredService<IGroupsDataStore>(); // Built by hand, because its DI constructor needs NServiceBus settings the target services do not have. // A zero cache duration makes every read go to the table. diff --git a/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs new file mode 100644 index 0000000000..b390acfd56 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Recoverability.ExternalIntegration; + +[TestFixture] +[NonParallelizable] +class PendingIntegrationEventsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_queued_event_is_copied_and_the_product_dispatches_it_as_the_type_it_was_raised_as() + { + var id = $"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}"; + await SeedSource((id, new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/1" } })); + + var result = await CopyCategory("PendingIntegrationEvents"); + + object[] dispatched = []; + ExternalIntegrationStore.Subscribe((contexts, _) => + { + dispatched = contexts; + return Task.CompletedTask; + }); + await ExternalIntegrationStore.DispatchNow(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(dispatched.Single(), Is.InstanceOf<FailedMessageArchivedPublisher.DispatchContext>()); + Assert.That(((FailedMessageArchivedPublisher.DispatchContext)dispatched.Single()).FailedMessageId, Is.EqualTo("FailedMessages/1")); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs index 6d07667274..349a2266cb 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs @@ -3,10 +3,12 @@ namespace ServiceControl.Migration.AcceptanceTests; using System; using System.Threading.Tasks; using NUnit.Framework; +using ServiceControl.ExternalIntegrations; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore; using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Recoverability; +using ServiceControl.Recoverability.ExternalIntegration; [TestFixture] [NonParallelizable] @@ -65,8 +67,27 @@ public async Task A_stale_reset_deletes_nothing() Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the delete shares the reset's transaction, so a refused reset leaves every copied row"); } + [Test] + public async Task A_retry_of_pending_integration_events_dispatches_each_event_once() + { + await SeedSource( + ($"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}", new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/1" } }), + ($"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}", new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/2" } })); + await CopyCategory(MigrationCategoryIds.PendingIntegrationEvents); + var copied = await Target.Count(PendingIntegrationEvents); + + await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.PendingIntegrationEvents))); + await CopyCategory(MigrationCategoryIds.PendingIntegrationEvents); + + using (Assert.EnterMultipleScope()) + { + Assert.That(copied, Is.EqualTo(2), "the first copy writes both requests"); + Assert.That(await Target.Count(PendingIntegrationEvents), Is.EqualTo(copied), "each request gets a new key on every insert, so a retry that kept them would dispatch every event twice"); + } + } + // The same reset --migration-retry makes. It keeps StartedAt and leaves SourceTotal alone, because the engine does not write it. - static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + internal static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with { State = MigrationCategoryState.NotStarted, Cursor = null, @@ -88,6 +109,8 @@ static RetryHistory TwoHistoricAndTwoPending() return history; } + static MigrationCategory PendingIntegrationEvents => MigrationCategoryRegistry.Find(MigrationCategoryIds.PendingIntegrationEvents)!; + static MigrationCategory RetryOperations => MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; static readonly DateTime ChangedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs new file mode 100644 index 0000000000..a553e8cc93 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Concurrent; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class When_the_copy_merges_or_changes_rows : MigrationAcceptanceTest +{ + [Test] + public async Task The_category_summary_counts_changed_and_merged_rows_apart_from_the_rest() + { + var retried = new IngestedFailure { MessageType = new string('m', 600) }; + await SeedSourceWithBody(retried.ToFailedMessage(numberOfAttempts: 2), retried.Body, retried.ContentType); + await SeedSourceSubscriptions( + new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), + new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); + var summaries = new SummaryLines(); + + await MigrateCommand.Run(MigrationSettings(), AllowingAnUnreleasedMigration(builder => builder.Services.AddSingleton<ILoggerProvider>(summaries))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(summaries.Lines, Has.Exactly(1).EqualTo((LogLevel.Warning, $"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages}: 1 copied (1 changed), 0 already present, 0 merged, nothing skipped")), + "the message type was cut, which the summary has to count even though nothing was skipped, and the dropped older attempt is not a change"); + Assert.That(summaries.Lines, Has.Exactly(1).EqualTo((LogLevel.Warning, $"{MigrationCategoryIds.Subscriptions}: 1 copied (0 changed), 0 already present, 1 merged, nothing skipped")), + "the second version is counted as already present by the checkpoint, and the summary has to show it as a merge instead"); + } + } + + // Keeps the lines MigrationStartup logs, which is where the summary is written. + sealed class SummaryLines : ILoggerProvider + { + readonly ConcurrentQueue<(LogLevel Level, string Message)> lines = new(); + + public (LogLevel Level, string Message)[] Lines => [.. lines]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(MigrationStartup).FullName ? new SummaryLogger(lines) : NullLogger.Instance; + + public void Dispose() + { + } + + sealed class SummaryLogger(ConcurrentQueue<(LogLevel Level, string Message)> lines) : ILogger + { + public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) => + lines.Enqueue((logLevel, formatter(state, exception))); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs index 135c7a213d..25c51dc849 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs @@ -167,6 +167,7 @@ public async Task A_retry_by_id_after_the_copy_stages_the_unresolved_message_and var retryIssued = new IngestedFailure(); await SeedFailure(unresolved); await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); @@ -187,10 +188,16 @@ public async Task A_retry_issued_message_resent_from_pending_retries_after_the_c { var retryIssued = new IngestedFailure(); await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); - Assert.That(copy.Copied, Is.EqualTo(1), "the test proves nothing unless the message reached the target"); + using (Assert.EnterMultipleScope()) + { + Assert.That(copy.Copied, Is.EqualTo(1), "the test proves nothing unless the message reached the target"); + Assert.That(await QueryTarget(db => db.FailedMessageRetries.CountAsync(claim => claim.UniqueMessageId == retryIssued.UniqueMessageId)), Is.EqualTo(1), + "the test proves nothing about releasing a migrated claim unless the copy wrote one"); + } // Pending Retries releases the claim, then sends a retry by id. await FailedMessageRetryDataStore.RemoveFailedMessageRetry(retryIssued.UniqueMessageIdString); diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs index 6c4ec5155a..86962c5a44 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -49,22 +49,28 @@ ON CONFLICT ({keyColumns}) DO NOTHING return inserted; } - public async Task<int> SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) + public async Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) { var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); + var columns = string.Join(", ", insert.Columns); var keyColumns = string.Join(", ", insert.KeyColumns); + var counts = insert.Columns.Except(insert.KeyColumns).ToList(); var written = 0; foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) { + // Grouped first, because ON CONFLICT refuses a statement that reaches one row twice. written += (await insert.Execute( dbContext, $""" - INSERT INTO {insert.Table} ({string.Join(", ", insert.Columns)}) - VALUES + INSERT INTO {insert.Table} ({columns}) + SELECT {string.Join(", ", insert.Columns.Select(column => counts.Contains(column) ? $"SUM({column})" : column))} + FROM (VALUES {ParameterRows(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}) + GROUP BY {keyColumns} ON CONFLICT ({keyColumns}) - DO UPDATE SET {string.Join(", ", insert.Columns.Except(insert.KeyColumns).Select(column => $"{column} = excluded.{column}"))} + DO UPDATE SET {string.Join(", ", counts.Select(column => $"{column} = {insert.Table}.{column} + excluded.{column}"))} RETURNING {keyColumns} """, chunk, diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs index 0ebab54fc0..2ce3a44741 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using System.Data.Common; +using System.Globalization; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -10,14 +11,12 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using ServiceControl.Persistence.EFCore.Infrastructure; /// <summary> -/// The migration's SQL Server statements. Two keys that differ only in case or trailing spaces can be one row here, because a +/// The migration's SQL Server statements. Two keys that differ only in case, accents, kana type, width or trailing spaces can be one row here, because a /// column's collation decides what counts as equal, so the statements compare keys under the collation the /// schema gives them and the batch is de-duplicated the same way before it is offered to the table. /// </summary> class SqlServerMigrationSqlDialect : SqlServerDialect, IMigrationSqlDialect { - const int IgnoreCaseStyle = 1; - OpenedKeys? OpenedState { get; set; } public async Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) @@ -55,7 +54,7 @@ FROM sys.columns AS c ?? throw new InvalidOperationException($"The migration target found no collation for {tableName}.{column}, so the SQL Server schema is missing or not current. Run ServiceControl with --setup against this database first."); collations[(table, sql.DelimitIdentifier(column))] = collation.Name; - comparers[(entityType.ClrType, property.Name)] = new PaddedKeyComparer((collation.ComparisonStyle & IgnoreCaseStyle) == 0 ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + comparers[(entityType.ClrType, property.Name)] = new PaddedKeyComparer(ComparerFor(collation.ComparisonStyle)); } } @@ -114,12 +113,13 @@ WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Colum return inserted; } - public async Task<int> SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) + public async Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) { var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); var columns = string.Join(", ", insert.Columns); - var partitionBy = string.Join(", ", insert.KeyColumns.Select(column => - Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column)); + var counts = insert.Columns.Except(insert.KeyColumns).ToList(); + var collatedKeys = insert.KeyColumns.ToDictionary(column => column, column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column); var written = 0; foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) @@ -131,17 +131,14 @@ public async Task<int> SetLicensingEndpointThroughput(ServiceControlDbContext db $""" MERGE {insert.Table} WITH (HOLDLOCK) AS t USING ( - SELECT {columns} - FROM ( - SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {partitionBy} ORDER BY [MigrationOrdinal] DESC) AS [MigrationDuplicate] - FROM (VALUES - {ParameterRowsWithOrdinal(chunk.Length, insert.Columns.Count)} - ) AS v ({columns}, [MigrationOrdinal]) - ) AS deduplicated - WHERE [MigrationDuplicate] = 1 + SELECT {string.Join(", ", insert.Columns.Select(column => collatedKeys.TryGetValue(column, out var key) ? key : $"SUM({column})"))} + FROM (VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}) + GROUP BY {string.Join(", ", collatedKeys.Values)} ) AS s ({columns}) ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} - WHEN MATCHED THEN UPDATE SET {string.Join(", ", insert.Columns.Except(insert.KeyColumns).Select(column => $"{column} = s.{column}"))} + WHEN MATCHED THEN UPDATE SET {string.Join(", ", counts.Select(column => $"{column} = t.{column} + s.{column}"))} WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"s.{column}"))}; """, @@ -163,6 +160,29 @@ static void DeclareUnboundedTextAsMax(IProperty property, DbParameter parameter) } } + // COLLATIONPROPERTY's ComparisonStyle is 0 for a binary collation and for one that ignores nothing, and otherwise sets one bit for each thing the collation ignores. + static StringComparer ComparerFor(int comparisonStyle) + { + if (comparisonStyle == 0) + { + return StringComparer.Ordinal; + } + + var options = ComparisonStyleOptions + .Where(style => (comparisonStyle & style.Bit) != 0) + .Aggregate(CompareOptions.None, (combined, style) => combined | style.Option); + + return CultureInfo.InvariantCulture.CompareInfo.GetStringComparer(options); + } + + static readonly (int Bit, CompareOptions Option)[] ComparisonStyleOptions = + [ + (1, CompareOptions.IgnoreCase), + (2, CompareOptions.IgnoreNonSpace), + (0x10000, CompareOptions.IgnoreKanaType), + (0x20000, CompareOptions.IgnoreWidth) + ]; + OpenedKeys Keys => OpenedState ?? throw new InvalidOperationException($"The SQL Server migration dialect is not open. Call {nameof(Open)} first."); sealed record OpenedKeys( diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs new file mode 100644 index 0000000000..ae192efe04 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.EFCore.Entities; + +static class CustomCheckRowMapper +{ + public static CustomCheckEntity Map(CustomCheckDetail detail) => new() + { + // The key a live report of the same check upserts on, so the first report after the host opens replaces this row rather than adding a second. + Id = detail.GetDeterministicId(), + CustomCheckId = detail.CustomCheckId, + Category = detail.Category, + Status = detail.HasFailed ? Status.Fail : Status.Pass, + ReportedAt = detail.ReportedAt, + FailureReason = detail.FailureReason, + OriginatingEndpointName = detail.OriginatingEndpoint.Name, + OriginatingEndpointHostId = detail.OriginatingEndpoint.HostId, + OriginatingEndpointHost = detail.OriginatingEndpoint.Host + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index 066d6e8a2c..6d8297c073 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -35,7 +35,11 @@ sealed class EFCoreMigrationTarget( new SubscriptionsWriter(migrationDialect), new NotificationSettingsWriter(migrationDialect), new TrialEndDateWriter(migrationDialect), - new RetryOperationsWriter(migrationDialect) + new RetryOperationsWriter(migrationDialect), + new PendingIntegrationEventsWriter(migrationDialect), + new CustomChecksWriter(migrationDialect), + new FailedErrorImportsWriter(migrationDialect, storagePersistence, settings), + new GroupCommentsWriter(migrationDialect) }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => @@ -84,13 +88,26 @@ public Task<MigrationWriteResult> Write( logger.LogWarning("Merged {SourceId} in category {CategoryId} into a row with the same key: {Detail}", sourceId, category.Id, detail); } + foreach (var (sourceId, detail) in prepared.Changes) + { + logger.LogWarning("Changed {SourceId} in category {CategoryId} on the way into SQL: {Detail}", sourceId, category.Id, detail); + } + + // Once per copy of the category, because the line describes every batch alike and a retry's reset clears the count. + if (prepared.RenumberedRows is { } renumbered && checkpointToExtend.CopiedCount == 0) + { + logger.LogInformation("Category {CategoryId}: the database gives the copied {RenumberedRows} new ids. Nothing in SQL refers to the ids they had in the source.", category.Id, renumbered); + } + return new MigrationWriteResult( saved, copied, prepared.Skips.Count, [.. prepared.Skips.Select(skip => skip.SourceId)], AlreadyPresent: alreadyPresent, - SkipReasons: skipReasons); + SkipReasons: skipReasons, + Merged: prepared.Merges.Count, + Changed: prepared.Changes.Count); }, cancellationToken); // Checked before the insert, because the subtraction below can only catch a writer that over-reports. @@ -129,6 +146,12 @@ public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, Cancel await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + // Only a reset that deletes is refused, because an optional category is retried after ServiceControl opens and keeps its rows. + if (storeKeyed is not null && await dbContext.GetSetting<DateTime?>(SettingKeys.MigrationHostOpenedOnTarget, token) is not null) + { + throw new InvalidOperationException($"Category {reset.CategoryId} cannot be retried, because ServiceControl has opened on this database since the copy and its tables may now hold live rows. A retry deletes every row in them, so it is only possible before ServiceControl first opens on this database."); + } + var deletedRows = storeKeyed is null ? 0 : await storeKeyed.DeleteCopiedRows(dbContext, token); var stored = await dbContext.UpsertCheckpoint(reset, token); await transaction.CommitAsync(token); @@ -142,7 +165,7 @@ public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, Cancel } else { - logger.LogInformation("Reset category {CategoryId} for a retry and deleted the {DeletedRowCount} rows it copied, because a re-read would add them again under new keys", reset.CategoryId, deleted); + logger.LogInformation("Reset category {CategoryId} for a retry and deleted the {DeletedRowCount} rows it copied, because a re-read would add them a second time", reset.CategoryId, deleted); } return saved; diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs new file mode 100644 index 0000000000..942dc4ea71 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +static class FailedErrorImportRowMapper +{ + public static FailedErrorImportEntity Map(FailedErrorImport import, DateTime failedAt, bool bodyStoredExternally) + { + var message = import.Message!; + + return new FailedErrorImportEntity + { + // The key a live import of the same message derives, so a later failure of that message on SQL updates this row rather than adding a second. + UniqueMessageId = FailedErrorImport.DeriveKey(message.Headers, message.Id), + FailedAt = failedAt, + MessageId = message.Id, + HeadersJson = MessageHeaders.Write(message.Headers), + Body = bodyStoredExternally ? [] : message.Body ?? [], + BodyStoredExternally = bodyStoredExternally, + ExceptionInfo = import.ExceptionInfo ?? string.Empty + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs index b7360c1725..f23837b53e 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs @@ -14,8 +14,9 @@ abstract record FailedMessageDocumentTranslation /// <summary> /// The document translates to a failed message row and the group rows keyed by it. + /// Changes has one sentence for each way the row differs from the document on purpose, and is empty when it differs in none. /// </summary> - public sealed record Translated(FailedMessageEntity Row, IReadOnlyList<FailedMessageGroupEntity> Groups) : FailedMessageDocumentTranslation; + public sealed record Translated(FailedMessageEntity Row, IReadOnlyList<FailedMessageGroupEntity> Groups, IReadOnlyList<string> Changes) : FailedMessageDocumentTranslation; /// <summary> /// The document cannot become a row, for the reason given, and <paramref name="Detail" /> says which document and why. diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs index a5f241096e..24d2b6435c 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs @@ -119,7 +119,33 @@ public static FailedMessageDocumentTranslation Translate(MigrationRow source) }) ]; - return new FailedMessageDocumentTranslation.Translated(row, groups); + return new FailedMessageDocumentTranslation.Translated(row, groups, ChangesMadeTo(document, metadata.GetValueOrDefault("MessageType") as string)); + } + + // One sentence per way the row differs from the document, for the warning the target logs after the commit. + static IReadOnlyList<string> ChangesMadeTo(FailedMessage document, string? messageType) + { + List<string> changes = []; + var groups = document.FailureGroups ?? []; + FailedMessage.FailureGroup?[] withoutId = [.. groups.Where(failureGroup => failureGroup is not { Id: not null })]; + FailedMessage.FailureGroup[] withoutTitleOrType = [.. groups.Where(failureGroup => failureGroup is { Id: not null } && (failureGroup.Title is null || failureGroup.Type is null))]; + + if (messageType is { Length: > ColumnLengths.ShortTextLength }) + { + changes.Add($"Cut the message type from {messageType.Length} characters to its last {ColumnLengths.ShortTextLength}, as ingestion does."); + } + + if (withoutId.Length > 0) + { + changes.Add($"Dropped the failure groups with no id, because SQL keys a group by its id: {string.Join(", ", withoutId.Select(failureGroup => failureGroup is null ? "an empty entry" : $"the group titled '{failureGroup.Title}'"))}."); + } + + if (withoutTitleOrType.Length > 0) + { + changes.Add($"Stored an empty title or type for the failure groups that had none: {string.Join(", ", withoutTitleOrType.Select(failureGroup => $"'{failureGroup.Id}'"))}."); + } + + return changes; } /// <summary> diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs new file mode 100644 index 0000000000..86fb1ebcdd --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// Turns a group comment from the source into the row the product's own edit would write. The writer skips a null or blank comment before it calls this. +/// </summary> +static class GroupCommentRowMapper +{ + public static GroupCommentEntity Map(GroupComment comment) => new() + { + GroupId = comment.Id, + Comment = comment.Comment! + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs index 726c80ad1e..f66dd7d99f 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs @@ -3,7 +3,8 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; using DbContexts; /// <summary> -/// A category writer whose rows get their keys from the database, so a re-read would add every row again. +/// A category writer whose rows a re-read cannot recognise as already copied, so copying the category again would add every row a second time. +/// Rows whose keys the database makes are one kind, and throughput days, whose counts the copy adds to, are another. /// The target deletes those rows when it resets the category for a retry. /// The delete empties whole tables, so it is only safe for a category copied while ServiceControl is stopped. /// </summary> diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs new file mode 100644 index 0000000000..326e2af514 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.EFCore.Entities; + +static class PendingIntegrationEventRowMapper +{ + /// <summary> + /// Builds the row for one queued request, with its context stored the way the product stores it. Throws when the request has no context, so a caller must skip such a request before it calls this. + /// </summary> + public static ExternalIntegrationDispatchRequestEntity Map(ExternalIntegrationDispatchRequest request) + { + // The product's own serializer, so the dispatcher reads a copied row exactly as it reads one it stored. + var (typeName, json) = DispatchContextSerializer.Serialize(request.DispatchContext); + + return new ExternalIntegrationDispatchRequestEntity { DispatchContextTypeName = typeName, DispatchContextJson = json }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs index c4e3faebb6..2ffa313724 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs @@ -4,15 +4,20 @@ namespace ServiceControl.Persistence.EFCore.DataMigration; using ServiceControl.Persistence.DataMigration; /// <summary> -/// One batch turned into rows, ready for the target to insert inside its own transaction. Nothing here has -/// touched the database yet. +/// A batch a writer has turned into rows, which the target inserts in the same transaction that saves the checkpoint. +/// Skips are rows not written, Merges are rows dropped because another row has the same key, and Changes are rows written with a value that differs from the source's. +/// The target logs each skip, merge and change as a warning after the commit. /// </summary> -/// <param name="Insert">Runs the insert and returns how many rows it added. The target calls it inside the transaction that also saves the checkpoint.</param> -/// <param name="PreparedRowCount">How many rows the writer built, which is more than the insert adds when the table already holds some of their keys.</param> -/// <param name="Skips">One entry per row the writer will not insert, with the reason and a detail line for the log.</param> -/// <param name="Merges">One entry per prepared row whose key the target treats as the same key as another row's, either earlier in the batch or already in the table, so the insert keeps the other row and drops this one. Each carries a detail line naming both keys and the one kept, which the target logs as a warning after the commit. The dropped row is counted as already present.</param> sealed record PreparedBatch( Func<ServiceControlDbContext, CancellationToken, Task<int>> Insert, int PreparedRowCount, IReadOnlyList<(string SourceId, MigrationSkipReason Reason, string Detail)> Skips, - IReadOnlyList<(string SourceId, string Detail)> Merges); + IReadOnlyList<(string SourceId, string Detail)> Merges) +{ + public IReadOnlyList<(string SourceId, string Detail)> Changes { get; init; } = []; + + /// <summary> + /// Names the rows the database gives new ids, or is null when it gives none. + /// </summary> + public string? RenumberedRows { get; init; } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs new file mode 100644 index 0000000000..b13ab708fb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs @@ -0,0 +1,71 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the custom checks under the id the product derives from the endpoint and the check, and never replaces one the target already holds. +/// A row with SourceStorageCheck true is skipped as SourceStorageCheck, which is harmless, and a SourceStorageCheck value that is not a bool throws. +/// A row with a required text value left null is skipped as RequiredValueMissing, and one with a character the database refuses as MalformedValue. +/// </summary> +sealed class CustomChecksWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<CustomCheckDetail> +{ + public override string CategoryId => MigrationCategoryIds.CustomChecks; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<CustomCheckEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.CustomChecks.LongCountAsync(cancellationToken); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, CustomCheckDetail Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<CustomCheckEntity>.For(dbContext); + List<CustomCheckEntity> rows = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, detail) in documents) + { + if (IsSourceStorageCheck(row)) + { + skips.Add((row.SourceId, MigrationSkipReason.SourceStorageCheck, $"'{detail.CustomCheckId}' under '{detail.Category}' is a check the source's own storage runs about itself, which nothing reports once the instance runs on the target")); + continue; + } + + var column = detail.CustomCheckId is null ? nameof(CustomCheckEntity.CustomCheckId) + : detail.Category is null ? nameof(CustomCheckEntity.Category) + : detail.OriginatingEndpoint?.Name is null ? nameof(CustomCheckEntity.OriginatingEndpointName) + : detail.OriginatingEndpoint.Host is null ? nameof(CustomCheckEntity.OriginatingEndpointHost) + : null; + + if (column is not null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"CustomChecks.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = CustomCheckRowMapper.Map(detail); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); + } + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, skips, Merges: [])); + } + + static bool IsSourceStorageCheck(MigrationRow row) => + row.Metadata.GetValueOrDefault(MigrationMetadataKeys.SourceStorageCheck) switch + { + bool flagged => flagged, + null => false, + var other => throw new InvalidOperationException($"Row '{row.SourceId}' carries a {other.GetType().FullName} as its {MigrationMetadataKeys.SourceStorageCheck} value, where the source sets a {typeof(bool).FullName}") + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs index b13a6e8e43..ee2ef84935 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -8,7 +8,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; /// <summary> /// Copies every per-endpoint setting as stored. -/// Where the name column ignores case, names that differ only in case become one row, and the dropped ones are +/// Where the name column's collation ignores case, accents, kana type or width, names that differ only in those become one row, and the dropped ones are /// reported as merges. The row kept is the one whose name matches a known endpoint exactly. /// </summary> sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<EndpointSettings> diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs new file mode 100644 index 0000000000..6e2b01156f --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs @@ -0,0 +1,146 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Abstractions; +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the failed error imports under the key a live import of the same message derives, and never replaces one the target already holds. +/// A body larger than the target keeps inline goes to body storage, under the body id the SQL import store uses for that key, before the rows are inserted. +/// A second import of the same message, in the batch or already in the target from another document, is reported as a merge. +/// </summary> +sealed class FailedErrorImportsWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter<FailedErrorImport> +{ + public override string CategoryId => MigrationCategoryIds.FailedErrorImports; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<FailedErrorImportEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.FailedErrorImports.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, FailedErrorImport Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<FailedErrorImportEntity>.For(dbContext); + List<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + List<(string SourceId, Guid UniqueMessageId, string Detail)> changes = []; + + foreach (var (row, import) in documents) + { + var column = import.Message?.Id is null ? nameof(FailedErrorImportEntity.MessageId) + : import.Message.Headers is null ? nameof(FailedErrorImportEntity.HeadersJson) + : null; + + if (column is not null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"FailedErrorImports.{column} is NOT NULL and the document has no message, or its message has no value for it")); + continue; + } + + var failedAt = row.Metadata.GetValueOrDefault(MigrationMetadataKeys.FailedAt) as DateTime? + ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} carries no {MigrationMetadataKeys.FailedAt}, so the import's failure time is unknown."); + var body = row.Body + ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} reached the writer without a body. The engine attaches a body to every row of a category that carries bodies, or skips the row."); + var entity = FailedErrorImportRowMapper.Map(import, failedAt, body.Content.Length > persisterSettings.BodyStorage.MaxBodySizeToStore); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity, body)); + + if (EmptiedValues(import) is { } emptied) + { + changes.Add((row.SourceId, entity.UniqueMessageId, emptied)); + } + } + + var stored = await FailureTimesStoredFor(dbContext, rows, cancellationToken); + var (kept, merges) = KeptAndMerged(rows, stored); + + // Every import of one message shares one body id, so only the row the insert keeps may write its body. + await WriteBodies([.. rows.Where(row => row.Row.BodyStoredExternally && !stored.ContainsKey(row.Row.UniqueMessageId) && kept[row.Row.UniqueMessageId] == row.SourceId)], cancellationToken); + + return new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, merges) + { + Changes = [.. changes.Where(change => !stored.ContainsKey(change.UniqueMessageId) && kept[change.UniqueMessageId] == change.SourceId).Select(change => (change.SourceId, change.Detail))] + }; + } + + // FailedErrorImportRowMapper stores a missing exception or body as empty, because both columns are NOT NULL. + static string? EmptiedValues(FailedErrorImport import) => + (import.ExceptionInfo, import.Message!.Body) switch + { + (null, null) => "Stored an empty exception and an empty body, because the document has neither.", + (null, _) => "Stored an empty exception, because the document has none.", + (_, null) => "Stored an empty body, because the document's message has none.", + _ => null + }; + + static async Task<Dictionary<Guid, DateTime>> FailureTimesStoredFor(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows, CancellationToken cancellationToken) + { + var ids = rows.Select(row => row.Row.UniqueMessageId).Distinct().ToArray(); + + return await dbContext.FailedErrorImports + .AsNoTracking() + .Where(import => ids.Contains(import.UniqueMessageId)) + .ToDictionaryAsync(import => import.UniqueMessageId, import => import.FailedAt, cancellationToken); + } + + // A stored row wins over every row of the batch, and an earlier row over a later one, as in the insert. A stored row with the same failure time is this document's own row, which a retry reads again. + static (Dictionary<Guid, string?> Kept, List<(string SourceId, string Detail)> Merges) KeptAndMerged(IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows, Dictionary<Guid, DateTime> stored) + { + var kept = stored.ToDictionary(entry => entry.Key, entry => (string?)null); + List<(string SourceId, string Detail)> merges = []; + + foreach (var (sourceId, row, _) in rows) + { + if (!kept.TryGetValue(row.UniqueMessageId, out var keeper) || (keeper is null && SameFailureTime(stored[row.UniqueMessageId], row.FailedAt))) + { + kept[row.UniqueMessageId] = sourceId; + } + else if (keeper is null) + { + merges.Add((sourceId, $"FailedErrorImports already holds an import of the same message, '{row.MessageId}', that failed at {stored[row.UniqueMessageId]:O}, while this document failed at {row.FailedAt:O}, so it kept the stored row and dropped this one")); + } + else + { + merges.Add((sourceId, $"'{keeper}' is an import of the same message, '{row.MessageId}', so FailedErrorImports kept the row from '{keeper}' and dropped the row from '{sourceId}'")); + } + } + + return (kept, merges); + } + + // PostgreSQL keeps only whole microseconds, so the stored time of the same document can differ from a re-read by less than one. + static bool SameFailureTime(DateTime stored, DateTime read) => Math.Abs((stored - read).Ticks) < TimeSpan.TicksPerMicrosecond; + + // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. + Task WriteBodies(IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> externalBodies, CancellationToken cancellationToken) => + Task.WhenAll(externalBodies.Select(external => WriteBody(external, cancellationToken))); + + async Task WriteBody((string SourceId, FailedErrorImportEntity Row, MigrationBody Body) external, CancellationToken cancellationToken) + { + var bodyId = FailedErrorImportEntity.ExternalBodyId(external.Row.UniqueMessageId); + + try + { + await storagePersistence.WriteBody(bodyId, external.Body.Content, external.Body.ContentType, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // The storage SDK errors do not name the body, so without this the error that stops the copy would not say which import failed. + catch (Exception ex) + { + throw new InvalidOperationException($"Writing the body of document '{external.SourceId}' in category {CategoryId} to external body storage as '{bodyId}' ({external.Body.Content.Length} bytes) failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs index 9fe40c125e..a1f596b00c 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs @@ -9,9 +9,11 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.DataMigration; /// <summary> -/// Writes failed messages, their failure group rows and a retry claim for each retry-issued message, inserting only -/// messages the target does not already hold and never updating one it does. Each category that copies failed -/// messages derives from this and names its category id and the statuses it copies. +/// Writes failed messages, their failure group rows and a retry claim for each retry-issued message the source still +/// held one for, inserting only messages the target does not already hold and never updating one it does. The claims +/// mirror the source, so a retry by id behaves as it did before the move. A retry-issued row with no bool RetryClaimHeld +/// value fails the batch. Each category that copies failed messages derives from this and names its category id and the +/// statuses it copies. /// </summary> abstract class FailedMessagesWriterBase(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter<FailedMessage> { @@ -39,6 +41,9 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo List<FailedMessageDocumentTranslation.Translated> translated = []; HashSet<Guid> translatedIds = []; List<(string SourceId, FailedMessageEntity Row, MigrationBody Body)> externalBodies = []; + List<(string SourceId, Guid UniqueMessageId, string Detail)> changes = []; + HashSet<Guid> heldClaims = []; + var groupIds = migrationDialect.KeyComparer(typeof(FailedMessageGroupEntity), nameof(FailedMessageGroupEntity.GroupId)); foreach (var (row, document) in documents) { @@ -61,10 +66,23 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo { translated.Add(translation); - // The insert keeps only the first document with an id, so only its body may be stored under that id. - if (translatedIds.Add(translation.Row.UniqueMessageId) && storeExternally) + // The insert keeps only the first document with an id, so only its body is stored under that id, only its changes are reported and only its claim is copied. + if (translatedIds.Add(translation.Row.UniqueMessageId)) { - externalBodies.Add((row.SourceId, translation.Row, body)); + if (storeExternally) + { + externalBodies.Add((row.SourceId, translation.Row, body)); + } + + if (translation.Row.Status == FailedMessageStatus.RetryIssued && SourceHeldRetryClaim(row)) + { + heldClaims.Add(translation.Row.UniqueMessageId); + } + + if (ChangesIn(translation, groupIds) is { Count: > 0 } changed) + { + changes.Add((row.SourceId, translation.Row.UniqueMessageId, string.Join(" ", changed))); + } } } @@ -77,32 +95,47 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo } } - await WriteBodiesTheTargetLacks(dbContext, externalBodies, cancellationToken); + // This only skips the bodies and changes of messages the target already holds; InsertMissing still decides which rows go in. + var present = externalBodies.Count == 0 && changes.Count == 0 + ? [] + : await MessagesTheTargetHolds(dbContext, [.. externalBodies.Select(external => external.Row.UniqueMessageId), .. changes.Select(change => change.UniqueMessageId)], cancellationToken); - return new PreparedBatch((context, token) => InsertFailedMessages(context, translated, token), translated.Count, skips, Merges: []); - } + // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. + await Task.WhenAll(externalBodies + .Where(external => !present.Contains(external.Row.UniqueMessageId)) + .Select(external => WriteBody(external, cancellationToken))); - // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. - async Task WriteBodiesTheTargetLacks(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, FailedMessageEntity Row, MigrationBody Body)> externalBodies, CancellationToken cancellationToken) - { - if (externalBodies.Count == 0) + return new PreparedBatch((context, token) => InsertFailedMessages(context, translated, heldClaims, token), translated.Count, skips, Merges: []) { - return; - } - - var ids = externalBodies.Select(external => external.Row.UniqueMessageId).ToArray(); + Changes = [.. changes.Where(change => !present.Contains(change.UniqueMessageId)).Select(change => (change.SourceId, change.Detail))] + }; + } - // This only skips writing bodies for messages the target already holds; InsertMissing still decides which rows go in. - var present = (await dbContext.FailedMessages + static async Task<HashSet<Guid>> MessagesTheTargetHolds(ServiceControlDbContext dbContext, Guid[] ids, CancellationToken cancellationToken) => + (await dbContext.FailedMessages .AsNoTracking() .Where(message => ids.Contains(message.UniqueMessageId)) .Select(message => message.UniqueMessageId) .ToListAsync(cancellationToken)).ToHashSet(); - await Task.WhenAll(externalBodies - .Where(external => !present.Contains(external.Row.UniqueMessageId)) - .Select(external => WriteBody(external, cancellationToken))); - } + // The group insert keeps the first of the ids its key treats as one, so a later group with such an id loses its title and type. + static IReadOnlyList<string> ChangesIn(FailedMessageDocumentTranslation.Translated translation, IEqualityComparer<string> groupIds) => + [ + .. translation.Changes, + .. translation.Groups + .GroupBy(group => group.GroupId, groupIds) + .Where(sameKey => sameKey.Count() > 1) + .Select(sameKey => $"Kept the failure group '{sameKey.First().GroupId}' titled '{sameKey.First().Title}' and dropped {string.Join(", ", sameKey.Skip(1).Select(group => $"'{group.GroupId}' titled '{group.Title}'"))}, because SQL treats their ids as one key.") + ]; + + // Throws rather than guess, because a wrong guess either blocks a retry by id or lets one send the message twice. + static bool SourceHeldRetryClaim(MigrationRow row) => + row.Metadata.GetValueOrDefault(MigrationMetadataKeys.RetryClaimHeld) switch + { + bool held => held, + null => throw new InvalidOperationException($"Row '{row.SourceId}' is retry-issued but arrived without a {MigrationMetadataKeys.RetryClaimHeld} value, which the source resolves for every retry-issued failed message"), + var other => throw new InvalidOperationException($"Row '{row.SourceId}' carries a {other.GetType().FullName} as its {MigrationMetadataKeys.RetryClaimHeld} value, where the source sets a {typeof(bool).FullName}") + }; async Task WriteBody((string SourceId, FailedMessageEntity Row, MigrationBody Body) external, CancellationToken cancellationToken) { @@ -136,7 +169,7 @@ T NamingTheDocument<T>(MigrationRow row, Func<T> step) } } - async Task<int> InsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList<FailedMessageDocumentTranslation.Translated> translated, CancellationToken cancellationToken) + async Task<int> InsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList<FailedMessageDocumentTranslation.Translated> translated, IReadOnlySet<Guid> heldClaims, CancellationToken cancellationToken) { var inserted = (await migrationDialect.InsertMissing(dbContext, [.. translated.Select(translation => translation.Row)], cancellationToken)) .ToHashSet<FailedMessageEntity>(ReferenceEqualityComparer.Instance); @@ -145,9 +178,9 @@ async Task<int> InsertFailedMessages(ServiceControlDbContext dbContext, IReadOnl // A group goes in only when the message its foreign key points at went in, so a message already in the target keeps the groups it had. await migrationDialect.InsertMissing(dbContext, [.. translated.Where(translation => inserted.Contains(translation.Row)).SelectMany(translation => translation.Groups)], cancellationToken); - // A retry-issued message needs a claim, or a retry by id sends it again while its first retry is still outstanding. + // The claims mirror the source, so a retry by id stays blocked or goes through exactly as it would have before the move. await migrationDialect.InsertMissing(dbContext, [.. inserted - .Where(row => row.Status == FailedMessageStatus.RetryIssued) + .Where(row => heldClaims.Contains(row.UniqueMessageId)) .Select(row => new FailedMessageRetryEntity { UniqueMessageId = row.UniqueMessageId, RetryBatchId = MigratedRetryClaimBatchId })], cancellationToken); return inserted.Count; diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs new file mode 100644 index 0000000000..3179f62c16 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs @@ -0,0 +1,79 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Copies the group comments, one row per comment keyed on its group id, and never replaces a comment the target already holds. +/// A null or blank comment is skipped as BlankGroupComment, because the product keeps no row for one. +/// A comment whose group id the key column treats as another's is reported as a merge. +/// A copied comment on a group the target holds no failed message in is reported as a change, because the retention sweep deletes it once ServiceControl opens. +/// </summary> +sealed class GroupCommentsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<GroupComment> +{ + public override string CategoryId => MigrationCategoryIds.GroupComments; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<GroupCommentEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.GroupComments.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, GroupComment Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<GroupCommentEntity>.For(dbContext); + var rows = new List<(string SourceId, GroupCommentEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, comment) in documents) + { + if (string.IsNullOrWhiteSpace(comment.Comment)) + { + skips.Add((row.SourceId, MigrationSkipReason.BlankGroupComment, $"the comment on group '{comment.Id}' is blank, and the product keeps no row for a blank comment")); + continue; + } + + var entity = GroupCommentRowMapper.Map(comment); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + var (merges, inserted) = await KeyMerges.Split(dbContext, migrationDialect, rows, cancellationToken: cancellationToken); + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + merges) + { + Changes = await CommentsTheSweepWillDelete(dbContext, inserted, cancellationToken) + }; + } + + // Group comments copy after the unresolved failed messages, so the groups the target holds now are the groups it holds when ServiceControl opens. + async Task<IReadOnlyList<(string SourceId, string Detail)>> CommentsTheSweepWillDelete(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, GroupCommentEntity Row)> inserted, CancellationToken cancellationToken) + { + string[] groupIds = [.. inserted.Select(comment => comment.Row.GroupId).Distinct(StringComparer.Ordinal)]; + var held = (await dbContext.FailedMessageGroups + .AsNoTracking() + .Where(group => groupIds.Contains(group.GroupId)) + .Select(group => group.GroupId) + .ToListAsync(cancellationToken)) + .ToHashSet(migrationDialect.KeyComparer(typeof(FailedMessageGroupEntity), nameof(FailedMessageGroupEntity.GroupId))); + + return [.. inserted + .Where(comment => !held.Contains(comment.Row.GroupId)) + .Select(comment => (comment.SourceId, $"The group '{comment.Row.GroupId}' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + + "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + $"The comment was: '{comment.Row.Comment}'"))]; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs index 8116cac289..6650d9b5fb 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs @@ -7,17 +7,22 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; /// <summary> /// Finds the rows the insert will drop because another row, earlier in the batch or already in the table, has the same /// key under the column's collation. A row the table already holds exactly is not a merge. -/// Returns each dropped row's source id with a line naming the key that was kept. +/// Returns each dropped row's source id with a line naming both keys, the source row whose key was kept, or the stored row, and the source row dropped. +/// Split also returns the rows the insert adds, and takes a describeSource that turns a source id into the words the line uses for it, quoting it by default. /// </summary> static class KeyMerges { - public static async Task<IReadOnlyList<(string SourceId, string Detail)>> In<TEntity>(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, CancellationToken cancellationToken = default) where TEntity : class + public static async Task<IReadOnlyList<(string SourceId, string Detail)>> In<TEntity>(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, CancellationToken cancellationToken = default) where TEntity : class => + (await Split(dbContext, dialect, rows, cancellationToken: cancellationToken)).Merges; + + public static async Task<(IReadOnlyList<(string SourceId, string Detail)> Merges, IReadOnlyList<(string SourceId, TEntity Row)> Inserted)> Split<TEntity>(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, Func<string, string>? describeSource = null, CancellationToken cancellationToken = default) where TEntity : class { + describeSource ??= sourceId => $"'{sourceId}'"; var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; var keyProperties = entityType.FindPrimaryKey()!.Properties; var queryColumn = keyProperties.FirstOrDefault(property => property.ClrType == typeof(string)) ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no text key column, so no collation can join two of its keys."); - Func<object?, object?, bool>[] columnMatches = [.. keyProperties.Select(property => property.ClrType == typeof(string) ? TextMatch(dialect.KeyComparer(typeof(TEntity), property.Name)) : object.Equals)]; + Func<object?, object?, bool>[] columnMatches = [.. keyProperties.Select(property => property.ClrType == typeof(string) ? TextMatch(dialect.KeyComparer(typeof(TEntity), property.Name)) : Equals)]; object?[] KeyOf(TEntity entity) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(entity))]; bool SameKey(object?[] first, object?[] second) => columnMatches.Select((matches, column) => matches(first[column], second[column])).All(same => same); @@ -31,8 +36,10 @@ static class KeyMerges } // A stored key wins over every row of the batch, and an earlier row over a later one, as in the insert. - var kept = stored.Select(entity => (Key: KeyOf(entity), Stored: true)).ToList(); + // SourceId is null for a stored row no row of the batch has claimed. + var kept = stored.Select(entity => (Key: KeyOf(entity), SourceId: (string?)null)).ToList(); var merges = new List<(string SourceId, string Detail)>(); + var inserted = new List<(string SourceId, TEntity Row)>(); foreach (var (sourceId, row) in rows) { @@ -41,20 +48,22 @@ static class KeyMerges if (match < 0) { - kept.Add((key, false)); + kept.Add((key, sourceId)); + inserted.Add((sourceId, row)); } - else if (kept[match].Stored && kept[match].Key.SequenceEqual(key)) + else if (kept[match].SourceId is null && kept[match].Key.SequenceEqual(key)) { // Claimed by this row, so a later row of the batch with the same key is a merge. - kept[match] = (key, false); + kept[match] = (key, sourceId); } else { - merges.Add((sourceId, $"{entityType.GetTableName()} treats {Describe(key)} and {Describe(kept[match].Key)} as one key, so it kept {Describe(kept[match].Key)} and dropped the row for {Describe(key)}")); + var keptRow = kept[match].SourceId is { } keptSourceId ? $"the row from {describeSource(keptSourceId)}" : "the row the table already held"; + merges.Add((sourceId, $"{entityType.GetTableName()} treats {Describe(key)} and {Describe(kept[match].Key)} as one key, so it kept {Describe(kept[match].Key)} and dropped the row for {Describe(key)}. It kept {keptRow} and dropped the row from {describeSource(sourceId)}.")); } } - return merges; + return (merges, inserted); } static Func<object?, object?, bool> TextMatch(IEqualityComparer<string> comparer) => (first, second) => comparer.Equals((string?)first, (string?)second); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs index cac5d17b6b..fc653b0d12 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs @@ -10,6 +10,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; /// <summary> /// Writes the licensing endpoints. An endpoint the target already holds, or a second one in the same batch whose key /// the column treats as the same, is left as it is and counted as already present. +/// Each endpoint folded into another's key is also reported as a merge, naming both. /// </summary> sealed class LicensingEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<Endpoint> { @@ -20,10 +21,10 @@ sealed class LicensingEndpointsWriter(IMigrationSqlDialect migrationDialect) : M public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => dbContext.LicensingEndpoints.LongCountAsync(cancellationToken); - protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, Endpoint Document)> documents, CancellationToken cancellationToken = default) + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, Endpoint Document)> documents, CancellationToken cancellationToken = default) { var insert = MigrationInsert<LicensingEndpointEntity>.For(dbContext); - var rows = new List<LicensingEndpointEntity>(documents.Count); + var rows = new List<(string SourceId, LicensingEndpointEntity Row)>(documents.Count); var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); foreach (var (row, endpoint) in documents.OrderBy(document => document.Row.SourceId, StringComparer.Ordinal)) @@ -42,13 +43,13 @@ protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext continue; } - rows.Add(entity); + rows.Add((row.SourceId, entity)); } - return Task.FromResult(new PreparedBatch( - async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, - Merges: [])); + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs index 5a03dd5353..eda0e848e0 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs @@ -9,13 +9,12 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.DataMigration; /// <summary> -/// Writes the licensing endpoints' daily message counts, setting each day's count rather than adding to it, so a copy -/// run twice leaves the same numbers. A batch naming an endpoint the target does not hold is refused with an -/// <see cref="InvalidOperationException" /> naming that endpoint and its source. A day repeated for one endpoint keeps -/// its last row. Days of different endpoints whose names the key column treats as the same are added into one row, -/// as the product's own throughput write would add them. Either way the rows folded away are counted as already present. +/// Adds each day's message count to the stored count for that day, and the database sums endpoints whose names its key treats as one. +/// A day repeated for one source endpoint keeps its last row, and a retry deletes every copied day first. +/// It reports no merge, because the endpoints writer reports an endpoint the key folds into another. +/// Throws <see cref="InvalidOperationException" /> when a batch names an endpoint the target does not hold. /// </summary> -sealed class LicensingThroughputWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<ThroughputDayRow> +sealed class LicensingThroughputWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<ThroughputDayRow>, IStoreKeyedCategoryWriter { public override string CategoryId => MigrationCategoryIds.LicensingThroughput; @@ -24,12 +23,13 @@ sealed class LicensingThroughputWriter(IMigrationSqlDialect migrationDialect) : public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => dbContext.LicensingEndpointThroughput.LongCountAsync(cancellationToken); + public Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpointThroughput.ExecuteDeleteAsync(cancellationToken); + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, ThroughputDayRow Document)> documents, CancellationToken cancellationToken = default) { var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); - var nameComparer = migrationDialect.KeyComparer(typeof(LicensingEndpointThroughputEntity), nameof(LicensingEndpointThroughputEntity.NormalizedName)); - var mapped = new List<LicensingEndpointThroughputEntity>(documents.Count); - var endpoints = new List<EndpointIdentifier>(documents.Count); + var mapped = new List<(EndpointIdentifier Endpoint, LicensingEndpointThroughputEntity Day)>(documents.Count); var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); // Date order is the order the product's own throughput write takes row locks in. @@ -49,11 +49,10 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo continue; } - mapped.Add(entity); - endpoints.Add(day.Endpoint); + mapped.Add((day.Endpoint, entity)); } - foreach (var endpoint in endpoints.Distinct()) + foreach (var endpoint in mapped.Select(day => day.Endpoint).Distinct()) { var normalizedName = LicensingDataStore.Normalize(endpoint.Name); @@ -67,19 +66,11 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo List<LicensingEndpointThroughputEntity> rows = [ .. mapped - .GroupBy(day => (day.ThroughputSource, day.DateUtc)) - .SelectMany(sameDay => sameDay.GroupBy(day => day.NormalizedName, nameComparer).Select(AddEndpoints)) + .GroupBy(day => (day.Endpoint.Name, day.Endpoint.ThroughputSource, day.Day.DateUtc)) + .Select(sameEndpointDay => sameEndpointDay.Last().Day) ]; - return new PreparedBatch((context, token) => migrationDialect.SetLicensingEndpointThroughput(context, rows, token), mapped.Count, skips, Merges: []); - } - - // The sum is set, not added to the stored count, so the reader must put every endpoint that can share a key in one batch. - static LicensingEndpointThroughputEntity AddEndpoints(IEnumerable<LicensingEndpointThroughputEntity> sameKey) - { - var perEndpoint = sameKey.GroupBy(day => day.NormalizedName, StringComparer.Ordinal).Select(sameEndpoint => sameEndpoint.Last()).ToList(); - var kept = perEndpoint[0]; - kept.MessageCount = perEndpoint.Sum(day => day.MessageCount); - return kept; + // Adding is safe to resume, because the batch and its checkpoint commit in one transaction, and a replayed checkpoint save is refused. + return new PreparedBatch((context, token) => migrationDialect.AddLicensingEndpointThroughput(context, rows, token), mapped.Count, skips, Merges: []); } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs new file mode 100644 index 0000000000..2c21cda8de --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the waiting integration events under new keys from the database, so a retry's reset empties the table before they are read again. +/// A request with no context, a type name longer than its column or a character the database refuses is skipped as RequiredValueMissing, ValueTooLong or MalformedValue. +/// The rest of the batch still copies. +/// </summary> +sealed class PendingIntegrationEventsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<ExternalIntegrationDispatchRequest>, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.PendingIntegrationEvents; + + public override int BatchSize(ServiceControlDbContext dbContext) => StoreKeyedInsert.BatchSize; + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.ExternalIntegrationDispatchRequests.LongCountAsync(cancellationToken); + + // The whole table, because the target is refused unless it held no ServiceControl data before the first copy. + public Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.ExternalIntegrationDispatchRequests.ExecuteDeleteAsync(cancellationToken); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, ExternalIntegrationDispatchRequest Document)> documents, CancellationToken cancellationToken = default) + { + var textColumns = StoreKeyedInsert.TextColumns<ExternalIntegrationDispatchRequestEntity>(dbContext); + List<ExternalIntegrationDispatchRequest> requests = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, request) in documents) + { + if (request.DispatchContext is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "ExternalIntegrationDispatchRequests.DispatchContextTypeName and DispatchContextJson are NOT NULL and the document has no context to serialize")); + } + else if (StoreKeyedInsert.UnstorableValueSkip(textColumns, migrationDialect, row, PendingIntegrationEventRowMapper.Map(request)) is { } unstorable) + { + skips.Add(unstorable); + } + else + { + requests.Add(request); + } + } + + return Task.FromResult(new PreparedBatch( + (context, token) => StoreKeyedInsert.AddAll(context, requests, PendingIntegrationEventRowMapper.Map, token), + requests.Count, + skips, + Merges: []) + { + RenumberedRows = requests.Count == 0 ? null : "pending integration events" + }); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs index e84dd7d149..63440ed63d 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs @@ -4,7 +4,6 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using Entities; using Infrastructure; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata; using ServiceControl.Persistence.DataMigration; using ServiceControl.Recoverability; @@ -29,7 +28,7 @@ public async Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, Cance protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, RetryOperationRow Document)> documents, CancellationToken cancellationToken = default) { var pendingInsert = MigrationInsert<UnacknowledgedRetryOperationEntity>.For(dbContext); - var historicText = HistoricTextColumns(dbContext); + var historicText = StoreKeyedInsert.TextColumns<HistoricRetryOperationEntity>(dbContext); List<HistoricRetryOperation> historic = []; List<(string SourceId, UnacknowledgedRetryOperationEntity Row)> unacknowledged = []; List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; @@ -42,7 +41,7 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo { skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it")); } - else if (UnstorableHistoricValue(historicText, row, RetryOperationRowMapper.Map(historicOperation)) is { } unstorable) + else if (StoreKeyedInsert.UnstorableValueSkip(historicText, migrationDialect, row, RetryOperationRowMapper.Map(historicOperation)) is { } unstorable) { skips.Add(unstorable); } @@ -81,44 +80,17 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo { var inserted = await migrationDialect.InsertMissing(context, [.. unacknowledged.Select(pending => pending.Row)], token); - // Fresh entities per attempt, because a failed save can leave generated ids on the previous ones. - // Reversed, so on a tie in completion time the first in the document still wins, as in RavenDB. - context.HistoricRetryOperations.AddRange(Enumerable.Reverse(historic).Select(RetryOperationRowMapper.Map)); - await context.SaveChangesAsync(token); + // Reversed so the first in the document is added last and gets the highest id, which wins a tie on completion time. + // That holds only while the store hands out ids in add order. + await StoreKeyedInsert.AddAll(context, [.. Enumerable.Reverse(historic)], (HistoricRetryOperation operation) => RetryOperationRowMapper.Map(operation), token); return historic.Count + inserted.Count; }, historic.Count + unacknowledged.Count, skips, - await KeyMerges.In(dbContext, migrationDialect, unacknowledged, cancellationToken)); - } - - // Read from the model, because MigrationInsert<T>.For refuses a table whose key the store generates. - static (IProperty Property, string TableAndColumn)[] HistoricTextColumns(ServiceControlDbContext dbContext) - { - var entityType = dbContext.Model.FindEntityType(typeof(HistoricRetryOperationEntity))!; - - return [.. entityType.GetProperties().Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{entityType.GetTableName()}.{property.GetColumnName()}"))]; - } - - (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableHistoricValue((IProperty Property, string TableAndColumn)[] text, MigrationRow row, HistoricRetryOperationEntity entity) - { - foreach (var (property, tableAndColumn) in text) - { - if (property.GetGetter().GetClrValue(entity) is string { Length: var units } value && property.GetMaxLength() is { } maxLength && units > maxLength && migrationDialect.TextLength(value) is var length && length > maxLength) - { - return (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}, on document '{row.SourceId}'"); - } - } - - foreach (var (property, tableAndColumn) in text) + await KeyMerges.In(dbContext, migrationDialect, unacknowledged, cancellationToken)) { - if (property.GetGetter().GetClrValue(entity) is string value && migrationDialect.RejectedCharacterIn(value) is { } rejected) - { - return (row.SourceId, MigrationSkipReason.MalformedValue, $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains, on document '{row.SourceId}'"); - } - } - - return null; + RenumberedRows = historic.Count == 0 ? null : "historic retry operations" + }; } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs new file mode 100644 index 0000000000..c21ac50312 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes rows whose primary key the database generates. +/// The model-driven insert refuses such rows, so this class adds them through EF in the caller's context, and the database assigns each key. +/// It also holds its own copy of the text check, because the base writer's check needs that insert. +/// </summary> +static class StoreKeyedInsert +{ + // EF splits its own statements under each provider's limits, so this sizes only how many rows one checkpoint covers. + public const int BatchSize = 500; + + /// <summary> + /// Returns every text column of <typeparamref name="TEntity" />, each with its name as "table.column" for a skip's detail. Read it once per batch and pass it to <see cref="UnstorableValueSkip{TEntity}" />. + /// </summary> + public static (IProperty Property, string TableAndColumn)[] TextColumns<TEntity>(ServiceControlDbContext dbContext) where TEntity : class + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; + + return [.. entityType.GetProperties().Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{entityType.GetTableName()}.{property.GetColumnName()}"))]; + } + + /// <summary> + /// Returns a skip when <paramref name="entity" />, the row mapped from the source document in <paramref name="row" />, holds a text value the database would refuse, or null when every value can be stored. + /// The reason is ValueTooLong for a value longer than its column, or MalformedValue for a value holding a character the provider cannot store. The skip carries the source id of <paramref name="row" />, and its detail names the table, the column and the document. + /// </summary> + public static (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableValueSkip<TEntity>((IProperty Property, string TableAndColumn)[] textColumns, IMigrationSqlDialect dialect, MigrationRow row, TEntity entity) where TEntity : class + { + foreach (var (property, tableAndColumn) in textColumns) + { + if (property.GetGetter().GetClrValue(entity) is string { Length: var units } value && property.GetMaxLength() is { } maxLength && units > maxLength && dialect.TextLength(value) is var length && length > maxLength) + { + return (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}, on document '{row.SourceId}'"); + } + } + + foreach (var (property, tableAndColumn) in textColumns) + { + if (property.GetGetter().GetClrValue(entity) is string value && dialect.RejectedCharacterIn(value) is { } rejected) + { + return (row.SourceId, MigrationSkipReason.MalformedValue, $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains, on document '{row.SourceId}'"); + } + } + + return null; + } + + /// <summary> + /// Builds a new entity for each source row with <paramref name="map" />, adds them all to the caller's context in the order given, and saves inside the transaction the target already holds. + /// Returns how many rows it added, which is all of them, because the save writes every row or throws. Call it inside the target's insert, so each attempt of the execution strategy builds fresh entities. + /// </summary> + public static async Task<int> AddAll<TSource, TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TSource> sources, Func<TSource, TEntity> map, CancellationToken cancellationToken = default) where TEntity : class + { + // Mapped on every call, because a failed save can leave generated ids on the entities it tried. + dbContext.Set<TEntity>().AddRange(sources.Select(map)); + await dbContext.SaveChangesAsync(cancellationToken); + + return sources.Count; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs index a748509255..29a57fd6c8 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs @@ -46,10 +46,14 @@ protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbCo rows.Add((row.SourceId, entity)); } + // The key leaves the version out, so a merge line names each row's version and endpoint as well as its source id. + var subscriptions = documents.ToDictionary(document => document.Row.SourceId, document => document.Document, StringComparer.Ordinal); + var (merges, _) = await KeyMerges.Split(dbContext, migrationDialect, rows, sourceId => $"'{sourceId}' (version {subscriptions[sourceId].Version}, endpoint '{subscriptions[sourceId].Endpoint}')", cancellationToken); + return new PreparedBatch( async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, - await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + merges); } } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs index 951088d8cb..fefcc7ee24 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -4,7 +4,7 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.Entities; /// <summary> -/// The migration's provider-specific SQL. Statements run inside the caller's transaction and insert only what is absent. +/// The migration's provider-specific SQL. Statements run inside the caller's transaction. /// </summary> public interface IMigrationSqlDialect { @@ -42,10 +42,10 @@ public interface IMigrationSqlDialect Task<IReadOnlyList<TEntity>> InsertMissing<TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TEntity> rows, CancellationToken cancellationToken = default) where TEntity : class; /// <summary> - /// Sets each day's message count: a day the table does not hold is inserted, and a day it holds gets its count replaced, never added to. + /// Adds each day's message count to the count the table holds for that day, and inserts a day the table does not hold. + /// Rows whose keys the key columns' collation treats as one are summed in the statement, so the database decides which names are one day. /// The rows go to the database in the order given. Every row's endpoint must already be in the table, or the foreign key fails the statement. - /// Two rows with the same key in one call are not allowed on PostgreSQL, which refuses the statement. SQL Server keeps the last of them, which also covers names that differ only by a trailing space. /// Returns how many rows the statements inserted or updated. It runs on the caller's open transaction, and throws <see cref="InvalidOperationException"/> when there is none or when a returned key matches no row given. /// </summary> - Task<int> SetLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default); + Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs index 67def01ba1..3f32f8227f 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs @@ -5,6 +5,8 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Raven.Client; using Raven.Client.Documents.Commands; using ServiceControl.MessageFailures; @@ -48,6 +50,37 @@ class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) }); }; + /// <summary> + /// Returns the batch with every retry-issued row carrying <see cref="MigrationMetadataKeys.RetryClaimHeld" />, true when + /// RavenDB still holds a retry claim for the message. It looks up every claim in the batch with one request, and returns a + /// batch with no retry-issued row as it is. + /// </summary> + public async Task<MigrationBatch> ResolveRetryClaims(MigrationBatch batch, CancellationToken cancellationToken = default) + { + string[] claimIds = [.. batch.Rows.Where(IsRetryIssued).Select(ClaimId)]; + + if (claimIds.Length == 0) + { + return batch; + } + + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + var claims = await session.LoadAsync<FailedMessageRetry>(claimIds.Distinct(StringComparer.OrdinalIgnoreCase), cancellationToken); + var held = claims.Where(claim => claim.Value is not null).Select(claim => claim.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); + + return batch with + { + Rows = [.. batch.Rows.Select(row => IsRetryIssued(row) + ? row with { Metadata = new Dictionary<string, object?>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = held.Contains(ClaimId(row)) } } + : row)] + }; + } + + static bool IsRetryIssued(MigrationRow row) => ((FailedMessage)row.Document).Status == FailedMessageStatus.RetryIssued; + + static string ClaimId(MigrationRow row) => RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(((FailedMessage)row.Document).UniqueMessageId); + // The translator checks TimeSent, SendingEndpoint and ReceivingEndpoint by exact type, but RavenDB can hand them back as text or untyped objects. // Only the newest attempt is fixed because the translator reads no other, and a TimeSent that is not a date is left as it is for the translator to skip. internal static void RestoreMetadataTypes(FailedMessage document) @@ -98,7 +131,7 @@ static void RestoreEndpoint(Dictionary<string, object> metadata, string key, Fun static DateTime AsUtc(DateTime value) => value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); - static DateTime? ReadMetadataDate(StreamResult<FailedMessage> result, string key) => + internal static DateTime? ReadMetadataDate<TDocument>(StreamResult<TDocument> result, string key) => result.Metadata.TryGetValue(key, out var value) && value != null ? AsUtc(DateTime.Parse(value.ToString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)) : null; diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index 18266881e0..a77fe39612 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -36,7 +36,11 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig new SubscriptionsReader(lifecycle), new NotificationSettingsReader(lifecycle), new TrialEndDateReader(lifecycle), - new RetryOperationsReader(lifecycle) + new RetryOperationsReader(lifecycle), + new PendingIntegrationEventsReader(lifecycle), + new CustomChecksReader(lifecycle), + new FailedErrorImportsReader(lifecycle), + new GroupCommentsReader(lifecycle) }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs new file mode 100644 index 0000000000..979be4d817 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs @@ -0,0 +1,38 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the custom check results out of the primary database, each one turned into the shared <see cref="CustomCheckDetail" />, a type the target can reference. +/// A null value in the stored document stays null, so the writer can name the column it would fill. +/// A result of one of the RavenDB persister's own checks carries the SourceStorageCheck metadata value, so the target can skip it. +/// </summary> +sealed class CustomChecksReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<CustomCheckDetail>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.CustomChecks; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix<CustomCheck>(Lifecycle, CategoryId, Lifecycle.Settings.DatabaseName, $"{RavenCustomCheckDataStore.CollectionName}/", resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, + static result => new MigrationRow(result.Id, ToDetail(result.Document), MetadataFor(result.Document)), cancellationToken); + + static IReadOnlyDictionary<string, object?> MetadataFor(CustomCheck document) => + document.CustomCheckId is { } id && document.Category is { } category && RavenStorageCheckIds.All.Contains((id, category)) + ? new Dictionary<string, object?> { [MigrationMetadataKeys.SourceStorageCheck] = true } + : ReadOnlyDictionary<string, object?>.Empty; + + static CustomCheckDetail ToDetail(CustomCheck document) => new() + { + OriginatingEndpoint = document.OriginatingEndpoint!, + CustomCheckId = document.CustomCheckId!, + ReportedAt = document.ReportedAt, + Category = document.Category!, + HasFailed = document.Status == Status.Fail, + FailureReason = document.FailureReason + }; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs new file mode 100644 index 0000000000..c45b0e0eac --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs @@ -0,0 +1,30 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using Raven.Client; +using Raven.Client.Documents.Commands; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the messages ServiceControl took off the error queue but could not store, each with its body and the time it failed. +/// </summary> +sealed class FailedErrorImportsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<FailedErrorImport>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.FailedErrorImports; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + Documents($"{FailedErrorImportDataStore.CollectionName}/", resumeAfter, batchSize, ToRow, cancellationToken); + + // A document with no message still gets an empty body, because the engine would otherwise ask ReadBody, which refuses this category. + static MigrationRow ToRow(StreamResult<FailedErrorImport> result) => new( + result.Id, + result.Document, + new Dictionary<string, object?> { [MigrationMetadataKeys.FailedAt] = FailedMessageStream.ReadMetadataDate(result, Constants.Documents.Metadata.LastModified) }, + new MigrationBody( + result.Document.Message?.Body ?? [], + result.Document.Message?.Headers?.GetValueOrDefault(NServiceBus.Headers.ContentType) ?? "application/octet-stream")); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs index 4cc1393ac9..1d32916de9 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs @@ -41,7 +41,8 @@ sealed class FailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : Migr public override async IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var databaseName = Lifecycle.Settings.DatabaseName; - var project = new FailedMessageStream(Lifecycle).Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses); + var stream = new FailedMessageStream(Lifecycle); + var project = stream.Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses); var batches = await IndexHoldsEveryCopiedFailedMessage(databaseName, cancellationToken) ? ThroughIndex(databaseName, resumeAfter, batchSize, project, cancellationToken) @@ -49,7 +50,7 @@ public override async IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, await foreach (var batch in batches) { - yield return batch; + yield return await stream.ResolveRetryClaims(batch, cancellationToken); } } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs new file mode 100644 index 0000000000..1397a35a4d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs @@ -0,0 +1,23 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; + +/// <summary> +/// Reads the comments operators wrote on failure groups. Each row carries the bare group id, because SQL keys a comment on the group id while RavenDB stores it under its own document id. +/// </summary> +sealed class GroupCommentsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<GroupComment>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.GroupComments; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + Documents(GroupsDataStore.CommentIdPrefix, resumeAfter, batchSize, + static result => new MigrationRow(result.Id, new GroupComment { Id = result.Id[GroupsDataStore.CommentIdPrefix.Length..], Comment = result.Document.Comment }, ReadOnlyDictionary<string, object?>.Empty), + cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs index 0b87d2ee76..a77b53067c 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs @@ -15,8 +15,8 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; using ServiceControl.Persistence.RavenDB.Throughput; /// <summary> -/// Reads each licensing endpoint's daily message counts out of the throughput database, one batch per group from -/// <see cref="ThroughputDocuments.EndpointGroups" />, with the batch's cursor on the group's last document id. +/// Reads each licensing endpoint's daily message counts out of the throughput database, one batch per endpoint in +/// ordinal document-id order, with the batch's cursor on that endpoint's document id. /// </summary> sealed class LicensingThroughputReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<ThroughputDayRow>(lifecycle) { @@ -27,13 +27,12 @@ public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int b async IAsyncEnumerable<MigrationBatch> ReadThroughputDays(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) { - // One group of endpoints per batch, because the batch size counts rows and one endpoint can bring a year or more of days, - // and because the writer adds the days of a group's endpoints only when they arrive in one batch. - await foreach (var endpoints in ThroughputDocuments.EndpointGroups(Lifecycle, CategoryId, resumeAfter, cancellationToken)) + // One endpoint per batch, because the batch size counts rows and one endpoint can bring a year or more of days. + await foreach (var endpoints in ThroughputDocuments.Endpoints(Lifecycle, CategoryId, resumeAfter, batchSize: 1, cancellationToken)) { var days = new List<MigrationRow>(); - foreach (var endpoint in endpoints) + foreach (var endpoint in endpoints.Rows) { var id = ((Endpoint)endpoint.Document).Id; @@ -43,7 +42,7 @@ async IAsyncEnumerable<MigrationBatch> ReadThroughputDays(string? resumeAfter, [ } } - yield return new MigrationBatch(days, endpoints[^1].SourceId); + yield return new MigrationBatch(days, endpoints.Cursor); } } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs new file mode 100644 index 0000000000..e1e9775e64 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the integration events the old instance had not yet dispatched, whole, out of the primary database. +/// </summary> +sealed class PendingIntegrationEventsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<ExternalIntegrationDispatchRequest>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.PendingIntegrationEvents; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(ExternalIntegrationRequestsDataStore.KeyPrefix + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs new file mode 100644 index 0000000000..bd92c28dbf --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Frozen; +using System.Collections.Generic; + +/// <summary> +/// The six custom checks the RavenDB persister runs about its own database, each as its id and the category it reports under. +/// The custom checks reader flags a result only when both match, so a result that reuses one of these ids under another category is not flagged, such as the audit instance's "RavenDB dirty memory" under "ServiceControl.Audit Health". +/// </summary> +static class RavenStorageCheckIds +{ + // Each check passes its id and category to the CustomCheck base constructor as literals, so there is nothing to reference. RavenStorageCheckIdsTests fails if the two lists differ. + public static IReadOnlySet<(string Id, string Category)> All { get; } = new[] + { + ("Error Database Index Errors", "ServiceControl Health"), + ("Error Database Index Lag", "ServiceControl Health"), + ("ServiceControl database", "Storage space"), + ("Message Ingestion Process", "ServiceControl Health"), + ("RavenDB dirty memory", "ServiceControl Health"), + ("Error Database Search Engine", "ServiceControl Health") + }.ToFrozenSet(); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs index 1207f6b3fb..77701e8a65 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs @@ -9,7 +9,6 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Particular.LicensingComponent.Contracts; using Raven.Client; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Conventions; @@ -46,37 +45,6 @@ public static async IAsyncEnumerable<MigrationBatch> Endpoints( } } - /// <summary> - /// Reads the endpoints in groups whose names can share a key in SQL: the same source, and the same name ignoring - /// case and trailing spaces. Groups come in the ordinal order of their last document id, which is each group's cursor, - /// and only groups after <paramref name="resumeAfter" /> are read. Throws <see cref="InvalidOperationException" /> when - /// the source holds no document with the id in <paramref name="resumeAfter" />. - /// </summary> - public static async IAsyncEnumerable<IReadOnlyList<MigrationRow>> EndpointGroups( - RavenReadOnlySourceLifecycle lifecycle, - string categoryId, - string? resumeAfter, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var groups = (await AllEndpoints(lifecycle, categoryId, resumeAfter, cancellationToken)) - .GroupBy(SharedKey) - .Select(group => group.OrderBy(row => row.SourceId, StringComparer.Ordinal).ToList()) - .Where(group => resumeAfter is null || string.CompareOrdinal(group[^1].SourceId, resumeAfter) > 0) - .OrderBy(group => group[^1].SourceId, StringComparer.Ordinal); - - foreach (var group in groups) - { - yield return group; - } - } - - // SQL Server's default collation ignores case and trailing spaces, and the product lower-cases the name it stores. - static (string? Name, ThroughputSource Source) SharedKey(MigrationRow row) - { - var id = ((Endpoint)row.Document).Id; - return (id.Name?.TrimEnd(' ').ToLowerInvariant(), id.ThroughputSource); - } - // With no id prefix RavenDB streams in reverse etag order and ignores startAfter, so callers order the rows and apply the cursor. static async Task<List<MigrationRow>> AllEndpoints(RavenReadOnlySourceLifecycle lifecycle, string categoryId, string? resumeAfter, CancellationToken cancellationToken) { diff --git a/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs index d9848ccb28..c1365095bc 100644 --- a/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs @@ -39,7 +39,7 @@ public ExternalIntegrationRequestsDataStore( ); } - const string KeyPrefix = "ExternalIntegrationDispatchRequests"; + internal const string KeyPrefix = "ExternalIntegrationDispatchRequests"; public async Task StoreDispatchRequest(IEnumerable<ExternalIntegrationDispatchRequest> dispatchRequests, CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs index e9cfd8f89d..acbc5081eb 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs @@ -44,7 +44,9 @@ public async Task<CheckStateChange> UpdateCustomCheckStatus(CustomCheckDetail de return status; } - static string MakeId(Guid id) => $"CustomChecks/{id}"; + internal const string CollectionName = "CustomChecks"; + + static string MakeId(Guid id) => $"{CollectionName}/{id}"; public async Task<QueryResult<IList<CustomCheckView>>> GetStats(PagingInfo paging, string status = null, CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs index 92fcfdf898..284700ead9 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs @@ -146,6 +146,8 @@ public async Task DeleteComment(string groupId, CancellationToken cancellationTo await session.SaveChangesAsync(cancellationToken); } - public static string MakeId(string groupId) => $"GroupComment/{groupId}"; + internal const string CommentIdPrefix = "GroupComment/"; + + public static string MakeId(string groupId) => $"{CommentIdPrefix}{groupId}"; } } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs index 07c39bd01b..2c38adfcb1 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs @@ -6,8 +6,10 @@ namespace ServiceControl.Persistence.Tests; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.EFCore.Entities; using ServiceControl.Persistence.EFCore.Infrastructure; @@ -77,5 +79,85 @@ public async Task A_value_holding_a_NUL_character_is_skipped_as_malformed_naming } } + [Test] + public async Task A_custom_check_whose_failure_reason_holds_a_NUL_character_is_skipped_as_malformed_and_the_rest_copy() + { + var endpoint = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SALES01" }; + var batch = new MigrationBatch( + [ + new MigrationRow("CustomChecks/1", CustomCheck(endpoint, "Sales.HealthCheck", "timed out"), new Dictionary<string, object>()), + new MigrationRow("CustomChecks/2", CustomCheck(endpoint, "Sales.DiskCheck", "disk \u0000 full"), new Dictionary<string, object>()), + new MigrationRow("CustomChecks/3", CustomCheck(endpoint, "Sales.QueueCheck", null), new Dictionary<string, object>()) + ], + "CustomChecks/3"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(CustomChecksCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.CustomChecks, MigrationCategoryState.InProgress, "CustomChecks/3", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<CustomCheckEntity>(ServiceProvider, nameof(CustomCheckEntity.FailureReason))} cannot hold the character U+0000, which the value contains, on document 'CustomChecks/2'" + })); + Assert.That(await Target.Count(CustomChecksCategory), Is.EqualTo(2)); + } + } + + [Test] + public async Task A_failed_error_import_whose_exception_info_holds_a_NUL_character_is_skipped_as_malformed_and_the_rest_copy() + { + var batch = new MigrationBatch( + [ + FailedErrorImportRow("native-1", "Deserialization failed"), + FailedErrorImportRow("native-2", "Deserialization \u0000 failed"), + FailedErrorImportRow("native-3", "Deserialization failed") + ], + "FailedErrorImports/native-3"); + var writer = new FailedErrorImportsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), ServiceProvider.GetRequiredService<IBodyStoragePersistence>(), ServiceProvider.GetRequiredService<EFPersisterSettings>()); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, writer, batch); + var result = await Target.Write(FailedErrorImportsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.FailedErrorImports, MigrationCategoryState.InProgress, "FailedErrorImports/native-3", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<FailedErrorImportEntity>(ServiceProvider, nameof(FailedErrorImportEntity.ExceptionInfo))} cannot hold the character U+0000, which the value contains, on document 'FailedErrorImports/native-2'" + })); + Assert.That(await Target.Count(FailedErrorImportsCategory), Is.EqualTo(2)); + } + } + + static readonly MigrationCategory FailedErrorImportsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + // The NUL goes in ExceptionInfo, because the headers are written as JSON, which escapes it. + static MigrationRow FailedErrorImportRow(string nativeMessageId, string exceptionInfo) => new( + $"FailedErrorImports/{nativeMessageId}", + new FailedErrorImport + { + Id = $"FailedErrorImports/{nativeMessageId}", + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = nativeMessageId }, Body = [1, 2, 3] }, + ExceptionInfo = exceptionInfo + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc) }, + new MigrationBody(new byte[] { 1, 2, 3 }, "application/octet-stream")); + + static readonly MigrationCategory CustomChecksCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; + + static CustomCheckDetail CustomCheck(EndpointDetails endpoint, string customCheckId, string failureReason) => new() + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = "Ping", + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + HasFailed = failureReason is not null, + FailureReason = failureReason + }; + static string Repeat(int codePoints) => string.Concat(Enumerable.Repeat(OutsideTheBasicPlane, codePoints)); } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs new file mode 100644 index 0000000000..75d1b0bc09 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +class CustomCheckSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_a_custom_check_as_the_neutral_detail_type_with_its_id_recoverable() + { + var endpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + var expectedId = await SeedCustomCheck(endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "CustomChecks"); + + await using var source = await OpenMigrationSource(); + + var detail = (CustomCheckDetail)(await CollectBatches(source, category)).SelectMany(batch => batch.Rows).Single().Document; + + using (Assert.EnterMultipleScope()) + { + Assert.That(detail.GetDeterministicId(), Is.EqualTo(expectedId), "the id must be recoverable from the same fields RavenDB used to generate it"); + Assert.That(detail.CustomCheckId, Is.EqualTo("Sales.HealthCheck")); + Assert.That(detail.HasFailed, Is.True); + Assert.That(detail.FailureReason, Is.EqualTo("timed out")); + } + } + + [Test] + public async Task Flags_only_a_row_whose_id_and_category_are_one_of_the_persisters_own_checks() + { + var endpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + await SeedCustomCheck(endpoint, "Error Database Index Lag", "ServiceControl Health", failed: false, reason: null); + await SeedCustomCheck(new EndpointDetails { Name = "Billing", Host = "MACHINE", HostId = Guid.NewGuid() }, "Error Database Index Lag", "Another category", failed: false, reason: null); + await SeedCustomCheck(endpoint, "Sales.HealthCheck", "Ping", failed: false, reason: null); + await SeedCustomCheck(endpoint, "Error Message Ingestion Process", "ServiceControl Health", failed: false, reason: null); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "CustomChecks"); + + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToDictionary(row => ((CustomCheckDetail)row.Document).CustomCheckId + "|" + ((CustomCheckDetail)row.Document).Category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows["Error Database Index Lag|ServiceControl Health"].Metadata[MigrationMetadataKeys.SourceStorageCheck], Is.True); + Assert.That(rows["Error Database Index Lag|Another category"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck), "the same id under another category is not the persister's check"); + Assert.That(rows["Error Message Ingestion Process|ServiceControl Health"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck), "another id under the persister's category is not the persister's check"); + Assert.That(rows["Sales.HealthCheck|Ping"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck)); + } + } + + async Task<Guid> SeedCustomCheck(EndpointDetails endpoint, string customCheckId, string category, bool failed, string reason) + { + var detail = new CustomCheckDetail + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = category, + ReportedAt = Now, + HasFailed = failed, + FailureReason = reason + }; + + await CustomChecks.UpdateCustomCheckStatus(detail); + + return detail.GetDeterministicId(); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs new file mode 100644 index 0000000000..27a23e01bf --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +class FailedErrorImportSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Every_import_arrives_with_its_body_attached_and_the_time_RavenDB_last_modified_it() + { + await FailedImportStore.StoreFailedErrorImport(new FailedErrorImport + { + Id = "native-1", + Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }); + await FailedImportStore.StoreFailedErrorImport(new FailedErrorImport { Id = "native-2", Message = null, ExceptionInfo = "Deserialization failed" }); + await CompleteDatabaseOperation(); + + DateTime lastModified; + + using (var session = await SessionProvider.OpenSession()) + { + lastModified = session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedErrorImport>(FailedErrorImportDataStore.MakeDocumentId("native-1"))).Value; + } + + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports))).SelectMany(batch => batch.Rows).ToDictionary(row => row.SourceId); + var withMessage = rows[FailedErrorImportDataStore.MakeDocumentId("native-1")]; + var withoutMessage = rows[FailedErrorImportDataStore.MakeDocumentId("native-2")]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(withMessage.Body!.Content.ToArray(), Is.EqualTo(new byte[] { 1, 2, 3 }), "attached in Read, so the engine never asks ReadBody, which refuses this category"); + Assert.That(withMessage.Body.ContentType, Is.EqualTo("application/octet-stream"), "the default the SQL import store gives a message with no content type header"); + Assert.That(withMessage.Metadata[MigrationMetadataKeys.FailedAt], Is.EqualTo(lastModified.ToUniversalTime()), "RavenDB keeps no failure time, only the stamp it wrote when the import was stored"); + Assert.That(((DateTime)withMessage.Metadata[MigrationMetadataKeys.FailedAt]!).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(withoutMessage.Body?.Content.Length, Is.Zero, "an empty body rather than none, so the writer skips the row instead of the engine stopping on ReadBody"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs new file mode 100644 index 0000000000..af5f2d510d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; + +class GroupCommentsSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments); + + [Test] + public async Task Reads_each_stored_comment_keyed_by_its_group_id_a_cleared_one_included() + { + await GroupsStore.EditComment("group-1", "Known issue, ignore until Thursday"); + await GroupsStore.EditComment("group-2", ""); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + var rows = (await CollectBatches(source, Category)).SelectMany(batch => batch.Rows).ToList(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { GroupsDataStore.MakeId("group-1"), GroupsDataStore.MakeId("group-2") }), "the cursor and every skip name the RavenDB document id"); + Assert.That(rows.Select(row => ((GroupComment)row.Document).Id), Is.EqualTo(new[] { "group-1", "group-2" }), "SQL keys a comment on the bare group id, so that is what crosses"); + Assert.That(rows.Select(row => ((GroupComment)row.Document).Comment), Is.EqualTo(new[] { "Known issue, ignore until Thursday", "" }), "a cleared comment is read, so the writer counts its skip instead of it vanishing uncounted"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs index 065b0fa62e..7497fef6f8 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs @@ -72,7 +72,7 @@ public async Task A_resume_reads_only_the_endpoints_after_the_cursor_whatever_or } [Test] - public async Task Endpoints_whose_names_differ_only_in_case_and_trailing_spaces_come_in_one_batch() + public async Task Endpoints_whose_names_differ_only_in_case_and_trailing_spaces_come_in_separate_batches() { foreach (var name in new[] { "Sales.Orders", "Sales.Orders.Archive", "SALES.ORDERS " }) { @@ -93,9 +93,9 @@ public async Task Endpoints_whose_names_differ_only_in_case_and_trailing_spaces_ using (Assert.EnterMultipleScope()) { - Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "Sales.Orders.Archive/Broker", "Sales.Orders/Broker" }), "each group cursors on its last id in ordinal order, so a resume never splits a group"); - Assert.That(batches[1].Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EquivalentTo(new[] { "Sales.Orders", "SALES.ORDERS " }), "the writer can add the two endpoints' days only when they arrive in one batch"); - Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EquivalentTo(new[] { "Sales.Orders", "SALES.ORDERS " }), "a resume after the first group reads the whole second group"); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "SALES.ORDERS /Broker", "Sales.Orders.Archive/Broker", "Sales.Orders/Broker" }), "the source groups by its own document id only, and which names are one key is the target's to decide"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name).Single()), Is.EqualTo(new[] { "SALES.ORDERS ", "Sales.Orders.Archive", "Sales.Orders" }), "each batch holds the days of the one endpoint its cursor names"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "Sales.Orders.Archive", "Sales.Orders" }), "a resume reads only the endpoints after the cursor, even one whose name differs from the cursor's only in case and spaces"); } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs new file mode 100644 index 0000000000..e5afca8d87 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +class PendingIntegrationEventsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_each_queued_request_with_its_context_as_the_concrete_type() + { + await ExternalIntegrationStore.StoreDispatchRequest( + [ + new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "one" } }, + new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "two" } } + ]); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.PendingIntegrationEvents); + + await using var source = await OpenMigrationSource(); + var contexts = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).Select(row => ((ExternalIntegrationDispatchRequest)row.Document).DispatchContext).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(contexts, Has.All.InstanceOf<TestDispatchContext>(), "an untyped context would serialize as the JSON library's own type and never dispatch"); + Assert.That(contexts.Cast<TestDispatchContext>().Select(context => context.Value), Is.EquivalentTo(new[] { "one", "two" })); + Assert.That(await source.Count(category), Is.EqualTo(2)); + } + } + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index 67a81ea63f..21502d2d23 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -9,6 +9,8 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using NUnit.Framework; using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.ExternalIntegrations; using ServiceControl.Infrastructure.RavenDB.Subscriptions; using ServiceControl.Notifications; using ServiceControl.Operations; @@ -143,7 +145,11 @@ public async Task Reading_every_category_creates_no_index() [MigrationCategoryIds.Subscriptions] = tests => tests.SeedSubscription(), [MigrationCategoryIds.NotificationSettings] = tests => tests.NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), [MigrationCategoryIds.TrialEndDate] = tests => tests.ServiceProvider.GetRequiredService<ITrialLicenseDataProvider>().StoreTrialEndDate(new DateOnly(2027, 3, 15)), - [MigrationCategoryIds.RetryOperations] = tests => tests.RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10) + [MigrationCategoryIds.RetryOperations] = tests => tests.RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10), + [MigrationCategoryIds.PendingIntegrationEvents] = tests => tests.ExternalIntegrationStore.StoreDispatchRequest([new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "seed" } }]), + [MigrationCategoryIds.CustomChecks] = tests => tests.CustomChecks.UpdateCustomCheckStatus(new CustomCheckDetail { OriginatingEndpoint = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }, CustomCheckId = "Sales.HealthCheck", Category = "Ping" }), + [MigrationCategoryIds.FailedErrorImports] = tests => tests.FailedImportStore.StoreFailedErrorImport(new FailedErrorImport { Id = "native-1", Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, ExceptionInfo = "Deserialization failed" }), + [MigrationCategoryIds.GroupComments] = tests => tests.GroupsStore.EditComment("group-1", "Known issue, ignore until Thursday") }; string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; @@ -188,4 +194,9 @@ async Task SeedEndpointSettings(int count) await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = $"Endpoint{index}", TrackInstances = true }); } } + + public class TestDispatchContext + { + public string Value { get; set; } + } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs new file mode 100644 index 0000000000..eefcf35846 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.CustomChecks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +class RavenStorageCheckIdsTests : RavenPersistenceTestBase +{ + [Test] + public void Lists_exactly_the_checks_the_RavenDB_persister_registers_with_their_categories() => + Assert.That(ServiceProvider.GetServices<ICustomCheck>().Select(check => (check.Id, check.Category)), Is.EquivalentTo(RavenStorageCheckIds.All), + "the custom checks reader flags exactly these pairs, so a check the persister adds, renames or moves to another category would be copied to SQL, where nothing reports it again"); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs index f2b5f8bc11..9faec057e1 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs @@ -13,6 +13,7 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; using ServiceControl.Persistence.RavenDB.DataMigration; [TestFixture] @@ -99,6 +100,32 @@ public async Task Sends_only_the_migration_metadata_keys_across_the_seam() Assert.That(row.Metadata.Keys, Is.EquivalentTo(new[] { MigrationMetadataKeys.StatusChangedAt, MigrationMetadataKeys.LastModified }), "a store's own metadata, such as @expires, never travels"); } + [Test] + public async Task A_retry_issued_message_says_whether_the_source_still_holds_its_retry_claim() + { + var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var released = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + using (var session = DocumentStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = unresolved.Id, RetryBatchId = "RetryBatches/staging" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(unresolved.UniqueMessageId)); + await session.SaveChangesAsync(); + } + + await CompleteDatabaseOperation(); + + var rows = (await ReadAll()).ToDictionary(row => ((FailedMessage)row.Document).UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows[claimed.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True); + Assert.That(rows[released.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.False); + Assert.That(rows[unresolved.UniqueMessageId].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.RetryClaimHeld), "only a retry-issued message gets a claim in SQL, even when the source holds one for another"); + } + } + [Test] public async Task Carries_the_documents_own_last_modified() { @@ -258,10 +285,21 @@ public async Task Reads_the_whole_collection_when_the_index_is_missing() { await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + + using (var session = DocumentStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); + await session.SaveChangesAsync(); + } var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); - Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { stored.Id, claimed.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); + Assert.That(rows.Single(row => row.SourceId == claimed.Id).Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True, "the whole-collection read must resolve claims too, or the writer stops on the first retry-issued row"); + } } [Test] diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs index 9a081a0b2b..b611b22fc6 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -166,34 +166,4 @@ public async Task The_key_columns_own_collation_decides_a_merge_when_the_databas "the dry run predicts merges with this comparer, so it follows the same column the statement does"); } } - - // RecordingLoggerProvider lives in ServiceControl.Infrastructure.Tests, which this project does not reference. - sealed class TargetWarnings : ILoggerProvider - { - readonly System.Collections.Concurrent.ConcurrentQueue<string> messages = new(); - - public IReadOnlyList<string> Messages => [.. messages]; - - public ILogger CreateLogger(string categoryName) => - categoryName == typeof(EFCoreMigrationTarget).FullName ? new WarningLogger(messages) : Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance; - - public void Dispose() - { - } - - sealed class WarningLogger(System.Collections.Concurrent.ConcurrentQueue<string> messages) : ILogger - { - public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Warning; - - public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) - { - if (logLevel == LogLevel.Warning) - { - messages.Enqueue(formatter(state, exception)); - } - } - } - } } diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs new file mode 100644 index 0000000000..c10838be08 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs @@ -0,0 +1,138 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingThroughputKeyCollationTests : PersistenceTestBase +{ + [Test] + public async Task Days_of_endpoints_an_accent_insensitive_key_treats_as_one_are_added_within_a_batch_and_across_batches() + { + await OpenAnAccentInsensitiveTarget(); + await LicensingDataStore.SaveEndpoint(new Endpoint("Café", ThroughputSource.Broker) { SanitizedName = "cafe" }); + var day = new DateOnly(2026, 1, 1); + + var first = await Write( + new MigrationCheckpoint(MigrationCategoryIds.LicensingThroughput, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null), + ("Café/Broker|2026-01-01", "Café", day, 100), + ("Cafe/Broker|2026-01-01", "Cafe", day, 250)); + await Write(first.Saved, ("CAFE/Broker|2026-01-01", "CAFE", day, 50)); + + long[] stored; + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().LicensingEndpointThroughput.Select(row => row.MessageCount).ToArrayAsync(); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(1), "the column's collation makes the two names one day, which the statement writes once"); + Assert.That(first.AlreadyPresent, Is.EqualTo(1), "the day folded into the other is counted as already present and summed into the kept row"); + Assert.That(stored, Is.EqualTo(new[] { 400L }), "every count for the day is added under the key column's own collation, whichever batch it came in"); + } + } + + [Test] + public async Task Endpoints_an_accent_insensitive_key_treats_as_one_are_reported_as_a_merge() + { + await OpenAnAccentInsensitiveTarget(); + var batch = new MigrationBatch( + [ + new MigrationRow("Café/Broker", new Endpoint("Café", ThroughputSource.Broker) { SanitizedName = "cafe" }, new Dictionary<string, object>()), + new MigrationRow("Cafe/Broker", new Endpoint("Cafe", ThroughputSource.Broker) { SanitizedName = "cafe" }, new Dictionary<string, object>()) + ], + "Cafe/Broker"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await ServiceProvider.GetRequiredService<IMigrationTarget>().Write( + MigrationCategoryRegistry.Find(MigrationCategoryIds.LicensingEndpoints)!, + batch, + new MigrationCheckpoint(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "Café/Broker" }), "the copy keeps the endpoint whose source id sorts first, and the column folds the accent, so the other is a merge"); + Assert.That(merges.Single().Detail, Does.Contain("kept 'cafe', 'Broker' and dropped the row for 'café', 'Broker'")); + Assert.That((result.Copied, result.AlreadyPresent, result.Merged), Is.EqualTo((1, 1, 1)), "every row the insert drops on a first copy is a merge the log names"); + } + } + + [TestCase("Café", "Cafe")] + [TestCase("cafe", "cafe")] + [TestCase("CAFE", "cafe")] + [TestCase("cafe ", "cafe")] + [TestCase("cafe", "cafes")] + public async Task The_key_comparer_folds_exactly_what_the_columns_collation_folds(string first, string second) + { + await OpenAnAccentInsensitiveTarget(); + bool sqlServerFolds; + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + sqlServerFolds = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database + .SqlQuery<int>($"SELECT CASE WHEN {first} = {second} COLLATE Latin1_General_CI_AI THEN 1 ELSE 0 END AS [Value]") + .SingleAsync() == 1; + } + + var comparer = ServiceProvider.GetRequiredService<IMigrationSqlDialect>().KeyComparer(typeof(LicensingEndpointEntity), nameof(LicensingEndpointEntity.NormalizedName)); + + Assert.That(comparer.Equals(first, second), Is.EqualTo(sqlServerFolds), $"the merge lines and the dry run predict with this comparer, so it must agree with SQL Server on '{first}' and '{second}'"); + } + + // Set on both tables, because the writer looks up each day's endpoint by name before it writes, and the test server's default respects accents. + async Task OpenAnAccentInsensitiveTarget() + { + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var endpoints = TableOf<LicensingEndpointEntity>(dbContext); + var throughput = TableOf<LicensingEndpointThroughputEntity>(dbContext); + + var ignoreAccents = $""" + ALTER TABLE {throughput} DROP CONSTRAINT [FK_LicensingEndpointThroughput_LicensingEndpoints_NormalizedName_ThroughputSource]; + ALTER TABLE {throughput} DROP CONSTRAINT [PK_LicensingEndpointThroughput]; + ALTER TABLE {endpoints} DROP CONSTRAINT [PK_LicensingEndpoints]; + ALTER TABLE {endpoints} ALTER COLUMN [NormalizedName] nvarchar(300) COLLATE Latin1_General_CI_AI NOT NULL; + ALTER TABLE {throughput} ALTER COLUMN [NormalizedName] nvarchar(300) COLLATE Latin1_General_CI_AI NOT NULL; + ALTER TABLE {endpoints} ADD CONSTRAINT [PK_LicensingEndpoints] PRIMARY KEY ([NormalizedName], [ThroughputSource]); + ALTER TABLE {throughput} ADD CONSTRAINT [PK_LicensingEndpointThroughput] PRIMARY KEY ([NormalizedName], [ThroughputSource], [DateUtc]); + ALTER TABLE {throughput} ADD CONSTRAINT [FK_LicensingEndpointThroughput_LicensingEndpoints_NormalizedName_ThroughputSource] + FOREIGN KEY ([NormalizedName], [ThroughputSource]) REFERENCES {endpoints} ([NormalizedName], [ThroughputSource]) ON DELETE CASCADE; + """; + + await dbContext.Database.ExecuteSqlRawAsync(ignoreAccents); + } + + await ServiceProvider.GetRequiredService<IMigrationTarget>().Open(); + } + + Task<MigrationWriteResult> Write(MigrationCheckpoint checkpointToExtend, params (string SourceId, string Name, DateOnly Day, long Count)[] days) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingThroughput); + var batch = new MigrationBatch( + [.. days.Select(day => new MigrationRow(day.SourceId, new ThroughputDayRow(new EndpointIdentifier(day.Name, ThroughputSource.Broker), day.Day, day.Count), new Dictionary<string, object>()))], + days[^1].SourceId); + + return ServiceProvider.GetRequiredService<IMigrationTarget>().Write(category, batch, checkpointToExtend); + } + + // Named from the model, because the table sits in the configured schema when the persister has one. + static string TableOf<TEntity>(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity)); + return dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs new file mode 100644 index 0000000000..b23deff4cf --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class CustomCheckRowMapperTests : PersistenceTestBase +{ + [Test] + public void Sets_every_mapped_property_of_a_custom_check() + { + var entity = CustomCheckRowMapper.Map(new CustomCheckDetail + { + OriginatingEndpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }, + CustomCheckId = "Sales.HealthCheck", + Category = "Ping", + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + // Passing, because Status.Fail is the enum's zero and would read as unset. + HasFailed = false, + FailureReason = "recovered after a timeout" + }); + + using var scope = ServiceProvider.CreateScope(); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs new file mode 100644 index 0000000000..4927d099d0 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs @@ -0,0 +1,142 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; + +class CustomChecksMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; + + static readonly EndpointDetails Endpoint = new() { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + + [Test] + public async Task A_check_the_target_already_holds_is_counted_as_already_present_and_keeps_its_stored_status() + { + var first = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: false))), CheckpointAfter("CustomChecks/1")); + var second = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: true))), first.Saved); + + var stored = (await CustomChecks.GetStats(new PagingInfo())).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.Zero); + Assert.That(second.AlreadyPresent, Is.EqualTo(1), "a replayed batch finds its row already there, where a plain insert would fail the batch on the duplicate key"); + Assert.That(stored.Single().Status, Is.EqualTo(Status.Pass), "the copy never replaces a row the target already holds"); + Assert.That(await Target.Count(Category), Is.EqualTo(1)); + } + } + + [TestCase(nameof(CustomCheckEntity.CustomCheckId))] + [TestCase(nameof(CustomCheckEntity.Category))] + [TestCase(nameof(CustomCheckEntity.OriginatingEndpointHost))] + public async Task A_check_with_no_value_for_a_required_column_is_skipped_as_a_required_value_missing_and_the_rest_copy(string column) + { + var missing = column switch + { + nameof(CustomCheckEntity.CustomCheckId) => Detail(failed: true, checkId: null), + nameof(CustomCheckEntity.Category) => Detail(failed: true, category: null), + _ => Detail(failed: true, endpoint: new EndpointDetails { Name = "Sales", Host = null, HostId = Guid.NewGuid() }) + }; + var batch = BatchOf(("CustomChecks/1", missing), ("CustomChecks/2", Detail(failed: false, checkId: "Sales.DiskCheck"))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a skip copies the rest of the batch, where a NOT NULL failure would fail the whole insert"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 })); + Assert.That(details, Is.EqualTo(new[] { $"CustomChecks.{column} is NOT NULL and the document has no value for it" })); + } + } + + [Test] + public async Task A_row_the_source_flags_as_a_storage_check_is_skipped_as_harmless_whatever_its_id() + { + var batch = new MigrationBatch( + [ + new MigrationRow("CustomChecks/1", Detail(failed: false, checkId: "Some.Other.Source.Check"), Flag(true)), + new MigrationRow("CustomChecks/2", Detail(failed: false, checkId: "Sales.DiskCheck"), new Dictionary<string, object>()) + ], "CustomChecks/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.SourceStorageCheck] = 1 })); + Assert.That(result.SkipReasons.Keys.All(reason => reason.IsBenign()), Is.True); + Assert.That(details, Is.EqualTo(new[] { "'Some.Other.Source.Check' under 'Ping' is a check the source's own storage runs about itself, which nothing reports once the instance runs on the target" })); + } + } + + [Test] + public async Task A_row_with_a_RavenDB_persister_id_and_category_but_no_flag_is_copied() + { + var result = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: false, checkId: "Error Database Index Lag", category: "ServiceControl Health"))), CheckpointAfter("CustomChecks/1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "the target holds no list of ids, so only the source's flag can skip a row"); + Assert.That(result.SkipReasons, Is.Null); + } + } + + [Test] + public void A_flag_that_is_not_a_bool_fails_the_batch_naming_the_row() + { + var batch = new MigrationBatch([new MigrationRow("CustomChecks/1", Detail(failed: false), Flag("true"))], "CustomChecks/1"); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(Category, batch, CheckpointAfter("CustomChecks/1"))); + + Assert.That(exception.Message, Does.Contain("'CustomChecks/1'").And.Contain("System.String")); + } + + [Test] + public async Task A_row_whose_storage_check_flag_is_false_is_copied() + { + var batch = new MigrationBatch([new MigrationRow("CustomChecks/1", Detail(failed: false), Flag(false))], "CustomChecks/1"); + + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons, Is.Null); + } + } + + static Dictionary<string, object> Flag(object value) => new() { [MigrationMetadataKeys.SourceStorageCheck] = value }; + + static CustomCheckDetail Detail(bool failed, string checkId = "Sales.HealthCheck", string category = "Ping", EndpointDetails endpoint = null) => new() + { + OriginatingEndpoint = endpoint ?? Endpoint, + CustomCheckId = checkId, + Category = category, + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + HasFailed = failed + }; + + static MigrationBatch BatchOf(params (string Id, CustomCheckDetail Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.CustomChecks, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs new file mode 100644 index 0000000000..eb0bcbfab4 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class FailedErrorImportRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property_of_a_failed_error_import() + { + var import = new FailedErrorImport + { + Id = "native-1", + Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }; + var failedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + // Inline leaves the flag false and external leaves the column empty, so each call covers the one the other cannot. + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, FailedErrorImportRowMapper.Map(import, failedAt, bodyStoredExternally: false), nameof(FailedErrorImportEntity.BodyStoredExternally)); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, FailedErrorImportRowMapper.Map(import, failedAt, bodyStoredExternally: true), nameof(FailedErrorImportEntity.Body)); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs new file mode 100644 index 0000000000..06024a2453 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs @@ -0,0 +1,133 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class FailedErrorImportsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + // Not a whole microsecond, because PostgreSQL keeps only microseconds and a re-read must still match what it stored. + static readonly DateTime FirstFailure = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc).AddTicks(1234567); + + [Test] + public async Task A_second_import_of_the_same_message_in_a_batch_is_reported_as_a_merge_naming_the_document_kept() + { + var batch = BatchOf(Row("FailedErrorImports/a", FirstFailure), Row("FailedErrorImports/b", FirstFailure.AddMinutes(5))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/b")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "FailedErrorImports/b" }), "6.18 and earlier stored each failed import attempt under its own document, and both derive one key"); + Assert.That(merges.Single().Detail, Does.Contain("kept the row from 'FailedErrorImports/a'"), "the target logs this line, so it has to name the document it kept"); + Assert.That((result.Copied, result.AlreadyPresent), Is.EqualTo((1, 1))); + } + } + + [Test] + public async Task An_import_of_a_message_an_earlier_batch_copied_from_another_document_is_reported_as_a_merge() + { + var first = await Target.Write(Category, BatchOf(Row("FailedErrorImports/a", FirstFailure)), CheckpointAfter("FailedErrorImports/a")); + var later = BatchOf(Row("FailedErrorImports/b", FirstFailure.AddMinutes(5))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), later); + var result = await Target.Write(Category, later, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "FailedErrorImports/b" }), "random document ids spread the imports of one message over different batches"); + Assert.That(merges.Single().Detail, Does.Contain("already holds an import of the same message")); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_document_an_earlier_pass_copied_is_already_present_and_not_a_merge() + { + var batch = BatchOf(Row("FailedErrorImports/a", FirstFailure)); + var first = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/a")); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), batch); + var again = await Target.Write(Category, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges, Is.Empty, "--migration-retry re-reads the same documents, and finding its own rows again is not a merge"); + Assert.That(again.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task An_import_with_no_exception_or_no_body_is_reported_as_a_change_once() + { + var batch = BatchOf( + ImportRow("FailedErrorImports/no-exception", "native-1", exceptionInfo: null, [1]), + ImportRow("FailedErrorImports/no-body", "native-2", "Deserialization failed", body: null), + ImportRow("FailedErrorImports/neither", "native-3", exceptionInfo: null, body: null), + ImportRow("FailedErrorImports/complete", "native-4", "Deserialization failed", [1]), + ImportRow("FailedErrorImports/no-exception-again", "native-1", exceptionInfo: null, [1])); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var first = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/no-exception-again")); + var again = await Target.Write(Category, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ("FailedErrorImports/no-exception", "Stored an empty exception, because the document has none."), + ("FailedErrorImports/no-body", "Stored an empty body, because the document's message has none."), + ("FailedErrorImports/neither", "Stored an empty exception and an empty body, because the document has neither.") + }), "a complete import changes nothing, and the second import of one message is a merge, not a change"); + Assert.That((first.Changed, again.Changed), Is.EqualTo((3, 0)), "a re-read finds the rows an earlier pass copied and changes nothing"); + } + } + + static MigrationRow ImportRow(string documentId, string messageId, string exceptionInfo, byte[] body) => new( + documentId, + new FailedErrorImport + { + Id = documentId, + Message = new FailedTransportMessage { Id = messageId, Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = messageId }, Body = body }, + ExceptionInfo = exceptionInfo + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = FirstFailure }, + new MigrationBody(body ?? [], "application/octet-stream")); + + FailedErrorImportsWriter Writer() => new( + ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), + ServiceProvider.GetRequiredService<IBodyStoragePersistence>(), + ServiceProvider.GetRequiredService<EFPersisterSettings>()); + + static MigrationRow Row(string documentId, DateTime failedAt) => new( + documentId, + new FailedErrorImport + { + Id = documentId, + Message = new FailedTransportMessage { Id = "native-8", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-8" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = failedAt }, + new MigrationBody(new byte[] { 1, 2, 3 }, "application/octet-stream")); + + static MigrationBatch BatchOf(params MigrationRow[] rows) => new([.. rows], rows[^1].SourceId); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs index 59afd305ea..b273ece28a 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs @@ -45,7 +45,12 @@ public async Task Every_retry_claim_column_is_set_for_a_copied_retry_issued_mess { var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); - var source = FullyPopulatedRow() with { Body = new MigrationBody(Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml") }; + var row = FullyPopulatedRow(); + var source = row with + { + Metadata = new Dictionary<string, object?>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = true }, + Body = new MigrationBody(Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml") + }; ((FailedMessage)source.Document).Status = FailedMessageStatus.RetryIssued; await target.Open(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs index 576d0e0cdb..814f5565a3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs @@ -22,6 +22,10 @@ namespace ServiceControl.Persistence.Tests; class FailedMessagesMigrationTargetTests : PersistenceTestBase { + readonly TargetWarnings targetWarnings = new(); + + public FailedMessagesMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetWarnings); + [SetUp] public Task OpenTarget() => Target.Open(); @@ -260,6 +264,221 @@ public async Task A_message_or_group_value_longer_than_its_column_skips_the_mess } } + [Test] + public async Task A_change_is_not_logged_when_the_batch_does_not_commit() + { + var longType = new IngestedFailure { MessageType = new string('m', 600) }; + await Target.Write(FailedMessagesCategory, BatchOf(new IngestedFailure().ToFailedMessage()), Checkpoint()); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.Write(FailedMessagesCategory, BatchOf(longType.ToFailedMessage()), Checkpoint()), + "the checkpoint it extends is older than the stored one, so the save in the batch's transaction is refused"); + Assert.That(targetWarnings.Messages, Has.None.StartsWith("Changed "), "a change line written before the commit would name a row SQL never stored"); + } + + [Test] + public async Task A_second_document_with_the_same_message_id_reports_no_change_of_its_own() + { + var first = new IngestedFailure(); + var second = new IngestedFailure { MessageType = new string('m', 600) }.ToFailedMessage(); + second.UniqueMessageId = first.UniqueMessageIdString; + var batch = BatchOf(first.ToFailedMessage(), second); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + + Assert.That(changes, Is.Empty, "the insert keeps the first document, so the second's cut message type never reaches SQL"); + } + + [Test] + public async Task A_message_type_longer_than_its_column_is_copied_with_a_change_naming_its_length() + { + var longType = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(longType.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] { ($"FailedMessages/{longType.UniqueMessageIdString}", "Cut the message type from 600 characters to its last 450, as ingestion does.") })); + Assert.That(result.Changed, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_failure_group_with_no_id_or_no_title_is_reported_as_a_change_naming_the_group() + { + var failure = new IngestedFailure + { + Groups = + [ + new() { Id = "group-kept", Title = null, Type = "Exception Type" }, + new() { Id = null!, Title = "Has no id", Type = "Message Type" }, + null! + ] + }; + var batch = BatchOf(failure.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => new { group.GroupId, group.Title }).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ($"FailedMessages/{failure.UniqueMessageIdString}", + "Dropped the failure groups with no id, because SQL keys a group by its id: the group titled 'Has no id', an empty entry. Stored an empty title or type for the failure groups that had none: 'group-kept'.") + })); + Assert.That(result.Changed, Is.EqualTo(1)); + Assert.That(groups.Select(group => (group.GroupId, group.Title)), Is.EqualTo(new[] { ("group-kept", "") }), "the message is missing from the group with no id, which the line has to say"); + } + } + + [Test] + public async Task Two_failure_groups_of_one_message_with_one_key_are_reported_as_a_change_naming_both() + { + var failure = new IngestedFailure + { + Groups = + [ + new() { Id = "group-a", Title = "First", Type = "Exception Type" }, + new() { Id = "group-a", Title = "Second", Type = "Exception Type" } + ] + }; + var batch = BatchOf(failure.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var titles = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => group.Title).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ($"FailedMessages/{failure.UniqueMessageIdString}", "Kept the failure group 'group-a' titled 'First' and dropped 'group-a' titled 'Second', because SQL treats their ids as one key.") + })); + Assert.That(result.Changed, Is.EqualTo(1)); + Assert.That(titles, Is.EqualTo(new[] { "First" }), "the line names the group the insert kept"); + } + } + + [Test] + public async Task A_retry_issued_message_gets_a_retry_claim_only_when_the_source_still_held_one() + { + var released = new IngestedFailure(); + var held = new IngestedFailure(); + var batch = BatchOf( + WithClaimHeld(RowOf(released.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), true)); + + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + Assert.That(claims, Is.EqualTo(new[] { held.UniqueMessageId }), "a claim the source had released would block a retry by id that RavenDB would have sent"); + Assert.That(result.Changed, Is.Zero, "the claims match the source, so nothing changed on the way in"); + } + } + + [Test] + public async Task A_retry_by_id_after_the_copy_stages_the_message_whose_claim_the_source_had_released() + { + var released = new IngestedFailure(); + var held = new IngestedFailure(); + await Target.Write(FailedMessagesCategory, BatchOf( + WithClaimHeld(RowOf(released.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), true)), Checkpoint()); + + string[] ids = [released.UniqueMessageIdString, held.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-by-id", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EqualTo(new[] { released.UniqueMessageIdString }), + "RavenDB would send the released message and hold back the claimed one, so after the move a retry by id must do the same"); + } + + [Test] + public void A_retry_issued_message_without_its_claim_flag_fails_the_write_naming_the_row() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(WithClaimHeld(RowOf(document, SmallTextBody), null)), Checkpoint())); + + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationMetadataKeys.RetryClaimHeld), + "the source resolves the flag for every retry-issued message, so a missing one is a defect that must not pass as a held claim"); + } + + [Test] + public async Task A_retry_issued_message_whose_claim_flag_is_not_a_bool_fails_the_write_and_stores_nothing() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + var row = WithClaimHeld(RowOf(document, SmallTextBody), null); + row = row with { Metadata = new Dictionary<string, object>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = "true" } }; + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(row), Checkpoint())); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().CountAsync()); + var messages = await Query(db => db.FailedMessages.AsNoTracking().CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationMetadataKeys.RetryClaimHeld).And.Contain(typeof(string).FullName)); + Assert.That((claims, messages), Is.EqualTo((0, 0)), "a flag the writer cannot read must stop the batch before anything is stored, not be read as either answer"); + } + } + + [Test] + public async Task Two_retry_issued_documents_sharing_a_unique_message_id_take_the_claim_flag_of_the_first() + { + var first = new IngestedFailure(); + var second = first.ToFailedMessage(FailedMessageStatus.RetryIssued); + var batch = BatchOf( + WithClaimHeld(RowOf(first.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(second, SmallTextBody) with { SourceId = "FailedMessages/second" }, true)); + + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "the test proves nothing unless the insert kept one of the two"); + Assert.That(claims, Is.Zero, "the insert keeps the first document, whose claim the source had released, so the second's held flag must not add one"); + } + } + + [Test] + public async Task Copying_a_batch_with_a_held_claim_again_leaves_one_claim() + { + var held = new IngestedFailure(); + var batch = BatchOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued)); + + var first = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + await Target.Write(FailedMessagesCategory, batch, first.Saved); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + Assert.That(claims, Is.EqualTo(new[] { held.UniqueMessageId }), "a retry re-reads rows an earlier pass copied, and must not add a second claim or fail on the first"); + } + + [Test] + public async Task A_changed_message_the_target_already_holds_is_not_reported_again() + { + var changed = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(changed.ToFailedMessage()); + + var first = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var again = await Target.Write(FailedMessagesCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That((first.Changed, again.Changed), Is.EqualTo((1, 0)), "a retry re-reads rows an earlier pass copied, and this write changed nothing"); + Assert.That(again.AlreadyPresent, Is.EqualTo(1)); + Assert.That(targetWarnings.Messages.Count(message => message.StartsWith("Changed ")), Is.EqualTo(1)); + } + } + [Test] public async Task Count_takes_in_only_unresolved_and_retry_issued_messages() { @@ -443,13 +662,36 @@ async Task<T> Query<T>(Func<ServiceControlDbContext, Task<T>> query) static MigrationBatch BatchOf(params MigrationRow[] rows) => new(rows, rows[^1].SourceId); - // Every row carries the two dates the source resolves, as a real source's rows do. - static MigrationRow RowOf(FailedMessage document, MigrationBody body) => - new($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary<string, object> + // Every row carries the values the source resolves, as a real source's rows do, and a retry-issued one finds its claim still held. + static MigrationRow RowOf(FailedMessage document, MigrationBody body) + { + var metadata = new Dictionary<string, object> { [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) - }, body); + }; + + if (document.Status == FailedMessageStatus.RetryIssued) + { + metadata[MigrationMetadataKeys.RetryClaimHeld] = true; + } + + return new($"FailedMessages/{document.UniqueMessageId}", document, metadata, body); + } + + // A null removes the flag, as a source that never resolved it would send the row. + static MigrationRow WithClaimHeld(MigrationRow row, bool? held) + { + var metadata = new Dictionary<string, object>(row.Metadata); + metadata.Remove(MigrationMetadataKeys.RetryClaimHeld); + + if (held is { } value) + { + metadata[MigrationMetadataKeys.RetryClaimHeld] = value; + } + + return row with { Metadata = metadata }; + } sealed class UnwritableBodyStore : IBodyStoragePersistence { @@ -462,6 +704,8 @@ sealed class UnwritableBodyStore : IBodyStoragePersistence public Task DeleteBodyIfExists(string bodyId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); } + FailedMessagesWriter Writer() => new(ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), BodyStore, ServiceProvider.GetRequiredService<EFPersisterSettings>()); + static MigrationCheckpoint Checkpoint() => new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs new file mode 100644 index 0000000000..c4b017a7af --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class GroupCommentRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + var entity = GroupCommentRowMapper.Map(new GroupComment { Id = "group-1", Comment = "Waiting on the payments team" }); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs new file mode 100644 index 0000000000..9f50b13692 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs @@ -0,0 +1,115 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; + +class GroupCommentsMigrationTargetTests : ErrorIngestionTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory GroupCommentsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)!; + + [Test] + public async Task A_comment_whose_group_id_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fits = new string('g', 64); + var tooLong = new string('g', 65); + var batch = BatchOf((fits, "Waiting on the payments team"), (tooLong, "Raised with the shipping team")); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new GroupCommentsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter(tooLong)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a group id exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"GroupComment/{tooLong}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 1 }), "an id too long for its column would fail the whole insert and stop the category"); + Assert.That(await Target.Count(GroupCommentsCategory), Is.EqualTo(1), "--migration-verify reads this count"); + Assert.That(details.Single(), Does.EndWith($"takes at most 64 characters and the value holds 65, on document 'GroupComment/{tooLong}'")); + } + } + + [Test] + public async Task A_comment_an_earlier_pass_copied_keeps_its_text_and_is_counted_as_already_present() + { + var failure = new IngestedFailure(); + var group = failure.Groups.Single(); + await Ingest(failure); + + var first = await Target.Write(GroupCommentsCategory, BatchOf((group.Id, "Waiting on the payments team")), CheckpointAfter(group.Id)); + var again = await Target.Write(GroupCommentsCategory, BatchOf((group.Id, "Edited in RavenDB since")), first.Saved); + var view = (await GroupsStore.GetUnresolvedGroupsByClassifier(group.Type, null, new PagingInfo())).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(again.Copied, Is.Zero); + Assert.That(again.AlreadyPresent, Is.EqualTo(1), "a retry re-reads the category from the start and finds what the first pass copied"); + Assert.That((view.Id, view.Comment), Is.EqualTo((group.Id, "Waiting on the payments team")), "the copy only inserts what is missing and never changes a row"); + } + } + + [Test] + public async Task A_comment_whose_group_id_the_key_treats_as_an_earlier_ones_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf(("group-1", "Waiting on the payments team"), ("group-1 ", "Raised with the shipping team"), ("group-1", "Edited in RavenDB since")); + var keyIgnoresTrailingSpaces = dialect.KeyComparer(typeof(GroupCommentEntity), nameof(GroupCommentEntity.GroupId)).Equals("group-1", "group-1 "); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new GroupCommentsWriter(dialect), batch); + var result = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter("group-1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpaces ? new[] { "GroupComment/group-1 ", "GroupComment/group-1" } : ["GroupComment/group-1"]), + "an id that differs only in trailing spaces is dropped only where the key column ignores them, an exact repeat on every provider, and the log has to say so"); + Assert.That(merges.Select(merge => merge.Detail), Has.All.Contains("kept 'group-1'"), "the target logs this line, so it has to name the group id it kept"); + Assert.That((result.Copied, result.AlreadyPresent), Is.EqualTo(keyIgnoresTrailingSpaces ? (1, 2) : (2, 1)), + "on an empty target every row the insert drops is a merge, so the merges must account for every row counted as already present"); + } + } + + [Test] + public async Task A_comment_on_a_group_with_no_copied_message_is_reported_as_a_change_the_sweep_will_delete() + { + var failure = new IngestedFailure(); + var group = failure.Groups.Single(); + await Ingest(failure); + var longComment = new string('c', 150); + var batch = BatchOf((group.Id, "Waiting on the payments team"), ("group-of-archived-messages", longComment)); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, new GroupCommentsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var first = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter("group-of-archived-messages")); + var again = await Target.Write(GroupCommentsCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ("GroupComment/group-of-archived-messages", + "The group 'group-of-archived-messages' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + + "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + $"The comment was: '{longComment}'") + }), "a comment on a group with a copied message stays, so only the other is named, with the whole comment because the sweep deletes it"); + Assert.That((first.Copied, first.Changed), Is.EqualTo((2, 1))); + Assert.That(again.Changed, Is.Zero, "a re-read finds the comment already present, and this write copied nothing"); + } + } + + static MigrationBatch BatchOf(params (string GroupId, string Comment)[] comments) => + new([.. comments.Select(comment => new MigrationRow($"GroupComment/{comment.GroupId}", new GroupComment { Id = comment.GroupId, Comment = comment.Comment }, new Dictionary<string, object>()))], $"GroupComment/{comments[^1].GroupId}"); + + static MigrationCheckpoint CheckpointAfter(string groupId) => + new(MigrationCategoryIds.GroupComments, MigrationCategoryState.InProgress, $"GroupComment/{groupId}", 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs index 2eb2c74e25..7b0661a1b2 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs @@ -9,6 +9,7 @@ namespace ServiceControl.Persistence.Tests; using Particular.LicensingComponent.Contracts; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; using ServiceControl.Persistence.EFCore.Infrastructure; class LicensingEndpointsMigrationTargetTests : PersistenceTestBase @@ -55,6 +56,89 @@ public async Task A_licensing_endpoint_with_a_value_longer_than_its_column_is_sk } } + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_in_one_batch_are_reported_as_one_merge_where_the_key_folds_them() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var keyIgnoresTrailingSpace = KeyIgnoresTrailingSpace(dialect); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { "Sales.Orders/Broker" } : Array.Empty<string>()), + "the row whose id sorts first is kept, and the other is reported only where the key folds a trailing space"); + + if (keyIgnoresTrailingSpace) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders ', 'Broker' and dropped the row for 'sales.orders', 'Broker'"), "the line names both endpoints and the one kept"); + } + } + } + + [Test] + public async Task A_name_differing_only_in_a_trailing_space_from_a_stored_endpoint_is_reported_as_a_merge_where_the_key_folds_them() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var first = await Target.Write(LicensingEndpointsCategory, BatchOf(("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })), CheckpointAfter("Sales.Orders/Broker")); + var second = BatchOf(("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var keyIgnoresTrailingSpace = KeyIgnoresTrailingSpace(dialect); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), second); + var result = await Target.Write(LicensingEndpointsCategory, second, first.Saved with { Cursor = second.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { "Sales.Orders /Broker" } : Array.Empty<string>())); + + if (keyIgnoresTrailingSpace) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders', 'Broker' and dropped the row for 'sales.orders ', 'Broker'"), "the stored endpoint is the one kept"); + } + + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the folded endpoint is counted as already present"); + } + } + + [Test] + public async Task An_exact_replay_of_a_stored_endpoint_is_already_present_and_not_a_merge() + { + var batch = BatchOf(("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var first = await Target.Write(LicensingEndpointsCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var again = await Target.Write(LicensingEndpointsCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges, Is.Empty, "a retry re-reads the rows an earlier pass copied, and the stored row is that same endpoint"); + Assert.That((again.Copied, again.AlreadyPresent, again.Merged), Is.EqualTo((0, 1, 0))); + } + } + + [Test] + public async Task Two_names_differing_only_in_case_are_reported_as_one_merge_on_every_provider() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("SALES.ORDERS/Broker", new Endpoint("SALES.ORDERS", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "Sales.Orders/Broker" }), "the product lower-cases the name, so both providers store one row"); + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders', 'Broker' and dropped the row for 'sales.orders', 'Broker'")); + } + } + + static bool KeyIgnoresTrailingSpace(IMigrationSqlDialect dialect) => + dialect.KeyComparer(typeof(LicensingEndpointEntity), nameof(LicensingEndpointEntity.NormalizedName)).Equals("a", "a "); + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs index bcae1c840f..56edd2b44f 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs @@ -75,8 +75,8 @@ public async Task A_day_repeated_in_one_batch_keeps_its_last_row_and_counts_the_ using (Assert.EnterMultipleScope()) { Assert.That(result.Copied, Is.EqualTo(1)); - Assert.That(result.AlreadyPresent, Is.EqualTo(1), "PostgreSQL refuses a statement that updates one row twice, so the repeat is dropped before it and counted rather than failing the batch"); - Assert.That(stored, Is.EqualTo(250), "the row whose id sorts last by ordinal order sets the day, whatever order the batch arrives in"); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the repeat is the same day of the same endpoint, so it is dropped before the statement and counted"); + Assert.That(stored, Is.EqualTo(250), "the row whose id sorts last by ordinal order sets the day rather than adding to it, whatever order the batch arrives in"); } } @@ -94,7 +94,6 @@ public async Task Days_of_two_endpoints_the_key_treats_as_one_are_added_when_the ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))); var first = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); - var again = await Target.Write(LicensingThroughputCategory, batch, first.Saved); await using var scope = ServiceProvider.CreateAsyncScope(); var keyIgnoresTrailingSpace = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database.ProviderName!.Contains("SqlServer"); @@ -102,28 +101,120 @@ public async Task Days_of_two_endpoints_the_key_treats_as_one_are_added_when_the using (Assert.EnterMultipleScope()) { Assert.That(first.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2)); - Assert.That(first.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the endpoint folded into the other's row is counted, not lost"); + Assert.That(first.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the folded endpoint's days are summed into the kept row, and the endpoint writer reports the fold"); Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 350L } : new[] { 100L, 250L }), "SQL Server's key makes the two one day, which holds both counts, as the product's own throughput write would"); - Assert.That(again.Copied, Is.EqualTo(first.Copied), "copying the batch again sets the same sum rather than adding to it"); } } [Test] - public async Task A_later_batch_replaces_the_count_an_earlier_batch_stored_for_the_same_day() + public async Task Days_of_two_endpoints_differing_only_in_case_are_added_when_they_arrive_in_one_batch() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100)), + ("SALES.ORDERS/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("SALES.ORDERS", ThroughputSource.Broker), day, 250))); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("SALES.ORDERS/Broker")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the folded endpoint's days are summed into the kept row, and the endpoint writer reports the fold"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 350L }), "two source endpoints store the same lower-cased name, so their days are added, not one kept as a repeat of the other"); + } + } + + [Test] + public async Task A_later_batch_adds_its_count_to_the_day_an_earlier_batch_stored_for_a_name_differing_only_in_case() { await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); var day = new DateOnly(2026, 1, 1); var first = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100))), CheckpointAfter("Sales.Orders/Broker")); - var second = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))), first.Saved); + var second = await Target.Write(LicensingThroughputCategory, BatchOf(("SALES.ORDERS/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("SALES.ORDERS", ThroughputSource.Broker), day, 250))), first.Saved); using (Assert.EnterMultipleScope()) { Assert.That(second.Copied, Is.EqualTo(1), "an update is a row the statement wrote"); - Assert.That(await StoredCounts(), Is.EqualTo(new[] { 250L }), "the count RavenDB holds now replaces the stored one, and is never added to it or ignored"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 350L }), "both endpoints store the same lower-cased name, so the second batch adds its count to the day rather than replacing it"); } } + [Test] + public async Task A_batch_written_again_from_the_checkpoint_it_already_extended_is_refused_and_adds_nothing() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var checkpointToExtend = CheckpointAfter("Sales.Orders/Broker"); + + await Target.Write(LicensingThroughputCategory, batch, checkpointToExtend); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.Write(LicensingThroughputCategory, batch, checkpointToExtend)); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the add and the checkpoint commit together, so a replay of a committed batch rolls back instead of counting the day twice"); + } + + [Test] + public async Task A_retry_deletes_the_copied_days_so_copying_again_gives_the_same_totals() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var first = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + var reset = await Target.ResetForRetry(ResetForRetry(first.Saved)); + var afterReset = await StoredCounts(); + await Target.Write(LicensingThroughputCategory, batch, reset); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterReset, Is.Empty, "the reset deletes the days the first copy added, because copying them again adds each count a second time"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "copying again after the retry gives the totals one copy gave, not double"); + } + } + + [Test] + public async Task A_stale_retry_reset_deletes_no_copied_day() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var stale = (await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker"))).Saved; + await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Upsert(stale); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.ResetForRetry(ResetForRetry(stale))); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the delete shares the reset's transaction, so a refused reset leaves every copied day"); + } + + [Test] + public async Task A_retry_reset_after_ServiceControl_opened_on_the_target_is_refused_and_deletes_nothing() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var copied = (await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker"))).Saved; + await ServiceProvider.GetRequiredService<IMigrationTargetReadiness>().RecordHostOpened(); + + var refused = Assert.ThrowsAsync<InvalidOperationException>(() => Target.ResetForRetry(ResetForRetry(copied))); + var stored = await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Read(MigrationCategoryIds.LicensingThroughput); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refused.Message, Does.Contain(MigrationCategoryIds.LicensingThroughput).And.Contain("only possible before ServiceControl first opens on this database")); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the days may now include counts the running instance collected, so deleting them would lose live throughput"); + Assert.That((stored!.Version, stored.State, stored.Cursor, stored.CopiedCount), Is.EqualTo((copied.Version, copied.State, copied.Cursor, copied.CopiedCount)), "a refused reset saves nothing"); + } + } + + [Test] + public async Task A_retry_reset_that_deletes_nothing_still_succeeds_after_ServiceControl_opened_on_the_target() + { + var checkpoints = ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + var stopped = await checkpoints.Upsert(new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.InProgress, "FailedMessages/1", 1, 0, null, null, null, null, null, "RavenDB is unreachable")); + await ServiceProvider.GetRequiredService<IMigrationTargetReadiness>().RecordHostOpened(); + + var reset = await Target.ResetForRetry(ResetForRetry(stopped)); + + Assert.That((reset.State, reset.Cursor, reset.LastError), Is.EqualTo((MigrationCategoryState.NotStarted, (string)null, (string)null)), "an optional category copies after ServiceControl opens, and its reset deletes nothing, so it is still allowed"); + } + [Test] public async Task One_endpoint_with_more_days_than_one_statement_takes_stores_every_day_with_its_own_count() { @@ -157,7 +248,7 @@ public async Task One_endpoint_with_more_days_than_one_statement_takes_stores_ev } [Test] - public async Task A_later_batch_sets_a_day_stored_under_a_name_that_differs_only_by_a_trailing_space() + public async Task A_later_batch_adds_to_a_day_stored_under_a_name_that_differs_only_by_a_trailing_space() { var endpoints = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); var endpointBatch = BatchOf( @@ -175,7 +266,7 @@ public async Task A_later_batch_sets_a_day_stored_under_a_name_that_differs_only using (Assert.EnterMultipleScope()) { Assert.That(second.Copied, Is.EqualTo(1), "the second batch's day is matched back to the row that produced it, not refused because the stored name keeps its trailing space"); - Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 250L } : new[] { 100L, 250L }), "SQL Server's key makes the two names one day, which the later count sets; PostgreSQL's keeps two"); + Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 350L } : new[] { 100L, 250L }), "SQL Server's key makes the two names one day, which the later count adds to; PostgreSQL's keeps two"); } } @@ -188,6 +279,19 @@ async Task<long[]> StoredCounts() static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + // The reset --migration-retry makes. + static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0, + SkipReasons = null, + SettledAt = null, + LastError = null + }; + static MigrationCheckpoint CheckpointAfter(string cursor) => new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index 3237b5700e..3e942eace6 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -51,8 +51,21 @@ public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_ Assert.That(model.FindEntityType(typeof(SettingEntity))!.FindProperty(nameof(SettingEntity.Value))!.GetMaxLength(), Is.Null, "NoValueCanBeTooLong exempts the settings categories because of this; give them a value-too-long test if the column gains a limit"); } + [TestCase(typeof(CustomCheckEntity))] + [TestCase(typeof(FailedErrorImportEntity))] + public async Task No_text_column_of_the_entity_has_a_maximum_length_so_no_value_can_be_too_long(Type entity) + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var entityType = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model.FindEntityType(entity)!; + + Assert.That( + entityType.GetProperties().Where(property => property.ClrType == typeof(string) && property.GetMaxLength() is not null).Select(property => $"{entityType.GetTableName()}.{property.GetColumnName()}"), + Is.Empty, + "NoValueCanBeTooLong exempts this entity's category because of this; give it a value-too-long test if a column gains a limit"); + } + [Test] - public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_them_for_a_retry_and_no_other_writer_does() + public async Task Every_writer_whose_rows_a_re_read_cannot_recognise_deletes_them_for_a_retry_and_no_other_writer_does() { await using var scope = ServiceProvider.CreateAsyncScope(); var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; @@ -67,14 +80,14 @@ public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_t { var storeKeyed = entities.Any(entity => model.FindEntityType(entity)!.FindPrimaryKey()!.Properties.Any(property => property.ValueGenerated != ValueGenerated.Never)); - Assert.That(typeof(IStoreKeyedCategoryWriter).IsAssignableFrom(writer), Is.EqualTo(storeKeyed), - $"{writer.Name}: a retry keeps the rows of a writer that is not an IStoreKeyedCategoryWriter, so a re-read adds rows with store keys a second time, and a writer whose keys the copy supplies must not delete"); + Assert.That(typeof(IStoreKeyedCategoryWriter).IsAssignableFrom(writer), Is.EqualTo(storeKeyed || WritersThatAddToRowsTheyCopied.Contains(writer)), + $"{writer.Name}: a retry keeps the rows of a writer that is not an IStoreKeyedCategoryWriter, so a re-read adds rows with store keys or added counts a second time, and a writer whose rows a re-read finds already present must not delete"); } } } - // Each is one Settings row under a constant key whose value column has no maximum length, so no source value can be too long for it. - static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; + // None of these copies a source value into a text column with a maximum length, which the two no-maximum-length tests above pin. + static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate, MigrationCategoryIds.CustomChecks, MigrationCategoryIds.FailedErrorImports]; // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. static readonly Dictionary<string, string> ValueTooLongTests = new() @@ -86,9 +99,14 @@ public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_t [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.MessageRedirects] = nameof(MessageRedirectsMigrationTargetTests.A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.Subscriptions] = nameof(SubscriptionsMigrationTargetTests.A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy), - [MigrationCategoryIds.RetryOperations] = nameof(RetryOperationsMigrationTargetTests.A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + [MigrationCategoryIds.RetryOperations] = nameof(RetryOperationsMigrationTargetTests.A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.PendingIntegrationEvents] = nameof(PendingIntegrationEventsMigrationTargetTests.A_request_whose_context_type_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.GroupComments] = nameof(GroupCommentsMigrationTargetTests.A_comment_whose_group_id_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) }; + // Their keys come from the source, but a re-read adds its counts to the stored ones, so a re-read cannot find them already present. + static readonly Type[] WritersThatAddToRowsTheyCopied = [typeof(LicensingThroughputWriter)]; + // The writers do not say which entities they write, so this list does. static readonly Dictionary<Type, Type[]> EntitiesEachWriterWrites = new() { @@ -103,6 +121,10 @@ public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_t [typeof(SubscriptionsWriter)] = [typeof(SubscriptionEntity)], [typeof(NotificationSettingsWriter)] = [typeof(SettingEntity)], [typeof(TrialEndDateWriter)] = [typeof(SettingEntity)], - [typeof(RetryOperationsWriter)] = [typeof(HistoricRetryOperationEntity), typeof(UnacknowledgedRetryOperationEntity)] + [typeof(RetryOperationsWriter)] = [typeof(HistoricRetryOperationEntity), typeof(UnacknowledgedRetryOperationEntity)], + [typeof(PendingIntegrationEventsWriter)] = [typeof(ExternalIntegrationDispatchRequestEntity)], + [typeof(CustomChecksWriter)] = [typeof(CustomCheckEntity)], + [typeof(FailedErrorImportsWriter)] = [typeof(FailedErrorImportEntity)], + [typeof(GroupCommentsWriter)] = [typeof(GroupCommentEntity)] }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs new file mode 100644 index 0000000000..9dfa1dd395 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class PendingIntegrationEventRowMapperTests : PersistenceTestBase +{ + [Test] + public void Sets_every_mapped_property_of_a_dispatch_request() + { + var entity = PendingIntegrationEventRowMapper.Map(Request("one")); + + using var scope = ServiceProvider.CreateScope(); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model, entity); + } + + [Test] + public async Task Writes_exactly_the_type_name_and_json_the_product_writes_for_the_same_context() + { + await ExternalIntegrationStore.StoreDispatchRequest([Request("one")]); + + using var scope = ServiceProvider.CreateScope(); + var stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().ExternalIntegrationDispatchRequests.AsNoTracking().SingleAsync(); + var mapped = PendingIntegrationEventRowMapper.Map(Request("one")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(mapped.DispatchContextTypeName, Is.EqualTo(stored.DispatchContextTypeName)); + Assert.That(mapped.DispatchContextJson, Is.EqualTo(stored.DispatchContextJson)); + } + } + + static ExternalIntegrationDispatchRequest Request(string value) => new() { DispatchContext = new TestDispatchContext { Value = value } }; + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs new file mode 100644 index 0000000000..f3ca3ea132 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs @@ -0,0 +1,120 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Implementation; + +class PendingIntegrationEventsMigrationTargetTests : PersistenceTestBase +{ + readonly TargetWarnings targetLog = new(); + + public PendingIntegrationEventsMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetLog); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + ExternalIntegrationRequestsDataStore Dispatcher => ServiceProvider.GetServices<IHostedService>().OfType<ExternalIntegrationRequestsDataStore>().Single(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.PendingIntegrationEvents)!; + + [Test] + public async Task A_copied_request_is_dispatched_by_the_product_as_its_concrete_type_and_then_removed() + { + var dispatched = Array.Empty<object>(); + ExternalIntegrationStore.Subscribe((contexts, _) => + { + dispatched = contexts; + return Task.CompletedTask; + }); + + var result = await Write(("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "copied" } })); + + await Dispatcher.DispatchNow(TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(((TestDispatchContext)dispatched.Single()).Value, Is.EqualTo("copied")); + Assert.That(await Rows(), Is.Zero, "the dispatcher deletes what it delivered, which is delivery rather than loss"); + } + } + + [Test] + public async Task A_request_with_no_context_is_skipped_as_a_required_value_missing() + { + var result = await Write( + ("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = null! }), + ("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "kept" } })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons[MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "both columns are NOT NULL, and a throw would stop the category at this batch"); + Assert.That(await Target.Count(Category), Is.EqualTo(1), "Count is what --migration-verify reads, so it must see the row the copy wrote"); + } + } + + [Test] + public async Task A_request_whose_context_type_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + // The stored type name ends with the assembly name, so a long assembly name makes it longer than the column. + var assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(new string('a', 460)), AssemblyBuilderAccess.Run); + var longNamedContext = Activator.CreateInstance(assembly.DefineDynamicModule("Contexts").DefineType("Context", TypeAttributes.Public).CreateType())!; + + var result = await Write( + ("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = longNamedContext }), + ("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "kept" } })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "ExternalIntegrationDispatchRequests/1" })); + Assert.That(result.SkipReasons[MigrationSkipReason.ValueTooLong], Is.EqualTo(1), "one value longer than its column fails the whole statement, so it is skipped instead"); + Assert.That(await Target.Count(Category), Is.EqualTo(1)); + } + } + + [Test] + public async Task The_first_batch_that_copies_events_says_once_that_the_database_gives_them_new_ids() + { + var first = await Write(("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "first" } })); + await Target.Write( + Category, + new MigrationBatch([new MigrationRow("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "second" } }, new Dictionary<string, object>())], "ExternalIntegrationDispatchRequests/2"), + first.Saved); + + Assert.That(targetLog.Information, Has.Exactly(1).EqualTo($"Category {MigrationCategoryIds.PendingIntegrationEvents}: the database gives the copied pending integration events new ids. Nothing in SQL refers to the ids they had in the source."), + "one line per category says the ids changed, rather than one per row or per batch"); + } + + Task<MigrationWriteResult> Write(params (string Id, ExternalIntegrationDispatchRequest Request)[] rows) => + Target.Write( + Category, + new MigrationBatch([.. rows.Select(row => new MigrationRow(row.Id, row.Request, new Dictionary<string, object>()))], rows[^1].Id), + new MigrationCheckpoint(Category.Id, MigrationCategoryState.InProgress, rows[^1].Id, 0, 0, null, null, null, null, null, null)); + + async Task<int> Rows() + { + using var scope = ServiceProvider.CreateScope(); + return await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().ExternalIntegrationDispatchRequests.CountAsync(); + } + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs index 8e43f3c55f..98e6b8c93f 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs @@ -34,6 +34,17 @@ public static async Task<IReadOnlyList<string>> DetailsFrom(IServiceProvider ser return prepared.Merges; } + /// <summary> + /// Prepares a batch without writing it, and returns the rows the writer reports as changed on the way in with their log detail. + /// </summary> + public static async Task<IReadOnlyList<(string SourceId, string Detail)>> ChangesFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(), batch); + + return prepared.Changes; + } + /// <summary> /// The table and column a property is stored in, as this provider names them. /// </summary> diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs index 58adff0d22..ded7804dac 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Persistence.Tests; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DataMigration.Writers; @@ -14,6 +15,10 @@ namespace ServiceControl.Persistence.Tests; class RetryOperationsMigrationTargetTests : PersistenceTestBase { + readonly TargetWarnings targetLog = new(); + + public RetryOperationsMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetLog); + [SetUp] public Task OpenTarget() => Target.Open(); @@ -151,6 +156,24 @@ public async Task A_second_pending_operation_with_the_same_key_is_reported_as_a_ } } + [Test] + public async Task A_merged_pending_operation_names_the_source_row_kept_and_the_row_dropped() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("unacknowledged|group-2|FailureGroup|older", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("unacknowledged|group-2|FailureGroup|newer", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "unacknowledged|group-2|FailureGroup|older" }), "the latest retry wins"); + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 'unacknowledged|group-2|FailureGroup|newer' and dropped the row from 'unacknowledged|group-2|FailureGroup|older'."), + "both keys print the same, so only the source rows, whose ids carry the completion time, tell the operator which run survived"); + } + } + [Test] public async Task A_pending_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() { @@ -214,6 +237,24 @@ public async Task A_retry_operation_time_of_local_or_unspecified_kind_is_copied_ } } + [Test] + public async Task Copied_historic_operations_get_one_line_saying_the_database_gives_them_new_ids() + { + var first = await Target.Write(RetryOperationsCategory, BatchOf(("historic/1", new RetryOperationRow(Historic("group-1"), null)), ("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + await Target.Write(RetryOperationsCategory, BatchOf(("historic/2", new RetryOperationRow(Historic("group-3"), null))), first.Saved with { Cursor = "historic/2" }); + + Assert.That(targetLog.Information, Is.EqualTo(new[] { $"Category {MigrationCategoryIds.RetryOperations}: the database gives the copied historic retry operations new ids. Nothing in SQL refers to the ids they had in the source." }), + "one line per category says the ids changed, rather than one per row or per batch"); + } + + [Test] + public async Task Copied_pending_operations_alone_get_no_line_about_new_ids() + { + await Target.Write(RetryOperationsCategory, BatchOf(("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + + Assert.That(targetLog.Information, Has.None.Contains("new ids"), "a pending operation keeps its key, so nothing about it is renumbered"); + } + static HistoricRetryOperation Historic(string requestId) => new() { RequestId = requestId, diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs index 0b437aece1..535520537d 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs @@ -79,6 +79,44 @@ public async Task A_second_version_of_a_type_for_one_subscriber_is_reported_as_a } } + [Test] + public async Task A_merged_subscription_names_the_version_and_endpoint_of_the_row_kept_and_the_row_dropped() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing.V2"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 's/1' (version 1.0.0, endpoint 'Billing') and dropped the row from 's/2' (version 2.0.0, endpoint 'Billing.V2')."), + "both keys print the same, so only the source rows tell the operator which version survived"); + Assert.That((result.AlreadyPresent, result.Merged), Is.EqualTo((1, 1)), "the summary counts the merge apart from the other rows already present"); + } + } + + [Test] + public async Task A_subscription_merged_on_a_re_read_names_the_row_an_earlier_pass_copied() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var kept = ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")); + var first = await Target.Write(SubscriptionsCategory, BatchOf(kept), CheckpointAfter("s/1")); + var reRead = BatchOf(kept, ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), reRead); + var result = await Target.Write(SubscriptionsCategory, reRead, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 's/1' (version 1.0.0, endpoint 'Billing') and dropped the row from 's/2' (version 2.0.0, endpoint 'Billing')."), + "the stored row is the one 's/1' copied on the earlier pass, so the line names that source row"); + Assert.That((result.AlreadyPresent, result.Merged), Is.EqualTo((2, 1))); + } + } + [Test] public async Task A_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() { @@ -103,6 +141,7 @@ public async Task A_key_the_target_already_holds_merges_only_a_variant_the_colum if (addressIgnoresCase) { Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'")); + Assert.That(merges[0].Detail, Does.EndWith("It kept the row the table already held and dropped the row from 's/2' (version 1.0.0, endpoint 'Billing')."), "no row of this batch claimed the stored row"); } Assert.That(result.Copied, Is.EqualTo(addressIgnoresCase ? 1 : 2), "the other address is copied, and so is the case variant where the column tells it apart"); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs new file mode 100644 index 0000000000..e84ab5f282 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; + +// RecordingLoggerProvider lives in ServiceControl.Infrastructure.Tests, which the persistence test projects do not reference. +/// <summary> +/// Keeps the lines the migration target logs, so a test can read what the operator sees. Messages holds the warnings and Information the information lines. +/// Register it as an ILoggerProvider through RegisterServices. +/// </summary> +sealed class TargetWarnings : ILoggerProvider +{ + readonly ConcurrentQueue<string> messages = new(); + readonly ConcurrentQueue<string> information = new(); + + public IReadOnlyList<string> Messages => [.. messages]; + + public IReadOnlyList<string> Information => [.. information]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(EFCoreMigrationTarget).FullName ? new TargetLogger(this) : NullLogger.Instance; + + public void Dispose() + { + } + + sealed class TargetLogger(TargetWarnings lines) : ILogger + { + public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel is LogLevel.Warning or LogLevel.Information; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) + { + if (logLevel == LogLevel.Warning) + { + lines.messages.Enqueue(formatter(state, exception)); + } + else if (logLevel == LogLevel.Information) + { + lines.information.Enqueue(formatter(state, exception)); + } + } + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs index 534e120b7d..08f6365e87 100644 --- a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -6,14 +6,11 @@ namespace ServiceControl.Persistence.DataMigration; public static class HaltThreshold { /// <summary> - /// Whether the skipped rows are enough to stop the category. Both limits have to be passed, so the floor - /// ignores a few bad rows in a small category and the percentage ignores a small share of a large one. + /// Returns true, which stops the category, when skippedCount passes minimumFloor and its share of totalCount passes percentThreshold percent. + /// The floor ignores a few bad rows in a small category, and the percentage ignores a small share of a large one. + /// skippedCount holds fault skips only, never a harmless skip that <see cref="MigrationSkipReasonExtensions.IsBenign" /> accepts, and + /// totalCount holds every row dealt with over the same stretch, copied, skipped and already present alike. /// </summary> - /// <param name="skippedCount">Rows skipped as faults. A skip the product would have dropped anyway does not belong here.</param> - /// <param name="totalCount">Rows dealt with over the same stretch as <paramref name="skippedCount" />, copied, skipped and already present alike.</param> - /// <param name="percentThreshold">The share of skipped rows, as a percentage, that has to be passed.</param> - /// <param name="minimumFloor">The number of skipped rows that has to be passed before the percentage counts at all.</param> - /// <returns>True when both limits are passed, which means the category stops.</returns> public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) { if (skippedCount <= minimumFloor || totalCount == 0) diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index 449c4cf85d..3dd50089d9 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -57,5 +57,7 @@ Task<MigrationWriteResult> Write( /// <summary> /// What the target did with one batch, and the checkpoint it saved with it. Every skipped row has a reason in SkipReasons. +/// Merged counts the rows inside AlreadyPresent that the target dropped because their key matched another row's. +/// Changed counts the rows inside Copied whose stored value differs from the source's on purpose. The target logs each merged and changed row. /// </summary> -public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList<string> SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary<MigrationSkipReason, long>? SkipReasons = null); +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList<string> SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary<MigrationSkipReason, long>? SkipReasons = null, int Merged = 0, int Changed = 0); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs index fadc585355..6670fcfbcb 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -18,6 +18,7 @@ public static class MigrationCategoryIds public const string LicensingReportMasks = nameof(LicensingReportMasks); public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + public const string PendingIntegrationEvents = nameof(PendingIntegrationEvents); public const string EventLog = nameof(EventLog); public const string CustomChecks = nameof(CustomChecks); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs index 13e8a0fbca..e3db874daf 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -38,6 +38,7 @@ public static class MigrationCategoryRegistry new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + new(MigrationCategoryIds.PendingIntegrationEvents, MigrationCategoryKind.Required, CarriesBodies: false, Order: 13), new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 14), new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Required, CarriesBodies: true, Order: 15), new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Required, CarriesBodies: false, Order: 16, MustFollow: MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs index 8462c1e454..d68b8e6e61 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs @@ -23,14 +23,16 @@ state is MigrationCategoryState.Halted public static class MigrationSkipReasonExtensions { /// <summary> - /// Whether the running product would have dropped this row anyway, which makes the skip no loss. The halt - /// threshold and the settle rule never count a harmless skip, so no number of them can stop a category or - /// leave it Failed. The list is fixed. <see cref="MigrationSkipReason.Unknown" /> is not on it, because a - /// newer build's reason read back here may be a real loss. + /// Whether the skip is no loss, because the running product would have dropped the row anyway, or the row + /// is a result of a check the source's own storage runs about itself, which nothing reports on the target. The halt threshold and the settle rule never count a + /// harmless skip, so no number of them can stop a category or leave it Failed. The list is fixed, and + /// <see cref="MigrationSkipReason.Unknown" /> is not on it, because a newer build's reason read back here + /// may be a real loss. /// </summary> public static bool IsBenign(this MigrationSkipReason reason) => reason is MigrationSkipReason.PastRetention - or MigrationSkipReason.BlankGroupComment; + or MigrationSkipReason.BlankGroupComment + or MigrationSkipReason.SourceStorageCheck; /// <summary> /// Whether no retry can fix a row skipped for this reason. diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index e13538cdf7..2bbf58b1ff 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.DataMigration; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -31,6 +32,14 @@ public sealed class MigrationEngine( readonly MigrationBodyLoader bodyLoader = new(source, timeProvider, options.BodyRetryBackoff, logger); + readonly ConcurrentDictionary<string, (long Merged, long Changed)> mergedAndChangedThisRun = new(StringComparer.Ordinal); + + /// <summary> + /// For each category this engine wrote to, how many rows the target merged into another row and how many it changed on the way in. + /// The counts are not saved, so they cover only the batches this engine wrote, and a category it resumed counts none from earlier runs. + /// </summary> + public IReadOnlyDictionary<string, (long Merged, long Changed)> MergedAndChangedThisRun => mergedAndChangedThisRun; + /// <summary> /// The categories of one kind to copy, in the order to copy them. Optional categories the operator did not /// ask for are left out. @@ -167,6 +176,7 @@ public async Task<MigrationCheckpoint> RunCategoryAsync(MigrationCategory catego // The target has committed this row, so every halt below settles from it. A halt settling from the // older version would be refused by the store as a conflict and lose its reason. checkpoint = result.Saved; + mergedAndChangedThisRun.AddOrUpdate(category.Id, (result.Merged, result.Changed), (_, sum) => (sum.Merged + result.Merged, sum.Changed + result.Changed)); // The result states the batch's outcome twice, as its own counts and as deltas on the checkpoint // it committed. The halt threshold reads the first and status and verify read the second. @@ -239,7 +249,7 @@ public async Task<MigrationCheckpoint> RunCategoryAsync(MigrationCategory catego return await Settle(checkpoint with { State = FaultSkips(checkpoint.SkipReasons) > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); } - // Harmless skips are rows the product would have removed anyway, so they neither stop a category nor leave it Failed. + // A harmless skip loses nothing, as IsBenign says, so it neither stops a category nor leaves it Failed. static long FaultSkips(IReadOnlyDictionary<MigrationSkipReason, long>? skipReasons) => skipReasons is null ? 0 : skipReasons.Where(reason => !reason.Key.IsBenign()).Sum(reason => reason.Value); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs index 808d5090f6..2ea004b4bf 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs @@ -16,4 +16,22 @@ public static class MigrationMetadataKeys /// The time the source last changed the row's document, as a UTC <see cref="System.DateTime" />. /// </summary> public const string LastModified = "LastModified"; + + /// <summary> + /// The time the source last stored a failed error import, as a UTC <see cref="System.DateTime" />. + /// RavenDB keeps no failure time, so the target stores this one as the failure time, and replay runs the oldest first by it. + /// </summary> + public const string FailedAt = "FailedAt"; + + /// <summary> + /// A <see cref="bool" /> that is true when the row is a result of a check the source's own storage runs about itself. + /// Rows that are not such a result carry no value. + /// </summary> + public const string SourceStorageCheck = "SourceStorageCheck"; + + /// <summary> + /// A <see cref="bool" /> on every retry-issued failed message. It is true when the source still holds a retry claim + /// for the message, and false when the source had already released it. Other rows carry no value. + /// </summary> + public const string RetryClaimHeld = "RetryClaimHeld"; } diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs index 6c4d0fe806..4c848b4f89 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -28,6 +28,12 @@ public enum MigrationSkipReason /// </summary> BlankGroupComment, + /// <summary> + /// The source marked the row as a result of a check its own storage runs about itself. + /// Nothing reports such a check once the instance runs on the target, so a copied result would never change. + /// </summary> + SourceStorageCheck, + /// <summary> /// The document's unique message id is not a GUID, and the row's key has to be one. /// </summary> diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt index 1e54a214bc..6c99c6b3e0 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -10,6 +10,7 @@ Required 9: LicensingThroughput, after LicensingEndpoints Required 10: LicensingReportMasks Required 11: LicensedEndpointDetails Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Required 13: PendingIntegrationEvents Required 14: CustomChecks Required 15: FailedErrorImports, with bodies Required 16: GroupComments, after UnresolvedAndRetryIssuedFailedMessages diff --git a/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs b/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs new file mode 100644 index 0000000000..d8bdb1d464 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Persistence; + +[TestFixture] +class CustomCheckStatusTests +{ + // CustomCheckDetail carries the status as a bool, so a third member would be copied as Pass. + [Test] + public void The_status_enum_still_has_exactly_the_two_values_a_bool_can_carry() => + Assert.That(Enum.GetValues<Status>(), Is.EqualTo(new[] { Status.Fail, Status.Pass })); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs index c82e610f09..214b13a2d2 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -12,12 +12,12 @@ namespace ServiceControl.UnitTests.Migration; class MigrationCategoryRegistryTests { [Test] - public void Contains_all_seventeen_categories_with_unique_ids() + public void Contains_all_eighteen_categories_with_unique_ids() { using (Assert.EnterMultipleScope()) { - Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(17)); - Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(17)); + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); } } @@ -36,7 +36,7 @@ public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() "NotificationSettings", "TrialEndDate", "RetryOperations", "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages", - "CustomChecks", "FailedErrorImports", "GroupComments" + "PendingIntegrationEvents", "CustomChecks", "FailedErrorImports", "GroupComments" })); } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs index 8843923d12..23c4080bfb 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs @@ -2,7 +2,7 @@ namespace ServiceControl.UnitTests.Migration; using System; -using System.Linq; +using System.Collections.Generic; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; @@ -10,9 +10,38 @@ namespace ServiceControl.UnitTests.Migration; class MigrationCategoryStateExtensionsTests { [Test] - public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => - Assert.That( - Enum.GetValues<MigrationSkipReason>().Where(reason => reason.IsBenign()), - Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), - "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); + public void Every_skip_reason_is_exactly_one_of_harmless_permanent_or_retryable() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(Kinds.Keys, Is.EquivalentTo(Enum.GetValues<MigrationSkipReason>()), "a reason on neither list is treated as a fault a retry can fix, so each new reason has to be placed here on purpose"); + + foreach (var (reason, kind) in Kinds) + { + Assert.That((reason.IsBenign(), reason.IsPermanent()), Is.EqualTo((kind == SkipKind.Harmless, kind == SkipKind.Permanent)), $"{reason} is {kind}"); + } + } + } + + enum SkipKind + { + Harmless, + Permanent, + Retryable + } + + static readonly Dictionary<MigrationSkipReason, SkipKind> Kinds = new() + { + [MigrationSkipReason.BodyUnreadable] = SkipKind.Retryable, + [MigrationSkipReason.PastRetention] = SkipKind.Harmless, + [MigrationSkipReason.RequiredValueMissing] = SkipKind.Permanent, + [MigrationSkipReason.BlankGroupComment] = SkipKind.Harmless, + [MigrationSkipReason.SourceStorageCheck] = SkipKind.Harmless, + [MigrationSkipReason.UnparseableUniqueMessageId] = SkipKind.Permanent, + [MigrationSkipReason.NoProcessingAttempts] = SkipKind.Permanent, + [MigrationSkipReason.ValueTooLong] = SkipKind.Permanent, + [MigrationSkipReason.MalformedValue] = SkipKind.Permanent, + [MigrationSkipReason.BodyMissing] = SkipKind.Permanent, + [MigrationSkipReason.Unknown] = SkipKind.Retryable + }; } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs index 15cf13aaad..ec774e7a09 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -75,7 +75,7 @@ public async Task A_category_removed_from_configuration_leaves_its_checkpoint_ro var target = new InMemoryMigrationTarget(checkpointStore); var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, [stillConfigured]); var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); - var previousRun = new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + var previousRun = new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.InProgress, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, null, null); await checkpointStore.Upsert(previousRun); var results = await engine.RunCategories(engine.SelectCategories(MigrationCategoryKind.Optional)); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs index 6135b96aa5..484a171e1a 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -3,6 +3,8 @@ namespace ServiceControl.UnitTests.Migration; using System; using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; @@ -248,4 +250,42 @@ public async Task A_category_already_Complete_is_left_alone_on_a_second_run() Assert.That(target.WrittenRows(category.Id), Is.Empty); } } + + [Test] + public async Task The_merged_and_changed_rows_the_target_reports_add_up_for_each_category_in_this_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d"), Row("e"), Row("f")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, new MergingTarget(checkpointStore), checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + await engine.RunCategoryAsync(category); + + Assert.That(engine.MergedAndChangedThisRun, Is.EquivalentTo(new Dictionary<string, (long Merged, long Changed)> { [category.Id] = (2, 4) }), + "the checkpoint does not hold these counts, so the summary after the copy has only what the engine adds up"); + } + + // Reports one row of every batch merged and two of the rest changed. + sealed class MergingTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(3); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(batch.Rows.Count - 1, 0, 1, null), cancellationToken); + return new MigrationWriteResult(saved, batch.Rows.Count - 1, 0, [], AlreadyPresent: 1, Merged: 1, Changed: 2); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } } diff --git a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs index 91945ae6e9..0bd83978d7 100644 --- a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs +++ b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs @@ -197,7 +197,8 @@ public void A_failed_category_that_skipped_rows_is_told_a_retry_re_reads_them() [MigrationSkipReason.BodyUnreadable] = 1 })], logger, - RunStartedAt); + RunStartedAt, + NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -224,7 +225,8 @@ public void A_failed_category_whose_faults_no_retry_can_fix_is_pointed_at_abando [MigrationSkipReason.RequiredValueMissing] = 2 })], logger, - RunStartedAt); + RunStartedAt, + NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -245,7 +247,8 @@ public void A_halted_category_whose_faults_no_retry_can_fix_is_told_abandon_give [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 120 }) with { LastError = "Halted: 120 of 127 rows skipped in this run exceeds the configured threshold of 5% and 100 rows." }], logger, - RunStartedAt); + RunStartedAt, + NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -268,7 +271,8 @@ public void A_halted_category_with_faults_a_retry_can_fix_is_also_told_what_aban MigrationStartup.ReportWhatTheCopyLeftBehind( [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 120 })], logger, - RunStartedAt); + RunStartedAt, + NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -288,7 +292,7 @@ public void A_category_that_failed_with_only_harmless_skips_is_still_offered_a_r with { LastError = "TimeoutException at cursor r-7: the target did not answer" }; - MigrationStartup.ReportWhatTheCopyLeftBehind([stoppedByAnException], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([stoppedByAnException], logger, RunStartedAt, NothingMergedOrChanged); Assert.That(logger.Entries.Single().Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them").And.Not.Contain("--migration-abandon"), "no fault skip stands in the way of a retry, so the exception's cause is what the operator fixes"); @@ -302,7 +306,8 @@ public void A_done_category_with_harmless_skips_is_not_offered_a_retry() MigrationStartup.ReportWhatTheCopyLeftBehind( [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 3, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.PastRetention] = 3 })], logger, - RunStartedAt); + RunStartedAt, + NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -314,12 +319,59 @@ public void A_done_category_with_harmless_skips_is_not_offered_a_retry() }); } + [Test] + public void Every_harmless_skip_reason_has_its_own_explanation_and_no_other_reason_has_one() => + Assert.That(MigrationStartup.HarmlessSkipExplanations.Keys, Is.EquivalentTo(Enum.GetValues<MigrationSkipReason>().Where(reason => reason.IsBenign())), + "the warning for a Done category explains each harmless reason it skipped, so a harmless reason with no explanation would be left out unexplained"); + + [Test] + public void A_done_category_is_told_why_each_of_its_harmless_reasons_is_harmless_and_nothing_about_the_others() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 8, new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.SourceStorageCheck] = 6, + [MigrationSkipReason.BlankGroupComment] = 2 + })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var message = logger.Entries.Single().Message; + + Assert.Multiple(() => + { + Assert.That(message, Does.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.SourceStorageCheck]).And.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.BlankGroupComment])); + Assert.That(message, Does.Not.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.PastRetention]), "a reason the category did not skip for would give the operator a false reason"); + }); + } + + [Test] + public void The_refusal_marks_a_harmless_reason_so_it_does_not_read_as_rows_a_retry_could_copy() + { + var failed = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.CustomChecks) with + { + SkippedCount = 7, + SkipReasons = new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.SourceStorageCheck] = 6, + [MigrationSkipReason.MalformedValue] = 1 + } + }; + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([CustomChecks], [failed], NewSettings())); + + Assert.That(exception.Message, Does.Contain("SourceStorageCheck 6, harmless").And.Contain("MalformedValue 1, no retry can fix"), "a retry skips the six harmless rows again"); + } + [Test] public void A_blocked_category_is_left_to_the_refusal_rather_than_reported_as_a_clean_copy() { var logger = new CapturingLogger(); - MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Blocked, copied: 0, skipped: 0, null)], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Blocked, copied: 0, skipped: 0, null)], logger, RunStartedAt, NothingMergedOrChanged); Assert.That(logger.Entries, Is.Empty); } @@ -332,7 +384,7 @@ public void A_failed_category_that_skipped_nothing_is_not_reported_as_a_clean_co var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; - MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt, NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -348,7 +400,7 @@ public void A_category_that_skipped_nothing_says_so_without_raising_a_warning() { var logger = new CapturingLogger(); - MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null)], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null)], logger, RunStartedAt, NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -370,7 +422,7 @@ public void A_category_an_earlier_run_finished_is_not_reported_as_copied_again() with { SettledAt = RunStartedAt.AddMinutes(-5) }; - MigrationStartup.ReportWhatTheCopyLeftBehind([alreadyDone], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([alreadyDone], logger, RunStartedAt, NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -393,7 +445,7 @@ public void A_category_this_run_finished_is_still_reported_as_copied() with { SettledAt = RunStartedAt.AddSeconds(2) }; - MigrationStartup.ReportWhatTheCopyLeftBehind([justDone], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([justDone], logger, RunStartedAt, NothingMergedOrChanged); Assert.That(logger.Entries.Single().Message, Does.Contain("7 copied").And.Not.Contain("already finished")); } @@ -408,7 +460,7 @@ public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() with { SettledAt = RunStartedAt.AddMinutes(-5) }; - MigrationStartup.ReportWhatTheCopyLeftBehind([failedEarlier], logger, RunStartedAt); + MigrationStartup.ReportWhatTheCopyLeftBehind([failedEarlier], logger, RunStartedAt, NothingMergedOrChanged); var entry = logger.Entries.Single(); @@ -420,6 +472,68 @@ public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() }); } + [Test] + public void A_category_that_merged_or_changed_rows_says_how_many_apart_from_already_present_and_warns() + { + var logger = new CapturingLogger(); + var done = Settled(MigrationCategoryState.Complete, copied: 15, skipped: 0, null) with { AlreadyPresentCount = 3, SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([done], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (1, 4) }); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Is.EqualTo($"{MigrationCategoryIds.EndpointSettings}: 15 copied (4 changed), 2 already present, 1 merged, nothing skipped"), + "a merged row is counted as already present, so without taking it out the summary says nothing changed"); + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning), "a merged or changed row is one the operator did not expect, and the per-row warnings above name each one"); + }); + } + + [Test] + public void A_category_that_merged_rows_and_skipped_some_counts_both() + { + var logger = new CapturingLogger(); + var failed = Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 2, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 }) + with + { AlreadyPresentCount = 5 }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([failed], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (2, 0) }); + + Assert.That(logger.Entries.Single().Message, Does.StartWith($"{MigrationCategoryIds.EndpointSettings}: 7 copied (0 changed), 3 already present, 2 merged, 2 skipped (RequiredValueMissing 2).")); + } + + [Test] + public void More_merged_rows_than_rows_already_present_is_a_warning_naming_both_counts() + { + var logger = new CapturingLogger(); + var done = Settled(MigrationCategoryState.Complete, copied: 5, skipped: 0, null) with { AlreadyPresentCount = 1, SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([done], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (3, 0) }); + + var messages = logger.Entries.Select(entry => entry.Message).ToArray(); + + Assert.Multiple(() => + { + Assert.That(messages, Has.Member($"{MigrationCategoryIds.EndpointSettings}: the target reported 3 merged rows but only 1 rows already present. Its key comparer disagrees with the database about which keys are equal, so the merge lines for this category may name the wrong rows."), + "every merged row is also already present, so more merges than that is the comparer predicting wrongly"); + Assert.That(messages, Has.Member($"{MigrationCategoryIds.EndpointSettings}: 5 copied (0 changed), 0 already present, 3 merged, nothing skipped"), "never a negative count"); + }); + } + + [Test] + public void A_failed_category_that_skipped_nothing_still_counts_its_merged_and_changed_rows() + { + var logger = new CapturingLogger(); + var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { AlreadyPresentCount = 3, LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (1, 2) }); + + Assert.That(logger.Entries.Single().Message, Does.StartWith($"{MigrationCategoryIds.EndpointSettings} is Failed (Halted) after 10 copied (2 changed), 2 already present and 1 merged.")); + } + + static readonly IReadOnlyDictionary<string, (long Merged, long Changed)> NothingMergedOrChanged = new Dictionary<string, (long Merged, long Changed)>(); + static readonly DateTime RunStartedAt = new(2026, 9, 20, 12, 0, 0, DateTimeKind.Utc); static MigrationCheckpoint Settled(MigrationCategoryState state, long copied, long skipped, IReadOnlyDictionary<MigrationSkipReason, long>? skipReasons) => @@ -428,6 +542,7 @@ static MigrationCheckpoint Settled(MigrationCategoryState state, long copied, lo static readonly MigrationCategory EndpointSettings = MigrationCategoryRegistry.Find(MigrationCategoryIds.EndpointSettings)!; static readonly MigrationCategory KnownEndpoints = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; static readonly MigrationCategory MessageRedirects = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + static readonly MigrationCategory CustomChecks = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; static readonly MigrationCategory UnresolvedAndRetryIssuedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; const string UnknownCategoryId = "SomeCategoryFromANewerBuild"; diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs index f36d874995..9c0626bd7e 100644 --- a/src/ServiceControl/Migration/MigrationStartup.cs +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -117,7 +117,7 @@ await MigrationStartupCheckRunner.Run( RunRequiredCategories(engine, toCopy, checkpointStore, timeProvider, logger, cancellationToken), settings); - ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt); + ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt, engine.MergedAndChangedThisRun); RefuseIfAnyCategoryDidNotComplete(toCopy, finished, unfinishedOutsideTheCopy, settings); } @@ -355,7 +355,7 @@ [.. checkpoints internal static string FailedDetail(MigrationCheckpoint checkpoint) { var reasons = checkpoint.SkipReasons is { Count: > 0 } counts - ? $" ({string.Join("; ", counts.Select(reason => $"{reason.Key} {reason.Value}{(reason.Key.IsPermanent() ? ", no retry can fix" : "")}"))})" + ? $" ({string.Join("; ", counts.Select(reason => $"{reason.Key} {reason.Value}{(reason.Key.IsPermanent() ? ", no retry can fix" : reason.Key.IsBenign() ? ", harmless" : "")}"))})" : ""; return $"{checkpoint.CategoryId} is Failed ({checkpoint.State}) after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{reasons}{LastErrorClause(checkpoint)}; " + @@ -372,8 +372,11 @@ static string LastErrorClause(MigrationCheckpoint checkpoint) => /// <summary> /// Logs one line per category saying what it copied and what it left behind. Skipped rows are warned about /// one at a time while the copy runs, and nothing else states the total or says what becomes of them. + /// The merged and changed counts are this run's, by category id, and the line counts merged rows apart from the other rows already present. + /// A category that merged or changed a row is reported as a warning even when it skipped nothing. + /// A category with more merged rows than rows already present gets a warning naming both counts, because the target's key comparer then disagrees with the database. /// </summary> - internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList<MigrationCheckpoint> finished, ILogger logger, DateTime runStartedAt) + internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList<MigrationCheckpoint> finished, ILogger logger, DateTime runStartedAt, IReadOnlyDictionary<string, (long Merged, long Changed)> mergedAndChanged) { foreach (var checkpoint in finished) { @@ -393,18 +396,29 @@ internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList<MigrationCheckpoi continue; } + var (merged, changed) = mergedAndChanged.GetValueOrDefault(checkpoint.CategoryId); + + // Every merged row is also counted as already present, so more merges than that means the merge lines were predicted wrongly. + if (merged > checkpoint.AlreadyPresentCount) + { + logger.LogWarning("{CategoryId}: the target reported {Merged} merged rows but only {AlreadyPresent} rows already present. Its key comparer disagrees with the database about which keys are equal, so the merge lines for this category may name the wrong rows.", + checkpoint.CategoryId, merged, checkpoint.AlreadyPresentCount); + } + + var alreadyPresent = Math.Max(0, checkpoint.AlreadyPresentCount - merged); + // An exception, a stall or unbalanced counts fail a category without a skip, and that is not a clean copy. if (checkpoint.State.IsFailed() && checkpoint.SkippedCount == 0) { - logger.LogWarning("{CategoryId} is Failed ({State}) after {Copied} copied and {AlreadyPresent} already present. The refusal that follows says why and what to run.", - checkpoint.CategoryId, checkpoint.State, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + logger.LogWarning("{CategoryId} is Failed ({State}) after {Copied} copied ({Changed} changed), {AlreadyPresent} already present and {Merged} merged. The refusal that follows says why and what to run.", + checkpoint.CategoryId, checkpoint.State, checkpoint.CopiedCount, changed, alreadyPresent, merged); continue; } if (checkpoint.SkippedCount == 0) { - logger.LogInformation("{CategoryId}: {Copied} copied, {AlreadyPresent} already present, nothing skipped", - checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + logger.Log(merged + changed == 0 ? LogLevel.Information : LogLevel.Warning, "{CategoryId}: {Copied} copied ({Changed} changed), {AlreadyPresent} already present, {Merged} merged, nothing skipped", + checkpoint.CategoryId, checkpoint.CopiedCount, changed, alreadyPresent, merged); continue; } @@ -416,15 +430,28 @@ internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList<MigrationCheckpoi var whatBecomesOfThem = checkpoint.State.IsFailed() ? WhatARetryCanDo(checkpoint) : checkpoint.State == MigrationCategoryState.Complete - ? "They were left out as harmless, because ServiceControl would have removed them anyway, and they stay only in the source database." + ? string.Join(" ", ["They were left out as harmless.", .. HarmlessReasonsIn(checkpoint), "They stay only in the source database."]) : "They stay only in the source database."; logger.LogWarning( - "{CategoryId}: {Copied} copied, {AlreadyPresent} already present, {Skipped} skipped ({Reasons}). {WhatBecomesOfThem}", - checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount, checkpoint.SkippedCount, reasons, whatBecomesOfThem); + "{CategoryId}: {Copied} copied ({Changed} changed), {AlreadyPresent} already present, {Merged} merged, {Skipped} skipped ({Reasons}). {WhatBecomesOfThem}", + checkpoint.CategoryId, checkpoint.CopiedCount, changed, alreadyPresent, merged, checkpoint.SkippedCount, reasons, whatBecomesOfThem); } } + /// <summary> + /// Why each harmless skip reason loses nothing, in one sentence or two that the operator reads in the log. Every reason <see cref="MigrationSkipReasonExtensions.IsBenign" /> accepts has one. + /// </summary> + internal static readonly IReadOnlyDictionary<MigrationSkipReason, string> HarmlessSkipExplanations = new Dictionary<MigrationSkipReason, string> + { + [MigrationSkipReason.PastRetention] = "PastRetention rows were past the retention period, so ServiceControl would have deleted them soon anyway.", + [MigrationSkipReason.BlankGroupComment] = "BlankGroupComment rows are blank group comments, and ServiceControl keeps no row for a blank comment.", + [MigrationSkipReason.SourceStorageCheck] = "SourceStorageCheck rows are results of the checks the source's own storage runs about itself, such as the RavenDB persister's checks on its database. Nothing reports those checks once the primary instance runs on SQL, so a copied result would never change." + }; + + static IEnumerable<string> HarmlessReasonsIn(MigrationCheckpoint checkpoint) => + checkpoint.SkipReasons?.Keys.Where(reason => reason.IsBenign()).Order().Select(reason => HarmlessSkipExplanations[reason]) ?? []; + // A retry reads the whole category again, so it is offered when a retry could copy some of the skipped rows, or when a halt left rows unread. static string WhatARetryCanDo(MigrationCheckpoint checkpoint) { From e79b1e9e4dc8c76ac722b0229dcf4dfde0c8001a Mon Sep 17 00:00:00 2001 From: Warwick Schroeder <warwick.schroeder@particular.net> Date: Wed, 18 Nov 2026 11:34:51 +0800 Subject: [PATCH 28/28] Remove logging of skipped IDs from migration engine processing --- .../DataMigration/MigrationEngine.cs | 5 ----- .../MigrationEngineFailurePathTests.cs | 19 ------------------- 2 files changed, 24 deletions(-) diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs index 2bbf58b1ff..77c3fcbd4e 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -189,11 +189,6 @@ public async Task<MigrationCheckpoint> RunCategoryAsync(MigrationCategory catego break; } - foreach (var id in result.SkippedIds) - { - logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); - } - processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; skippedThisRun += bodySkips + FaultSkips(result.SkipReasons); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs index 9d02e7bb4d..39ce6653c2 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -377,25 +377,6 @@ public async Task The_halt_reason_says_at_the_start_when_the_first_batch_never_c Assert.That(checkpoint.LastError, Does.Contain("at the start")); } - [Test] - public async Task Every_row_the_target_skips_is_named_in_the_log() - { - // The counts say how much was left behind. Only the log says which rows, and it is the way back - // to them while the RavenDB database still exists. - var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; - var source = new InMemoryMigrationSource(); - source.Seed(category.Id, Row("a"), Row("b")); - var checkpointStore = new InMemoryMigrationCheckpointStore(); - var target = new InMemoryMigrationTarget(checkpointStore); - target.RejectKey("b", MigrationSkipReason.PastRetention); - var logger = new CapturingLogger(); - var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); - - await engine.RunCategoryAsync(category); - - Assert.That(logger.Entries.Select(entry => entry.Message), Has.Some.EqualTo("Skipped b in category KnownEndpoints")); - } - [Test] public async Task A_stop_between_batches_leaves_the_row_in_progress_at_the_batch_that_committed() {

xo`JizLP*zEnp&Ra1xoksRq=eQbGj`$cg}fn%^A4J za54@h;Q875eF#0h%2XB6pL_7Lc<0EhVO`SRLl6%6b($e+$O5@&x-|$3SaTGzg`g?p zM?H|XHm!f#y1oc5Uv7Z`hk@X!R}^u;jPAekit+xNub0j0aV$oB_r;+wN8>BRW~Z?pYkDxU^fW2ecHJtwzrv0UF7L*IF4PtYSLb9 zJ}K}Zfu?5OZW)<`4qMNc$l9?mHr=+T{;o%1Gp}r`+Lpdo&3b4!Xy5K zCFJf5Vwqlq#!T>{hPJ>TsNy>P?a~^XF3kOU2x|%ID&30W#1Va!8robq7N3TyHI`2+ zffu%RTDjh^G{6C*qZFG#Iso5zJT1rJ?-qQ-Yj=MUGzyHa3acHlgxid&QR)e6zMuIG zm#lm%DdTTf@?6vpxcCM0hj!^fNciw8nS(E>m*>$ui=*S&Y0xBn3Clvx_^!i{JX_}~ zHb(25<*!iAP9`3AO#L%&KYD}bxl^f#6)<7c^Ur0DVRPzdVNx{&9|CW{vUtE?LhF8r zAc*o5!ITClNok(-uIU@)O+u@}#waTa=*}-pY>Aym^Vf1J`ihW$pU3?oKb6jjxdu)G zw#S^9<)x4^=v8D3S(+_?Lo|(_)E<@uihW~ZE6X#-c5E4zeEAgh{r%A)$kZJ34j8kd zV5ljTBB=~RhQMt_2~ykR$TV!10P)OIP7a;0fLOxX%~0vKUTrKsgstN@gX2zPWS^X` zL`d$J5!hR|2W1F;5&i3d-2O!O(c$qa})j|;}MN#s9hNE6%7%W<$2M{rG z|H_{ECO6606RHVU_@x@IS_J+b?>VhB^E*E-FkerojB@2s%4so86i&kJoyAhjA$HC`fODOqb!a2Rdi-xDCsp$B0XLraGE?O}&$H)*; zoV_SXHEY1h*IH8d`_?tEO=FwLn;_AO)l4+|jDu{-~ZZzRrnVuhVT zlkH(t1uDanQM*7$co!fAiWb;ff#J zWssnpATec_K^4$`Eodiu*HUW_d}Zm^FcYqZE&*j7(cjf6mENx()D~9pxKYMEa+Z7k?9t7P?Trl$ zg&8}zOVg`HszvS$nH+rh4JytDP%dsUjfGN_^u6Dt7&K0(IOjnTe`I0Dn`{K#T5acO z0j6H>?4~I-<@BGXUx9a$`s=dwyPv(Yb2nCR*SzAZF%l zc6y+FI77n)Ks82S!MDX%L512cu5gwBLvcbr+2OnLi+*Nr#AhAj%%04zMHV8b38mf^MCXczK}!&S#g;58*<_8^WU* z5o3(^3N6b8Yo}4sh_Sxhm<|7eL&*1QhKXtSphO3)^{QGt_Hpp6ILAX49l~Wsru{TF za7NyoX>wQ<`pJQWvzIjbdphiiCqs`F1$44*Zz;o%WOCsk)P>BKxEV4eLg%KUD(By| zAEqA)?|8q>VKd(r5yBbM?ku9>6?||((Xch{c-GT0N&32~nqsv-(Fm$GrL;TJGWskl z#{PjohXW?i6`}U1LWHPesGi95>_)H-IfQR#p^p0HaQe4V#v0SxM$b8KX7UCH80>xgXD`WaB$F&KU$?z`4^UkR$!Bqe2eWDT1s$h%OCQ&7o+kQZx+NcN zUysY07eIxUi}u1+fEn(kFL7g)XvjF=*p>;45<{*R=PEc2mv`^vB!q`01WEM&n$~UE zM%6u`*MYz%)|`LqQip>7$$g$4rGBM$G{qn5twSQ&G&T2S2qF6JI^!CgSBVDdq(L?#JiH0M>w4GnZ4ebC8!dt7%YnZPpF}eoSfJ}y8c**IiZ`F_tOI(# zEKmS1Lob|Zcy$eP6?`XaU}kBE`t_ZF3eqa!aZ<5>D}xLyO(r0gX1#s;vXcW>q6dD& zw%kfnqyLLxKR&l&&kVqd0Hh4O*!*LniQU`6zv;Yn$DsJkeLD|GfHRXRP)aL7Cm} zXYiU~Tt5G2aC3f^xy%eDmG}qY=}znXFV#^AEkZ6%#WAkjH4F|0_;w#%rgU{&O`nzp z(4etg;<0}?-3FGe6Ki}yf4Oa6J5fep>6qprr+P_#2$=nmqJx2u5~=>nwcA!@htwK> z1%GG8*=a&$qH|EYX*+19qNF+5_jz;<-8|QdeEQsS=4pJ<{Eke#z1eK79kCHFO_Km3 zuLN0GVv_p#plp1?q>INrwU%XW9Vtmv(aGmKx#_{oIvs{UR;M$E&Cp1fUjpOY7&kFA zv!vz2)?J7K^H23ozPp-AHSZ~<$ zJ{1~{Dj^9yW&jXOo5%1#@Rx23KM6m5QtqpRw`U8d2`=(`7@&1YNNCwRZJ=l5c7IkI z9WL~9BRj=B#fH&gT51Su+^v#6c(8LUhKb?lF?XHqqO_H=M&7DGzUBEl390QS`4 zt&;Ay?D&-mCbx`X^sQDM9i12pBPYJqc*8;9Yp@g_=y4ZU=Y+W5vZ3c>c<8i>g+oI) zn1yU=XqBW-fumKz7p6+kZyk=@U67=4VG8Li_SsbCkm)#$vtS`7Nt!+8@gHOD6f?;;_ukHiH35KvYgXYi(v zBo~7Cx2F@%zJI^}i=hg>d%4=1Pgk?Quno`_O*&}2hQ8fZd%v02z)*E#By2pL*hhVs zc$@!a5KYJEJWFYbFwgXGzSh>}(IHTZmo&DYJL24A7a%0qW<&c61~$c9=rJ52&Zo|D zWE}vT{B1swU9Q z2n%#e<$I)Sa|BuXO#M32yGM8z#RR@|QOAvN{WvO1jD655a20+pFajK3x2*7r`wy_yyKFaA{g(o6xiCTMVTe%wGX?L|6aVP2ha zI`35H8KZKeA0S?%Xg+C*LM%DfH12bk*P}UA*j#z~z*=1Q#>rs`5X_$@Kcj}(?XdJ` zN}kcpm3WX^SP$tCIDDG5ZbI$#9*hWI)lKM}q9u2>dtHeRl%-tt6)*TKj*j`r+p?9Q zL#WnqGJ!t`F&BhOY+knR>7H*~5Z&_d{=l8Ry=-4wHYako=yrkH6J1PTXnxu7c6l|q>JHLwu zWjZNB`hB{9C%lpEAEFphVCrZ*5deOBOuE1MU5|}iRo`ti`=E-n3+AyYyp5#jk#XAo z?f*Z}a>7Mrg0r-73bV{%%blaA#-2ME?}{f)mDDW{o(XkhTFcaMc4QM$^% z!PU;$>#+@jL_FM9txlmA^Y;+S7{!%yw18*7c4?&pxpb?+A#adTS2pcY90QhhxQq4x91FHx*Rh?A_fx1CjM9EQR0$(+z$S6UW9M2=zCn>~BSRiQ)hGmhu(1L5=*~VW3w*o8^VW`P?1o?=+GdZ5;ZPLL+`l$AG+Q# zI;<^h*N&6MXxyYh<21HyJB@8yjcwbut;V+97>&*LtaR^v-glgDd}H)?lPt_N=Z)*> zWVUABP~AEq#UZ=PCb+|FZ=6riB0D*}uTVsHzl6!)7QHeZ!~Co>zs7R#a~0ENt~Uad zzTOW%2(U9}{X~~+HjAsQ?C`#(SA|?!^CY@`TVt*_bLR$TxsDCcx|mv_bHuc?$E1)& zN=tt)5rM(ffcT-!Fl~R z^#~r40~y*Az6j)B{1M2aj6mNbx~s%=?wtyg75xi|#%EkgB03Xl@t^iJ+!*%xcnCIn zfwU+H%`^qi)X2!hi29Q$x|>(~hj?8Am>tc6A>pE%O$;l|=SPvvm!-t$ZfnnaW*W!x zoHhh<9# zoUrIq`p)`CXPB4Y-K{TehPc&<_g@B1QC8cZe@fOuyIoY`EJkPuXM4c_&&x1@L4hGr z1AB2wKNo?Z_+pdg*IPOu$-3*`^N6*ni1qpD?&M9wer%0ulfN8m#;2=fa%?#t4C0u8 zMy8TDKSU}-GpUU3;O1Q3NUn}e9NRDgUM(@bNfJQ@%q-y2J)DZv!R}*y_1$1PL}#x% z#(m;<^LwM?Q|%VOfXk2c#ktGu7!k*y{hQv}7T&@(YxQvH`7_?>m5vhi84nyhwd>49 z%STg0f4g2AOP^@?$DqkwhAzYR7UdfRc!>@)eLs$m6)P;cRDy<8a^AKkH8e+}00%=Ig{roSytYswNea+P{({#|lZUMWE6Z z8tl|TKD9GkV}B5X&lDCbVxN39 zy?b#Wp-lh{pCfgqG@lMU*^ejvjb`e11(A3wC2H{&O253qJzg}HO6il!*+#8zk77ZV z^|bxnFE4*lSx=`2RTS%`v9Si;s3dNTy$k-y?(1=1XMbim z^CR}P05?*j!_%28G{IM`JK+}>$L5pwXa5%3eF{PSJ+GOgdQzU&uSgAugY~$7AU_fU*jz z&L}O4>PP2q=aW^nmNK}cm!k!;YUvPI3|ty#yoOQNk7k~2>n+?s^2o>CAzd0>q>xP1 zH}#QbZqLuJa;T&<^^}zz?BrA5fAdD(AdpzU!CZ~t$VWTg`vI#l|D9XURPz- zl75qNm-9x~J8rb@v|oWo5AxU^;*znq#D6q$)7alYPG7d?me;3R3geO@n}lABL9A}CbpJ~32X z{f4Y~0^pwEB{Vgk88RT1!EvUJ*U8YV_T(SjrFTo{jN2oT5sbrqR>+Kyzn?eV57d${ zU`eKhgBO{`i^TR6kX^F4hYI_d7&mcr`1_JxyX=q77IWjk>>Kr}e5l?Ix`+QcCX{rz zPf{`q!f1 zB#ftjp>%SEa*`^A`h{Ai!Hph~Kdirz%V^S1MeH%WKGT{bK?aU{cR5NO+h^ z4ie&0Nqc@|P&mU^(TI-UGwIv;l4oafy8UJKDI7d?4T^K>!(gOky_mn+82X&Vai|pW z1Xz=hT5Lvp#B4j9RfgbuN_g+SsBrer{D-_Uha_n{wiVfe@|GNCrlt1~Diz}s16(8e z9JO&7_eT*&I$p@RtHB00{hQDUZeM69i_HGBljz&av5@4Kke>?RWmTB z{ANZl12C$xp~=WK;rr zIwI+(R$HpbDSf(8>cB?YLd@v{%yy2ZE~+@V|8rFUgJO$U0GC zvT$5lytZR(l6)e6xe%z6o3Rg=6)Hcfa53D^p>(tBDCXP4-S}z<8R$q-&x)+p6-9!! zLkC+{)o_@!rP0y9`NUcepS#s5JtOx&%<&9b#wh!(#zuw( zDB~RP&vJzosmK9qUDx10y%?X(z8uN437pj6tp#~?#Y)OljIoMiAF))~g;ws2GR*7ZI1-H}2KGy*ho|fcxz*Mx=k~1qqj|kw9IS>@eKs2X`uxtw>7t$%yJ7F?b zRIS|kVsJ2Dy#xh}PpabzI)D zMPeCHK zT+qirb9u1*X&OeKIk``&3MRN7>yV;sHYHjB!X%{yQIc?p#yDXx!Wb}B#R5oOA)vi) z`GS;n8?x@~oHE>q+^`G_3u^+8*+ziD;$1}yqn8waw^hK$a#Gfj8RSzgB{kMkl~k-=j4#v)1%OZam8BMDxkkLdpS_*ffDM{^SPl)u=dyv8Wu zJbJ5z`8ZLj$tt*Nv?Wr4KO7?gK^qK}YKw%}=t2ps; zW)W7DUzc!k_eeNLgh8YTtu5nzz4^^Q4+44(^%#AvS70Vh|hrC61do1Rg+d1&LHiw?Z@x zIKI?G)RHhRRTqVwqDv+7E-~r5hT(&j6+q)Mrt{~$MEt~9LJ@qw#-=rX0ev>sj1myw@e&muB;pKU< zaXj}=#w7I3-=4u7F}9V&p#mC-aUC@T@F0%ifsj zVi+tku1s=J zb8;C*x`l3F#o^yOhny7IV<4q-5y5#HDYq=9o5@*;8%JVX&-Ld;!G0|yE1qps%7hsa zIJEaO9w`+=7#D{BTXO+$GeDsmXvO955=0wfs9$GMTM9IonOc$`-tV|2ILR3jlm%We zo!g~Dy)gz>G{Egb_2UUs8Zcp8zixuE`>lD|DKtbojY|L{y~Ho+>T!4=M22*X`f^5 z^$J;R&R_X+E0w9^WTER{lN}ScEKx_sS!Z)tXWms&E^yOxxHn~E8R0rM&C#(iab+9c zE`P4fpP@jc-4LJEAMf>yDIMKJ0tZoO$Zf&xo(ttjQ-&jTEEWi+Q`5p35|zpO*HZr% zIcTkM?QKl_F|k1m@+BI_U!D%>F5>0uO~e%C2YBbYg<4sKafjxrhJx9ly--UMhRa@E z1_sD{vIHOnu90M}reHSpYXw+r3E7Glk;zlh-S+f*Fm%ztXHzCK(sEg$P#g3TW1Rcp z*7eM(a7X>0+)@QXzm+oizE4UkSvXRgrMcxKj_l}WG8--Z(O<0e3Ab7t- z@;-IyM;n}5dV3Dq98K$0GW{<^RNqR8$%OR|9DexFnkfP z!Hjs8*yRY{s^8&uO2ZbklG9H&&>J3L^obo?aV=IoJM3!iry|R94ZT>c@DF`O49@TR z+`dXlp5QH45H?$=jjmjum}P2HU~z*e1nzGfVAJ7|hcQh1zCamYi|d9a*|wNK`0|e9 zid11yeBN+0tsY-+6rQ}`v9+Y-0X4H2wwV}|3cknr^RmAeCSQMp1*kyT*v-!Pt=F0! zYqbI4b*!kv7uQN2zhS*!=dy<{{&-yp_sABLO>jx)Un+%1uD>(Mdez+?FvA!dft0T#ruc2Pgc#>>=3UU&D&v^KA|9cF{m z-HBemL4@qu+hW^^wFY@a*6{ETpQ|_O2?0p#WAv2iZ%Mc-^9To*sCTH#e`M1@r)mQK z;5@E`Jq;1vOzBA-=(j_aSV2+O(@@v6b1`5^wD$Xq|D*-lEQuO4u0?}v!+vT-poZ-x zM%y$1(GOS~fq#a&{Qm4dc0DAty1+mla3mlV!g@G=XGGvu@aG4+?kduDG{IXkkoK>0 zG?<*q%GH4K?sZ^i$}FOx1)mfBS0owc%Ll5?Bde|ki`F=BU|Q|ShB-U`q`QIX0C@Zx zz7e${@i!-*Lc_;mCfFN&V0whGboyJM1gEAKda@^4GkjW}zAx1^Z8QAWPCO?6pE6~} z=bM4#H^^|3T$ml|246=1ZQva^_8@gbg0=eQM13EeO z5_@$@fe9W~qAtLtXeMUC@qAiUYXR$1g$^4|~}a~V4`!L-OpX$d9)d~>bt{!|I7P8yD9!>)%}$|!z^RK9Th94T7P-zntx{rX21jQ&F3q3 z5MUXgZg@}N5C8Nfh3E$Ulti&qAxi05yiH$19q@9ZeULP6W_-PEe9$1N6rs(1i#Z|l z2zwI?gM0<7d|+IdVXGMhG#0Z1&hDKQ0$cxgu`l!L*MdgoMKZp&81G`h3as?nUImQg zo!vn`utQukHI}iHvqjt0v*L;|tl&Ii`qqO{L+4|S0%O8|l3g@5x~vEVRc$4LZIb6H zcVS%T;rjO7-_Ozr-NE$Vm{Y{`x#7p;tMjVF zav4Y6%VYWJA8<6Z)N;bvIvK;7?fdEB(rzhI*z194Mu75`6)ZMV&@Sypn(D~Cz&WKK z0@x1781N=DQ#rD<9N*;zWK{GSOY|?k*wa7FU+NPxbjc)=(WCC@?!7nAwd;I=1-Fw< zyE#%QSt;H_$Ft3Z^MNN(BAMwk&&CI21UwNg5rKwp2ZdF)lJ`y4VLo~X^%d2{y-bcN zd`PuGu@Eh-5op;#wt9fhgWkm($ri{IV}=ibCb)$P_X0ITpZ}X{Tb7e9XQm~-{t`@! z*%W5wJ1U^kTxiFhg0ON}HdpG+I@4~{l^EE5O6yr~&Zk%YB{~XI^)R)(6H$#sUCNoZ zCQDxTb#X4<=*XYTH5Dk1KmWJ{98{pm=8iIZERg*|eC$&w>cUs;a6Ej!xadJQtmg<$ ze%1jp8?GMW5v2W5{Mk!`uoJ9|<^Cx3mO)I{k@|U_z!+%YF|r98`i4tgNbLcaB3bT3 z-Sr;E>cq|80@o-Jp^j+{3Y~F6S8deQ+Q!JKY4zeYoa$ZA^`j%Z7eIKZvx3Wy9g{-( zG9QoW4o~Fcmn}Ydo&XBg`Z%9fc0{hdj4&knu_362P5ifXs%)Y$5?j<8yzj<`!$xU* z;*qCs5On-dpw__G+(W;|X@Qq64ASqGp61{3vd8v*D$#5ZH$lvr9|r zR>m0PL?Lqq&e5mmxSo3q#uSdl?j=7MY=M_3N!|clJ^EXES6dW=K@wrFpa-%Q&aG-H zZmppmTWih&F3-%S>Bm{mujCffp?So_4r9l)d$Bz`CtNsa>Ln%ja{NYR`oDMJjAi{< z5D1w2AOy!!qR??&Mo#6)fZ*2#C45MYGRw#x1-vG5`-MmO!fcN3Bj^obc1%%7(TH+a z@D4vHS0DK`Q3s41dsGrzdkj;AtoazKoxAW1bLauvXwOCK14ko)*k`^re3V#m)(VBg zA3lK-ThqzC8C7jFH#2Rx!2C61Ratb7?BjfRaooz$$c{+lq*q!G2%HbWGX=o3q zD-;%@Nj@lK&Toh`u$*7B%S@wP4D}onn7pattQmPk84xKtvkEa+`-5~C*U4uQ6Qui; z7*!{J`(QeeB%SxPOh1A|fTTQ=K|(!1u@ELO2(7$jfcPAEYA7*??~Klu@r74|(*WUX zl5vcps&Lsc&8a1P;6zKFF^%9X0U%86fr`^tGEreB%wf6U1}hYa<@YYUr>$}8+o#ED z&BpujngyDs?|UIMN?ij`TbF;Za#Vbwvchx#-DzD@V}^s-%cx}*6wU`dNTW4f0CGr> z;u0qZ4tn;~l*1v43QHyaSNK74St|9UelFIq7x?1m>p`o)nf6Hu7!ORZje-?>>C&v^ z_V*jgJC9s5su}ll%HED-t<%BZt|}Z+CCxK#tD(=pxgYM6qBADP7* z2xvC(F9S%C;42mP*{-RA12k(ZgZvM<-Oi=l-q^TBtB=NzTtS~|Mo-l=^H-iR1)$)7 z&Dra*GdHkQipC&{wGqNg6!7n+I`IZU)I6~yaew$lXt=}aieqS0P#bEJ%E{b0pV;w$+5DBt;p?!X>ZdaI%Zk~T0~brKYOa?2L)sGF@Yx_4rEPSdY!%`7LBrw0Un9r zy*h3};tSOfDgkb>#|XkCq>N-@L}U@$-(sC;E^4U+>;KIt6C#DThzJ)OJcTgTi4Cr& zDQ}l5=L46ISDAEJ)tUhGmpt9oRZZ>)0^^qo_d^|+V`(5qwV9!~WZ}DqH>Q?HD_@(r zb(uks{Nxv1J@O#BHCzGCoNNZ~rkk=(tys*zz&=~9>NE(VbzE#|le_)8JXQ{Tuc=d8 zsOF%gGC@eP*-H-S+;m|%*1_*i@=uKf3QJX{g%eMsk|t2GHolW)I1@zYwCStjeoh%;8vVje`;!e?qP=SD~IYPrW zss{@bQG#qpK!h!M;Dd{B0rX^YjtBYPY>G`x0&2t5PO4@@~$^JRmLwU;X_J_FP8LPeIHVCo4I zeRL;|zK9SHhS$r$G4KZ-9G*2i&7BO_$;&p}i^->RRXB zWNK|gM;L~=Q1@Pt*+O5L(~`lc;4sh=92=HOFd;~)-CyBd5BI(m`aIkHQPv=7ntDS|MO zz=a0Md#}%8>gw@Lb(bM#B#=13(dKkg`z_key|M|v+bnEf|JSJsc2YQqYqzE+x#jS! zYA+O^kfbpgis>xQ2!C6ITCOR}XNyT|Av_x>b1c{5Gi4QXT1lM}FoX`ZM;DZNE| zjB|Z$Ks8e#8Cw*3X3COuP_wimiw*zUQuxrsgE7gzi!9_>Tot=!KHRrihfO)h94J5i z2}vn{jTVTjgSYT?`*yC$`!2VriQ6s~n4GTzJ05J0uZyfJ3=5WMnfTE=!X7TK))P`S zL*myqqn>3<9yt$s`?~g_VlwEBxT5&{Pze4M=lsdnqZYt1}2^X((p_R}7;9)*`1uqKN7h$qZjd4m1H##4=X< zIFC>|l$J0;b25_KC8OsoD3ZReArU<$O@tYdyfRH>DEx*{_8&NQ`hWaM_((@mp@cp;HFBK$TOA9J*O^+MDj)dFbDjso>{5wR;kJY| zn27D7(B-7`()F;}Dg=&*#7Mec-h)}{lQ(lFy6QI(T^KAzSYV5lxE43CU9gJMoXgZw}4KaBnpoi%M}h?{XTm$QJX98#?p+aPGvO<~~>D!Bl- z1cVGr*cT~16}kQ~!R9*qYAm)8^iU@sFA;YPNqQpG5A>fYXXTiAJmYA8AU|*$+uImR&k|>-CJo zF^QL#Gw4-jS!;J%`&e=)hH5Z<=$xYttsC?55BgUVf5g3;A)39(GagVLBMF@Iy1jeN zqpAcmHqmdSmHX^t?F6(0P_zrQ$)MG3LSONbL=m)R6NaYPvTAJ52W;aphJTv$eSvj& zfgY2Gv*3f)^nc2SRxHyIRQS0Ls9wuW=FQTs#AMo;#%_?2PtW5=HKk(F*THUm!|25O z`RRae)|v)ivF-AitqC*Rr-6B|+1Q{+&?EBCo4x%&$%-31pioRjXmCbrx>a>{PUUtQ z+I)%G+AnQCO?cYP{w?(V&qX*b>~{`sUu}oqcZbYBVRE}SnQps-w8l39YVIwhBESiJ zJG7qJ(s44a&i)gnO*Lsu%4gC;rM*hi{ht;fa-o-GIw@do%dp?`x;@Ob&A#L4cz!}J z!7#JnWEy`D>j;f+;JJF%kziu7HjN} zy{A@+mM^HA+J> z<=yCniHwe-rm+E1831?S-MIrHz{{0vHTopHkeg5nvqNW9Ld;l#c>1}Xg+(TA%fUgQ z44lj;2utDT<0~5GT`vc*4sY7eo>p>Txii#_kE;y3o5QrqZU^e(>UB3t0!J{{JBz7H zm|ceWab)+IeH3$|@tB^Scg5{rr}G=)o;ALGf+aH!7l%{S{)WXYOliMszNj;NA`jW& zaU=jYoF(pdwzT~lpNLiUxvB7JcBmfB8Q=$G7V#RbH?~fq^1DO6r{l!7ZvKJt94Q0( zy|sTt)r|rl!`AtVkEZ1Dii@9XlZP|BL=psO%(HEdTx;PDlje zC0%%rxB7A54j+F`tocbDdaewSo?l*?^g5z z;hN9%>&tzZOTfC^Nn^#$O@X`9LHyL`0vobQSCgCB5W8FR!fH0JJiB&ma}Aw?seg>LL?We?nwYqsb(O6$e;v`Zpzqqr1mu*bx)@P}z}nzUs}) z%WJ)KtxczG+|QL!sXvzgY;5zAyJYA!zmP9lX?E(Cu`)Lbdvn`7v?*1g>W>cwv_?l6 zMVQU#aIrnC@LpNmYX9D}*6+}EctjerNUlGyqQ3w0CVGZE$XJ;`qPeP zKIpfx`fq3oT;AMB8;rVCiyMDY-2+C&&JjBeb65r)u~k*82tlylH%faAtMx3xW%}|_Hm!LXwzI67KLGb**5ZFBK!bn>`u%e@ODseqsqBtn zTzY4$K@yHBT>@D6k|WDHVEo~E;5kU-Q=6$tv#QU`u{!bj1j!8`Psex@td*h;Nkn42 zIluYnk1gIyFDnmv<-~^5_VUVx4$-X_mO9cO_mYC@7|CE8h3Xb z(LCJ*Tx+H_eg(R*lga2z}~3e6!sV>T$wx-Ci6(yUj-pr!=-+zFLig;cZ@1 zg|(OYXzFO475_1Gja4W`!gcC9{iolVE(h%zU#o~#7(;CWW+E8}qe-2`2tR5Teo(coh;+YVwCSZ0)6$!gF%pK-imVpcvp;3d-=DjydeYdRIzK;@V`a~8fT zMJo(%-^QLyUSrFaSI>O4F26^E9L$cfmfu29ZNRL}gn+ntyR| zqGpU3u~;(&p%*V2@h1GM0L|$)+NFciG3~a}I9DYAG!BNP}~k%va)W z=b#}et7(5q&=`0&i_i50zcl?##3R2a;$y=dGwRAn^bC4?<+?9TwBdAW+H@bN#67yH znrK?DtIt?0#|8vLJ-Q9FS1sdri2hpsi2pcPdhi6mqX9rU8UAt+Cs!U~&>?00Dg zM2NFUc$JE>YJ-9ZWefzaQ0X0k6j4dx?c6((P%bl}%eh&RBz`r$tUL?j^CP(-nIgk| zjU6FShF_^+GfShPGC(3Lp49myrleF>{;V}7k?(G8BET_SZ{6{1s$kDWJh=)-B^6nq zy%ymp*=qM?po9vzvf+Zph3p$X@lDW4=-!>s1kS!xLy}Wy@8ScQRqu zDb)j2?xB&TxyaQNTmQKT7$M?Pq!fcWbu=lL0_$SdrrhVUEtG@xlfqoLjVDi z|09Il#Wrj@4LwSZ0b)?wzc)c#M!P;L#RSV}zIBMr1JDz0yX=*5^?f4)*Byg_CA2K4tZ|x~TKD2hp3w-D0(ALz7(LmG^0Q&9dR-^~C z3${S?CzgSISS}~colxc;O&PLGJ;Y^ zw%B>fV#H=Tc}|L&J1(L=67Lxy>YY}7Fw^vZdVNrYb^+2YWdWQjn1zvYn=xd7hyvB(RKZp5@pR$A5MmHAYEkww`hBu+-;~T*`UV4XO{RDdh|Gj!?-M39MPW>%dhNglvZi0wy zb)p!6VEIp9!cDK#cHIdIM29jT$PGm%31uOm3{$j75KufzR%tj#g z=issHhKeXF>vbF~&qaofMY9k=<=$@7~< zc9C-?m)teb8W=e?8Rx;)nxk9VA|^j^yYRYDZpCe-9LZ=rs^ig>~CB0cTvs{m+5$Uk!M`2UklyWo=-P83^@kUyyHa55R!XHPsur)?Sywk44kY*RIgo1++qvVo2ZdX?ynXenPTF7zvpP>%y z3QOx{)!!M0^Ehr@>X?h0rU3-0pOnl8d(?r-yR1+}dZlNg0PM57Ell2JSf0TyO)7K< zL0z>b#)l^+Pl!lN%WonRPy`2oW*7k+e<+O@`6kHg-|+~Em7?uw?wx+2Dm{~9(3?r*Dtvcl3C_bc3e z)k^nmA-J0cuTEMgwe4tGMaJ^EU|_PDzV zn_2q;vjG{c_ijGqhs3QdLX%%-ynu*({mL6&DOu%IqG#3gfxsVx!ICbZ@8sj28yU`Z z3%SN5bf=dpX2V0bC05&XFs(sMIZt2&dSq@ha=#fLN~%7U5v(_lbTPPK1Aianz(<#< zlrhK%)z3Jr$W_dckT(ilH86X@0F#^){1Hvxxt~~(9gAZhyj}*BWwt& z1EVH6dq{*5e>3$u%1D{a_IvERGfq#Y?|qHN4~66$gy^X3!QzB?I22?e(7)5G^AeZMQ`33B{t++%L=rKlmA7PU~8*yokQ03<->x0Om6z3z=a`)Jt|C(ke;XE%jopGtS zO&+JNGTRqu3~TMcA>-!|olEO_Qb?Uad!_CQf?Y+jBkSc+;pNG_GN{(l|b%ZE@4l|G7?isr8mRpc$_kD363@I&*ZGh zZ5A#=Dh+xT5%Zj&Wwp;ODqOWZvI0H?{;RKagDw`#m4GZv=oc2iDhV3d_h{&QIP#P> zO7MfTA3!raEoEYX#O}h^H-*6c5dXAbPqx2rO}r1G<5j0k6LjnGAN;>^Opv&JVteD? z=$Le{qT^Z~8hz}a|54~N14s`R=`h*c2O!R4f4CCvLeHVVSM-?A?2~HD@vWFuHb^x+Pb__jfEs<(B*$_(J?$c<78xH1U@`8=!frjXe4j57WqeyU~Vd0s+ z3SodKbjeX>`XP?g`9myY9n#+1yA=jMw(Rn4{il~7w``GNO+C-)s#U&DZtHKVkGEmY z8Jg!u;`o$vqu>lb$8@Wd9K$T^2xJ}PfgJiAK0<)Kk{%$%Vl0~sm#y=lRHFc$d*dQr z9?+b|`)4zt97mAL-9WO7oFA5w#Tjsij0;4l?7-~o%k*Xvt@e9Pda|q1(x!M4$75!u z(e_z{QHv<4_IpSV=&Mh+E!2J*VGX*r-dv+IPDxAs4w@`uB9hI2Q~3~otnObuU(-9D zGLYOlcn(##jd9+__>pvnp?Y7ux0uU(Co7n@Ic8@Jj56bih>@Fd)x!I$_ zX^uh9VB~K@qwNV)ZS=i@7Xz(_Ow6+dd*&g(sF7b9?x^rm#{+BLVx!0m*L{4tYhWBX z(#7-siO!rgoCht=$q+C(*1%Cv%vrMQrbK0(lA(sfkNxmv9e`Z-1$yTNYr5HlKx&MR z+U>Aqwt744;JH$7mz0-dKtpJd@34D$)#{fF3WObL38`eD?jSDIr(iqt+|3@lIL{M} zIlF!SmosKv&$|x8u0$OFgEc3F0P-2RYxQ8uqoa?mL&`YsIruQlpS@k!O`mU5h;cNN zLHM6j2z}@%6i?hJ6tT=t?(#lFLxJIvZ4RluQ~Sl_AQ!2i8fIMhW2{jT065 zW@U0P?K3eom+OvLP`Ia3?=kbrHOOHmG;EH*K}8q<;1qzwCBJvjJ&gT@tmH_}pUdga z>0Izu#K)Li{|$S_Me+g20|RCT@A%$t!rpX-WWe}|Bd&8kXJ}G1ORO=V(w`OdEaK{_ zKYYp_7E%hxSevsM%UA1vT!5jk$8G>=T>JWXZF()@X73hW!(EJ1Y_N_3`;!8G3Oq#H zRXjM6k-mR&tygS;0dh}nX-z$YqgMXp<0U#W>x8KHoyO*mK8eCwR70R^vL1<~e+K{U zsP^NS@iNVi{>lT1Ffl>;yXuP^0)K*y&>MH62mD?N#pMLr2xuKAZ56XePlG25}uPe~# zQE+GQV7Y=bft0L zUrRtNK>0%J<%Tcv(m2u6STC<2>K;aVmMF6qasqE$%N=>%$HgtAlUIe9^}~}gLA7a+ zx~X`EeFH zf;%yChU{forkdeB*#{bG(nq2umCC8Umcr`~P(h=T5{qHlZT7o+zaU5-wgVYzFwtbR zUxYBTBpk_VNNu0Ex?IqHec10~M(8e^FFZ#}fk+?Mn3LY1uB_3k7?i9~)g5IY=}3Y> z!lSl7s?CQ8kL(Sc7(z=?Lpt`LtHt*t=tLr26htoAy;aud79bPa!*A2S9rA4QvZ{AW zYsVMojF5GhlsT-z2WqJ8E$|fFb$D(5{0VAEfK^ISi*h?>tPia?%E=jM_5-$jNprX| zN{PH8F^$_Dvgrs3E(5nZT~GTDM{pJK<9QJQNm&5ms~Eg? z13r)&wPdrShWlZQ0l4(w?`F4dfjp3jOY@lE1>n_e9M4w(YkCG7fZY-c4iG&Y4T+1A zZeTyBrRaC|MES-obmrjFmp)U z4A>J1JaIRxJ+EE^MCErvNyaCl&(ZMS{oK!2FD~bClag(8w^&Y%PCzAX_G?Q^4v?<( zkNN*)HH&rohRAJ^0At1M5)NOF@o$Q^D$QD!mf;_+#F6+jA_!>&H$>raMLT_db(2?c zJJuBo6532Ge`)bL?{NF2v#IN%I{b7PI`!viO8?Hymf@ZAy5kqYO^KTgEmEyID!dn{ z-MN*}qzVkxSq<(g;Yzh9*s}k8eaxh_I4NipgoDvDqb>7<+e*#c@x6P6f1(nHEyc^9 zaq-I-Q&YF}`#Mt_d=oYt?aB=TNL3iC@8f|BKJyoply&0~r_BRcW~8VG>=T~Z0d!h@ zpvK+M*tAjFYPY+rD!pZzK^UJ!z#Xp^O~0)c85$)LBO9?) z)Ows9vl&l-&9OsWLDB z-(DtkP{IkfTiv|y%+`I~FV-EpeP!Xcsn^Nw!qRizga#dU)we|C}`H&Cn_*6sW0?)mCGXLSS#B#-De ziG9}Hk2nnyKFoQAdH>nj=ol9vU0Bx{$=^+9S}~6i9wQ>`W^AT&Go(u}R!6dL2!^Fp z?8^sN!c$}c2B#2UMibMHOEX92NSTh_C4#Dm$U)LB zL4WDr>MZb{nt=a>#fu(eU*PvCLL-bcv&n5(jlqNG8~6jw*qmdyV+u?Nb<(51l{xCtP{wWOFt*a`{#6X%Dn+?J>w@FAhaK40y7;2hbcJ* ziP0_(`pA5n6z~XwF;eWJHARNND-ib7#%*KM=Zl@{qWYj9Q^opDeZ|E0>u@u6s3YU{ ziwZx{MZ(ulDHr3Wg?M7-jS95G*8N}|&VAqNcR6ekyqkFS|J&O8Tj4mS-Z9p`F!ejE{kIoK%LXVAI4GGbM@jYK7i`&M~%O@wPQT;N`g5{pnS z*0@(>!whP(p~CXT#-92zv8YHCcf1U#hLRXxQ2NduU}|F__LCnQw3<5=>ePY5=Mz+^ zBua&6Qu;Dj{h0qCcbz(MZzHN7be%5!^Zk1jasvQipky+tfa4qRm&nXkZgLkL5>a-i zEpiFa{2CN!9475_T0u_?1O^(g%bYtXx%MFAB^#|bnZC6(jYt*6oiST25x}Jf8bOi z2dJ}}axuC{4ssc~8b^du>n46vM8{+PKa{-%P?qc4?kyrAjYvzEbV@e}2-2X^-Q9vT z(%m7Yv~+iOmvo7Ach`44xYqxD_ulWG+20J~tZ`gC&mGr&U*~xq$MKVL&kgxlsIBJp z-Oz9qx`AFEp+8-j)v(vW8Fy|V))1YTPS#^`hzS#+t=z9Kz*hfz8 zufR@>CY$Qm0`U_&aHf_4o)O+zo7gU|)U#x>Tv2=iNZ7c!BIhfE;{KsGySlQvuC_lV zdfN6g>tE17h=UU0!Xk$^)HCXc+k7g7J3!Z`yRO+45GB7$_c^vsNx!nQLs?e5hrOcv(%|bPY z->GMIAwd{X-UtntrZy&(FVs znSQ2w6_{!#wWdDEyYQL3UpC%d=v%7mTk~iaut*D+qn8@@zUfW=!onxV^Nwzlg_660 zS9`hzabBpIr3&U*lMtbOl-ftnumKsF5$_Jbu%t+{v@>q)sI33Q`Z=etHWf4+aWBFf#3LLAU&*z8@X^pEMs~ThE*?4 zT`5+I)o1Q7Lx)gXKi}}30}O6McQ|*-;_5>`@v6GWf`=gBCGoRFSt*0ZWY6;;E@Zp` zNymj+KQ}ZB#?un0JrEzh)KwU2@COM2p9w`!)UB=`H#RLkStFf2$+?R~rJ-az`u9Eu zp4ScP_htSLL*~dR~RC#@>PV{9cm%iRUZh~rKWX{G^gF6!@@1BFSNIX2M)N5 zCONm_%zv@C9xd4}v0CfZFcf=1!5#7(Lua1+w|YB2e<|$dM3%Bb29+&1brQ@P+>A1$ zI|09xH}GI->V0|^CbvIA8}`EXj#h>o3BxjN3+Oc$vB}b-R4eAaMJ{j(|1h!@7%v#^ z-f{J6uP?TgBJiCvn$Dw8Z+AVD z9~TcD(IOCu!Zjr)pcP$S;;QUoudi=eD_H|$&U5TpYlm#~1ijlW{d-Cd!30#RzwCFt z#I&Y9nRjY)FtZ&EP`*-0DmZ{HrKVDYh#d`C%{~a+*k(QK186t|JolzlPj$`;l=*A| zWG8t2p}C*$jg%Ve>MzL{uC6G*Kn4lMuds?E`O^zon*ul((kPl6y~${Y2?0w3a%8O| z6a+t3N9Kx(hG-{H*wAaYy1&e9`)T>sY`9)vaDyOyt3Ou}JH^6;qy!Q6B7)aXXe6WH zUU*@qh)Yd&eTz!CfM@dw?R}y63RN7<7G4}b1u1m|6w%~a-?Vt^AEoKyI* z6!qRY2&NS9MV*dTkU;V@BQZhASq@JoYnx~LKk*zgqo*s46?)1Z_Y;?cC-Y1E4)}NA z`{o?E_QbyK(HYZezUi(Usw1iX9yX6Tdy%r`+#Stub%4a)j6=G^6K9GKZcWI~Yz_$O z1wd4IwX5?Oqp0HK&qR>l!vWl=(l7PHpNC`a-?>rEN2H{AVGVg_{Z2{^P{})SeIPtI z3jagIQ~~^I0OcB%c-jIk%{Ao6xyR1Z6&sw@cLt=`|9%Ge(AV19}l(!5P>LS;l=mddjSuDn-#DEnWI5_=kLb_oCxX{K*A1F{|!4N zjqgW;e98>~5;tEN8eDz6u~O%sbGYLCZ-5h)Lyd=khBFU}&{eqqoGaqHwKAwO3Yy)U#rKn+nEpI zEq9=RB_0YeQyRmvcY=Hpxh?HviQ83oAXL}N3H`l-)1-DVS@-raxm0Ph3$Bj4|h zp+)~B%RxW~kYrb$$l;#8jOt`I;T2!VuC*yqJVku^0)Gs%jG65*pK!x4{69#F)Z)c^7%ZbXO- zZ^RaN=EoG%{I`9O@_8B88%btf-7g%!{e1kI5y1>K<(0aEZ5jp|hd?efN9cH23YeFs zn9PhgE5XV1MS%{Z#SSV}@F~XsK^zP6B7*c+HjJV!tUd2nI>deZAF;Dh6L6?VUZXp) z5R>@v_h@i7U#;~w77dkK=a$m-o)~T$&J zswGLX4u0C!Qh(ykjj-cTTM@gbpDu|E@%RfMHRSoCNa-MUGa@{8XCL{{RkN9C2YhGQVt0$wqlMO-G=bGLl9qn`<2NvT-X0d9s)!}8G?xbTu1UjqqK(? z%ezh>*&DzKLqyu7#KH3qxUpMko3EA?ItTA^V8gp zGBQhkS^tT3da_Dd1*e=e+X(QU)is1ztaHo&tZNE~bZ3){CT5=DLEDetCK0ibAG&f& z0hy!^(lTjVrhCfQah*?7o~K1K6(m(w;2Pa2WUud#p4>N6fFfmi2cgowe}(kONuily zVX++VjY)YbOs^cVtS{+zL?&-t!5N`P^`Xd6 z{-ri^o{+h#;Hya;{MGeU5o9GmV?A2>Uy(efiscX9YNK~ec{2!{s=y-@T~p%}6E)5r zj!>+A6G>IW5?Zc&E>9_r!{h?{7EIue%(MxtU%O7>XjZ0=k8)Gp1k%W=()MI8Aj9HR zU(zW0Y-wc-vkXUeuU54?jj&^3o#K#&XRt^!ynY`h;IL~)F!l2E`QSkk--S`Rp0bUH z^ZvwUdB`c6h&b=&#k}qh;;DukT<|XNAXZ@6s#?)ZkgYp(>h&P3u%WllN-L{Y1)~*J z_ArSwxvAV>1@fG}&+e^{Ha5l`D=84m)T3S}eDZHEV1U6**&(d4aqJTH9QDnR;;&q0 zd~?x0*PisZzt=5GeijJ9VRVh{I_ejYKGwn%KlAdU#Jhv@PoBNhmoH&awtB;(8QYJd z+Q+Cq!FUY+N&yfoY{QUvEN4B13vF1k#9&v3t^RYvcB4R?e?ijIIOTUE|nej?8gY?2^Qwa zS}b_G1MK|?iRP{^Hoqd4+b#D6U_eVtL}DYhsVdoi6?6)HT1Cf^=bpaCMoE~@?oDWA zs~!yB8^KPX+Y+3KDiC%uxJ`l`i1b7vS62rO#t0#a=<^`f;2sDIj-u-ZSk%C{QEtQo z3_O{!#uQj&gzZgTj;}dyrl_0x!$0wc&8enz^Cy@zH)dmmM>0wAPteG;XW_ry*Yv|f zvNcuNXP*nVgD5LSk1tR4brpUpFuZUcyTgqqioGh6?21-}C*>=z&!P1ZrBp0TB#HL6 zvZqn|L;~Zqy`TqW)|}Ok8lpQN{QCL@<`)K#YJlUbT1W%cocH2fz2z_uErd+LYr@As z#V4$)Wwz7HlbEl41Rd3dA9FI{3*kTfy~oBi=zzmA0a?=MW$*3+7qMkh_}MCRqVrkd zuS0Qo{dG=?>LdaI!4w(BP*pO26a~fN4M(;mV`gDuGN?#imKU@BwK?6YD5aZEdZ|E6 zig7q7!Ls&8y)(+dQ^R^sfoq(uYa!9F_PVe=I1a~Yh1pwA08ounia^{8FjM^g1E$sh zXAPMF_*UI#ktqgTCt4vaPF>hDb!uG=Qo?7;x~!8HfZCMHS_dKVvxg%sdQYc#ysy{^ zsX6B^f|C0(%}AG0C;=*Fnj>jA5yp4~URpAZDU}+rw|$U&cyG@EX!=6|h$P}x8|Y+5 z@6TIb!`t_il^7j7!sV6bWMqmFo*a; zIZo|U>_8|~$WBErn)xTaj=zLilYl54tGC!{QNh^;_QC7|jEMQUgZX2zc`JE{tP8)* zO442h;1WOQKx8Z?wnNyf(Pn*q`l)>YIejNL34u8V3%tTvB3dEiihsMp$Y8?wGI5b< zvEG15Pr3pX-z@=kNtL4j&8#95J3CZ94BnlHVt zSP9RNl3IA1BgI91lplH3L7_rAZBlek7o^4NLPN+=qbK}Ob2%^^Xxa1X-XC@VJ*Hv! zCn%fo0>!nFsO3>88utBX4p;J(7w|M{7+d3?83q?waU7l^MD+@Fc8#Fn>yehH0iDv3 z#r*6v^NXfI_%Q-r&czK&X=sRvyxGrQ2g1`M>>*_-QCxWXt#D17-M{e7?ymw|$3Mh~ z1TsK|&}CS`YLU_Z)7FA0;6gP$^U&+F+WUG%Cq0r#)fSm$=$Vo}L+7rgTJnC}1_VG@ z%p#jpQ#$|?ssPr<;g;Jl=yBYMiLM)1K?5r5`nz{RDQ2e>wORY)NH7H&?yva~hLCf( z8GW4NSWgDO^LOrj9%b>@Gc)vo{rTwcSxoTubF-%D=TmL7C>ZLeK~5WX6(hthCw^yQ z&8VB=D19D{07k1~va7aug$XBXItyR&$inMemwSFgNe69>g%XOu=9{|+KaVcX3sI2B zKS9A^Kb+6VwV2Pax-U}4HG9D&34p!{Pj5RsPd9Uk$rgqH$Mpj58{`&5+3LyUfgT$# zF(H3%LI}$g+5gC}Tb_pA5=*N12x+8t%s|rP(K^u0YtC_w$E;oA(t~Xa70*l@jH;dnVNAU>e9}` z$w3m)gYx41DH#JE_^>S&9sIHf&Z!4WoAup}#u2^KA^BH^y+e$E9`S3vp~7B|g!B3L zV#CIAx!E6!cdI7=I0cU3RAinIkLzz&VSVq4>@yZ(i8EJ8sfO!0%rtCA3d?B zdwB2_KGg+rF)iNy!btUEGb*G=u@oWxbs|*dVUUl%DBuR21<8dmQz%m|bwBl8U&%VM zN`%>qY-qUIWc320wUn#X6S&$!dV2bi6Z^B*QjV#lNyRhSJKdx!n*K85W0LDsYWu=d zhw|j$TQ{k1bG3!$DS6!89`RWjtSgD+Ty0rexq|FcfWQ@tUx)~F<&`fTI|zhXJp*a^ z=#jz38lXc{y~3gopi^^GG>?Ur)yK@$f74@mwBN2C9{u>`+=Nbe5|8&o z9p4SRdO&f!kXPLp-obLDaU;o__K7~il*%2n>cjTo3Qh=rqxhX z?=GwNJ-ePhb_4l|J1@NkwfH)JF2DU2GtL1ScYB{gF)(P1MiQl7UjEXetmkLCt=lGY zp!ivTtp*NGmgPO$EG_5QMc}7{UV-MrcU_>)70EIF-%?hGcq7qZu7;cOv_3gs>?v}` zgrjF}5Z{?CT@m&04jveVlc(%EJ=9F95a2?SlEJfhj_!PWOPbbeqa&vu!?Ho=JP&s| z=rRhAe){@Fp6G7&K_t4?USd|Cz4eO*49!PNQ&7Vc#yE0Ox6aOaQRPiDr)t8? ze!|#Ea~`4{D`b^O2>5{}0qv4y&Gld`#zBT_2yr7cNZK?)jpq6mXqQSxP-1;>z*9)Z z!}^c=5xf{tQKVBgq+YIm40{9aQ$EzlsCQ0tyu&H>jaeS_J4))o?2uF@+!j}-QaXDV z?ZdKKxzEW9FK$jnWosPPb?rLrGXfe&>Wzy$?2f`_0&hIU45v$|ZCXVmztL9AH=G_d zzwJ+pvGroq?B^Dc{rONNxI@5zQEo%NZu!}&HC-2r80k?fX^LfX*H^x>LL(`w_C0|`I5t!o<~<9;<+!+ z2lvy>tqDbaRxO17c{yVyE4|W(1MI$BXZl%0|7!j;XFu%pc8}EUTi$GCFAFG<67)F> zgJJD7Un_Hkp#)CVRY|p#>y5b;g%cnz04)^QYJV}AI|CN^2*$sNAO=AB(Xi@c?ajph z2>=zpeI*Dzy(ZGWah(fsOjXMw9~w`>|1pn{5r;&z{N@BxWXqvEfRc{zlKj5?hv3i^ zKw$Rof?&c!{kH$OAy+vbYF2Y?CB7s54_r9rbq^MWXtePBPlE z8VUo-gV%~!{3~v>5B<6ptTmyCvJx6ldnaE-HN$ht9ka;u#FnPw6nURs|U~CUA>zB5| z!^7pfo}9H#HTpOEsKI?|z)`iudlHrULW57*%<>fgq1hnERuFb!J1aRbOn!GNKRJFIo^wNZsOrzSBrk9s;O$q0V* zaJh~`tuooY?%HRZHv6*_*%QZxZ|U4*QQ@i z_HsNrjI-qX<1*9OMPC;&{^r-Hy36lCi?otM5=6O3WUG?E>~CC84p3o(`ygz4_Aj*`%I?c2VE)n^pN1_UWov6sHk zFN@WWVw&d)S&JFnws7cQBpr>)_?{lYkXZX)!|j|s{dsGVBswxU7y~=9cP$XB9@Xxw zW~+&Kk<7~q8+C3$?ROC-ep&nGW2Tip>ob1Xu*z7^*2=Qdjv_UR8z=MIWG*uzE67_s z;v76!O)MNfyXAo3z`N8>)*5G^w(%l%gWvL0)*?C!zg5PL-sK{J9>loXfI3Ad*{Dwg zacP?VdoJc2qTn}NzT%TBSR316ralAQp5E;Q@AnsRXzxxXy+|BIEew4Pblfbc8Ku*V z>uhiCciXj6?*Ev#BK(l^$yckN`cu=LDf;`DW)H6(TUm*)?DlcQad?}XOZxejblrlS zJny*c{MadEfrfzg%Be-Y!otQyLFhHsY=B;Q%BID2LgGX^dqJ}cM?Lvr*AUUTKT*{$ z@ClaR3kqy8@rG<}Em03gnp{erK0dUhJd9Ea^4}XgW1)V-BTH}W|B1D7+GU?A5_o28ZN%#8SL>Bg167!LuC%?c*icB% z!O%JqIBVb)^?)LI==I)R6fte$JJc#??9{z%iEwx%>eKnrIV9RKgD^{Vms;zypEb3t zl(0jZu`Cup#cxmGc%s?yAWy_uJrEZu;Hu>d`OX^=mJ-}#TT#l}ulyZft&wHmW+caY ztq0zgY&KhN$_t4_@cTg6b5P*_EPeF#ss4wb(qi+)0QcjENUZ>pPomSa+B`aFaiX{H zMeWC;5ivM2EeCmNUpTxM8fxl$_6c5-Kxg{JQ36q3Zqxb;dx6-qLD5$ih$%*#lU7tx zE%GR)rae|xDxd6z0(n=4sU7G?H=BP|v>f*KSLbwG;sgO3c4L!uAK*}Zb}4?Ra!&Zb zJCzW%;d2{EvmmGTYtf5m?!#i)yTv5EdXCrFG9KrLXCKg{-Dpt20i$J#f7Z_2kgeVU zM?vLjFE&P@qrLo>l){Y;-Qxg1583^lVrD|up|pHW0dM++*3^}^Hqm6f^Xb=Pqr&wR zsH<-INP&m&vVAjr^(C#WbfU+^$h&44_Ryi%h+_c-ryND~z1jW}J+5`5dl%VFzU$p; z*Fs{PNc5l24Hn(!P}3C56ZSGb(~7GqeR}?1CLBi?ON@U=%K)aGx}cwP*qUq==Qs3< z;HYo;F6fWTMT8ttWQt;0=l>O~O!RC2vqAN)&sRBtPW-g5#rdYjevthcmV9Up=Sk%P z?}k=2H7(19EX^beVlun(w%E({VW)D^vh2vsdC~`}@(l>x{?$`I&>|bWZt5V-kuqG;MoS%seJdfW59{#M9Pj&wf zq!cV2{4lG}mf}NB)v6ttfrZ=KtvBCENu=8vLMG<8zpeJa8~Z=GCv~jy1SsVE z7mHgv+_q)Q%yuu!b3V5WN+RFLpS4yFXCLsWPf-@O5nWl6Fxi7cjLR?aB(bT5a^uxVqfw0xfUh>Lh;dl-hIq>?4hNt=#UNi#KqE z=Rb@E-UZ%#y|^!Yzxb`Ku&=hkR#F%!J5^UEe@xTzuj*EP|d=WfHp3*F`V z-6v8`I7@uxceLr>$Mrec{8{ zdA!GA1qF6k-M>%QoC#3yLy4cVDQlgB;6B1N|IjvTegC#wJjA_cZ4#ApIGnf4)d>on z)L`i-KqofIV``BAABXqt_7*Jt$L2*Rq-*Bl{d=0WJs0Q1^X}2IZtq*Z9%|)Ou94Qu zeo*F`TrE3xPKmpw1r*ZCe`jU!KBa6I&3HFO1{s%bx{V-%@e2io|DvOVgJvEUL`MY6 zs1-k2^S#`I{WSW=18Ao;B5t{8Z~6tine~f%#qjxf*}OHxDKl&?!8rTw6tWq zW#iTFv)_7oTvz#4OWs=lB8kMi;#?i@-+{a7Gp{)9-O=xRz z_qauQ;t`8J_H}9di&O+a#{o7Zm-l_nq=wRW2DY1a9tGFeul=e7BM3ppjb>|PlWa#7 zgyjiFquOW@kSrM1#)&1m{)d~mP@;H9tBMO}$Qv!LjE{P97#T1^+$$3;arT-22%&=e zr~30rO;vH6`TYIS{p4`d(yN157LVzADzyf>;B(MMBM zOXKQJgU0VN95KY%c_)AeQEPiD5eKQGA*ogdefoAnaiP=`UkH}xH7WFf%1fb_tA5_6 z37Hw-BoSHH?#mE$-N|d+1O}+!uM6C6V4}YIu+%<+1wST&cJ=vu*pC6|&$ss7#sK~h zlp$!D?P%}vPd8D1ScJOiU%yTh5BR;tq7M&_fS_WbdC9fB3}_vIRKURx4GY{mjkKgY z&HW#W_yo1y>bstXwei?5n{rm+Ukju@(4W0poZnroC{7&Ak8(78hSY|~%o zOmeU5^81lx`~prjK>H;$wIt`#A%5(Z*ye-<@!i>uEy9hDBKYi!U@JG0r;zvJUQqIRj{X`j5#q1uPlG z%?c$O6Q|}sVd!Vx^-p1?_=%G>EV%0{h|NqYqh(ToP3U5$fy7qi-tU<703^ADt2L+P zeky5A=R<4JMRA^cmy#ZG3cE-KRlL5Tmmj=6BsMc4c_vO)G~=?>TBXJc0ds+;)4zCK z0-w)C+;!L9Tc#J+D%pRJQ(6-zGeoO(go`yR##%K#CDUI}>@D(|Rn>)2uFE4LNy~lSHdIjvF_`9Y%Dh}jwmdl^;t}X#4PM~= zNfXHRh2+~1EJKM3P}7(nA|ZNo4G;dX)KsqOh#22tr|RedNeAAyJJ;y#|8uY3`>*Kg?BeBcc39SJhEtoe;PX+*=_sx%iiin>>T5t~~ zXfxz*|D4yqu~9=Fa$Wr1+L<`auy8{#Jq|9c5zZPJG}XDG2zsJiy+~p@1*^8Kl!4s~ zNB+WJuNIqh5HoIzuFn{Il#3t3Np@i<{#%g>d{zfZ-h~C;NF}ysib_Hsj98{VK{sjqPqW`uEZyHUA=r!GOjmwQ{>JOoez)9r>x=D(}zTS$;^fAhIs63*6VlElNo9tJtTigNqO27 zUsod7oGvoeDt33N^vKQyJ}{g^ux3PzHCy(Ol46_Dy(5rGsXlkY5>h5(px4%tx{+aE zIIn5tv1AO+;d~p|Ci!2|W{}tkCG}tGmlWo|#D?-l5=)*MwX~C2_|~2O9av^$l=-iA zCmXetw}r@W493Ey+Il8IS-Z-T*{B`FXj4(*Z7fBKY9dZEF%^ebL>DCP_D;HTgP+s5 z>X3vc!nCZ+S3Fp0V{15wx*f_dm~Q=-uDe*V-|Pv|K89)d&c|sp2IGGG-2F8Kv3^K+ z$?)WDeB!AfnJ5eV1r_!jkAW5fO^*mwkvqRAB)?=K{ew|T(=?K0blk3h9q&yevJIK3 z%`)u2=Nfo6h(*5#DsfLg%(hYM<}%JV>U=3YVzuyP<}4FlhaORPIp%B6nc!jg7%4!w zVkrEcm`pupmwl7Um!dMRh%5W2c@T-e9fFyf_USGM|Mm9ee&H}}deSQ?>kQnPG;CwF zmXqF?j2x5$4>JcOcvEadkvuzlNNS5q&k?EE$`KM%Twdcj;YD9n5tuu^1l}GhZBWV)KZ=) zVSn7NUj>6wOIM7GlYmDpgf6AD{>90c6$Y4o?u#VI&vB?IBw6ih{s(@jt-r7{ae&Cb z$tz*d537IgM(`(e6d)N|SX@Z)FTghXHq{;9)L-LC`>x+E59k^E<}9!t9h+@$u1)y2 z7f@=%J*OLBxH8u7oBy$MFol`f`ut~%i6S3(gnU*q&PZ&srQ@q=;M7@byVCJ0qx#su z>`<0-CAlx7t@aVTUfOF}S#^43Jq@;q8Y)NFV(klsmL!Xe+r^@43-Skq1AkAT`=OEn za~IUFDRt7~V#RK)(Hd%!UGe^I=V|^fA!+2RJ-#(xrt-lJvz>RC?p~=BAa5%~z!&c% z<-!DqHJegYP<+|Zc{Okyqao=es8UXSTrhjc`$y^p)@<>)@%7OT{f`lBrvvrYmkBRl z=QAu;Q#0@9WvL-pd3|Fe{Qw0&wgA#o?gOU&g0&zgie(c-k8(Wa6TpfbUkB&cU!9SI4G% z+|KFxhJE)=K|2e*;-oy?(>5i<>y_4lx;r1CIydeMiAWIfoK`ZX<55!O{W@geRpMhY zK~J5R|o0=a<=LW2ZHv#EJ7ZIxKuyywO5>?&dBfnZk~Y@SJSMUQAUdPrG1*ZX3(jyeWI21^ngZClU3Gwu$ByQKYxX(C}n0K-Y^Z96o?ruH=f4f*Y#d4GC|GZ&>_YHTDYA0ox-f}R1ijbG!#6K~L z%WCx!+b-~oH+`+1v5F9bWh}esDLf0O5SY4U=bY%>8otNb6YvVi92s+!Iw>xgQbc8q zmg?r<-8k=}Tnh>GLbrSR9a1J89;HP)GMSN9Slu}+eeE#HcInTiy)_FGxc}*V6_?0jEv?)5R#vY)%Y9>Z;d?Z@vMQa2b_Y`R{Xq~BThy_7`=PZS-hW^ZmYm)-MM za}wXxq>As#HgCQko}IH?u7G`Nl8B*#fB)?7<$Biphuk1y=2dx7#kUH1Qr$-ZAp3=H zpn4phrd5%^&x_1lT!*ppMw(Dt*0)-^%Fc{Jc=V4VK*d7xc{)D9`=Sun^&3o)3;9Q1 z+|>&nrOVCA$|JMYM~NSJj&sf?+e_EKlZ_ZYh4Yb&=vz(6ADS;&(CY?yhG`Kqi(one zC*W7xzs4@A|4mmyM4FG6+aX&(K8Op3k^=7K_ow)Sxep%5Oup!92|tHkxjMB!zpmR{ zXg6}N;?_-O)BVu@mh_%bYB>g143(sB;H=!=sUgNhZ&h0n^33{=#`s6T`yzV?Xw_+Z zyp3R(aiR5#mCIe-@zE5$l$pT0@Q03{VcQLb&4Y6YDa>V?SG6~n2dL!yy=iDS`|Zd! zLJH^MuI4=N z1?O+myXGtQKX~B|I;G1$+7WD(GwUvzDIxVxj*ZRM8v-sq29|{`@f$ieI9L)sFP{KH zo3sQPsVn2>iSb2@)$*|=I8pIUyo`+^_=7z&72HPABXARNT0wlqI+>mj!_$y*5>-Eitk(odwseu*_o z^%s4w09Kp9M-pSF zfW^7X0J=!i^_Rv^);hhmMQBS5H)RHGww_;4BSksw5Q)6V8o^V=ly8Ex?WQ zth7JMDU9|GugJJ)36I_KI76|a_f^6DEw)-dn4xT+xm1ar@*=#sJUJkZmxU@ zk0yBT5;osneO>!L{KCIg4Eak2s80FpVJt)dgUEhyOafB|7b1w z^6+m`nz+KE1FNVD>^2rMp}Y0jYZpKs_eMeh>*@M?z`XmaF0d`U5qTWUqCK+_>%+4r zb=D`p5hKKj^~T_BkIK@&QNL!c+-)zlQ$6hns+@0~Gy56dItd|qp+HS<%&R>R97{RLxb z?iMUf|Kn4h7#bql@xdbZF840-`1@BfK`yd!-7W^UKfRm2@3+HN#vlZ5jhZLGN9S9q zSn4R-D!JIc#Idb8DjQ>-!_85h@Ve8R?6Qlb-Oc{3N2@RO0TK~@vgs~P51r?!0{(C?XKc6xEVeZP=b2rm)s18j`;YjXWPp;z#*Ta&9JQRc?ghU{!+x^j<6>?7; zoj;#r+)bzp#y^pU)&i9o5arU6iHEt;aN~i03yT1R9$H29BJN<_2({$fk%M;z$Y}r@ z=20Nv%SQ~`HO5(k3InDg+n)+W{pGgxC?R149u5x-{$zBJJ_XJ^#FAi^Zg?QM^0adGfqU*D8>eGIzlYVI}2DY~n({f*ct>~x= z5nhs7@vr&2y4~(Mfgh^?qxFXg_5TOhZ7-gP%k(9B81B@H;HKvP(40pDUI-6@E&hii z^wqO}8-L=KphZ+u+oB|kUFG5C*mQO9Q!EC&0EM_wSkcbB_r!0&V2U;=I!dxp=5}Y+T5KqRfQ)f z{v-DNJ6k_Zckl_RRxH1?(YQ9)6=u~Cx3e@mYoEGgsf%iEPTW+l#zBuopV|oOUtPkn z4|rU9>4==Tle1C1&4Bci2&N!^PABmO<*n zp6SlhAIu7#>}z@+WW^r!#~im$Jr-^_g7N4OPM@y^^33MYa1>&*G_eeC$wgJt*MIp% z`qQc=ZO40!{qz}JB7}5TeXQT-TByj>|%gNau0?g(bk z00L3Lmc&-Yl#`fGMvZ8@M^tF>)q%45 z==FYY8t_U)r23o|%LZ-o4kF51W{x+OHKRB{dbE!JcYs-or{AbY=mC;&N#c8`keYXd zp_myp;+Oj$Y+=&<59xL-E^_DK+o0Q3&_M8NcypaV7NdrA=v}gU z-b!@(j<*SbSxAz6gEBqud%@)3ox82>MBTGC!!us?UGZ^$`U@)Q=ls9DfFmq#O4D?-v^j(ChzwQCOAmRf{OFk^SxPIYzf6hn2riPl@&pTfrVqe`*as_68<6+-h%l8}J0h(Wt}tHCrhSh^>X!Mn{2<41yI zNHv0c<%Dy=x><4-8;531p?+PIywJMCtT)H7i5v7dBns4t@z}d5<0MVtBdE zJ=_k%3VsTy7H;R8VyX$O5(OB#s@}*6S)|(jm0^UyoL*J@Ifvd%9i$vpd2|2NN1m1dkWp=vu`b=6qOus=>rhBXS z=kU&Vy#wgZ!D!0L4gphW%g5*)U0JtEAQzEXC6oi>-OJGvznnnH!S ziMThcz2K`O8FEh$zpYG~A%bW3~ zBs##Ee}(Q$sk5=!Ltz5;*T*(L9@&DCTW=;!;^$~%^P+8_u&h_sYGE-H$&B=)_Y1)u zQye5>D2V*l7mzm~QFQOZtFywp;C2Q<)A{1$s6GW1FX`?6m?IxcnBB}g7c+$NHTgDJ z6$Kmmar`;6VhTHa2#Vduxm0elDHq#lCUs5Vbjx}5+DMBr2qhg`#j8q}F)X4xmCg-z zoko#BVkIeT@go-8Pu%iOA%x**YTR81F5qREpLANd=Lk3@6XWO9zh5B(4@Dw~GzqdS z{3kB{q~S2PMDIqbS$xVk^7=_26n7-3vwwU}!<>d}V1wciMGz`yHb-yjh)~zmTPE19 zJTaHRPFa2+cb%uhT+DMJI)!Q{ObqS$Kd{(TgL~-2x>vhb!@Ez7QyU4zUS+;V(bYPJ z#r&adV9In7gh@;7u*BX+7@ACH*)W!z zyFmJ~T1dM@Uoi{~PF~0-CI_17`Zp>Fm>h#&>~LGt72{4fLZ|9A8pM(#% zBGRsxJo(gHktu!g>BOIzAQ6V1^=(D2o~{a0oZOqq?)a2IT~mt~>-|PKl^#vFt+U{} zh@P2IeQ&ABB7vB}sG_(Q-_3i?C((*+?n9S}7+$;xdG?fe zm?UGi%ClJ-g>?~2hsn4TLu!-370&#hC)4>QAQPc(nz_rcRP+#^&Z;{a&qg&B-ZkQu z|ICW&+p2g(E7XecjzrtkPN;^iL2g z5ugGMmVODTPlWewhC~<@-agj--eS5x(}&(%C_+dP$YGn&=VQ|7&|Od_FC#Jff_$V3 zDlRBadJB3>2gHZ91oF+xiLMJ#RnDFz?y*doDK!o(1-|9hK%VfG&f}Wv-J{gXia2hQ z#?i1;22>=KO#K4tI2g_{Brhilg;V<;=kw(tUXbY?^jW%NAak@^s&lsU&Af(m%L}+! z$+FMr8qP_EpWcwc`lG8G%wIhblh9(ZnJ;;mvL;%HK+lvV)(D5_x!^Ja=0`Iby58w( z4PoXsN?=b0Gq1JfQ^R~TKHg9P3|dH&BWpc-R;}M;L;ri<@%SUa(=XAGB)^AKG`knZ z*MNzu|3QN z#G36S-0pCo@U9=+CumWnvZvrMI+eza6SNaGtc$H!(VS+`XCe9%0GalKzDmFuI#>FF0 z#(e)&Bh{@4ufzTH3c1gUJ!Kc}Gh|y{df?jzTqSyILaWfSNxlZBlqjeoM z)nlF)A%~WgMe(_*zWDL+Enf3eF0(=%Cl&|kZuVdJv&Z_iXlgX>JX7jpwX&(q6N*kk zZW)pIaATwBTu%L0s5Wv;$*UZd%k}&wd8tx9^TUy_NAmB-4c0!CGpM#P?a7MiwvPUJ zYAZSf5fyg?);fxKIfs-4Cx_Apg8PGh2fOhZ=?V|O|FIh(30V;xrm zpkBKX3v(7zL1JDSUrH>RZ+#t2%%XErTP#x&(I#&Hm8h!BLa*8Cvc4XN*C79lWEhER ziX1;qz}k|`KI01u_Old@N&_VJOpdQp-}_8l*y_JPYVk!SyZiWUCJMFG!Z>w==!{Q_ z`M+6gZH8OS7TN@q~HE4 z={sul4A+oAF(nw*v~2DYkAd0GC!W%67Rqs7D!$UbGlY?9-zZncZ5FakAEU{$v7Gh~ z<`^zt(#KrM>tJZ8JVGYbSotsRpOX#2(+hbKv4|!h^!&Jy0#g15?qre4;J$MzEqET5 ziEC`d?_xZ~Rw5ym;bea~LmM=YY=D*8`JpxM)u)i&>~qW@9=a$ySezBr>3~mfgJW{1 zBSUl+P`}}H|4^M{uIH}9ixG~ouZc6zEI*H+WQm18GFVkC3190|`KCpClJ!2NtB6p; zpjl?tvtL34-Q;F8^<-EW)Yf&0bJbQJk#nsz;&K0FJle$ksz+KKUJr+^-xv~vP;MQB zd&M-o;Fn`qvNLX~cCAgLgS;Y!h~@HB&7yk^o;%`6w4~mJji!Wix=0>f_?cw8c6(VN zQrg(}mxrpc+k{uQPZ8QjXV}pd2wN!Q2LDjx8kcUEwowVkbAD+zyc@VFdY4+SY zI&xSkYeJnAw_JZ(e}g0S(-C1WbthNZGm_Tj>%SGfzO^T-AqWSt7 z9N4x#+deoxc{io#k3msaW*t8Y%UBxF+oDLxOa6~YIa8ms;w&nG{FZH@$y(>LCIj|o z!E8iVsak!ffwIiQ%_qv1wdu#&YS$$$=LeV3JoMD3s=pg9`+U~Sq}GJwUMx>oT`Mu{ zR#4^Uf)&DEP{v*`A10{d#z_MGN}(bmwF@xlSD_9bd?4*Y_&-Y1AtDqZtsYf#Xqw+m zzIKiD6k<8g@@7Jtk!*2G65*~KVElhvon=@X-M98rXiF(tC{Vn(Lvfel?yg0GyE`pf z+}+)SI|PcmTX1)G=cLbH-gDl3ORmghX7E67qgeD8N;Pv zG5?h0G&MV%`+nH;cT!#Q>AAZ?CQ7&r;buKVO69BFR>ChR0KZguF+13T7fQhFnUrO*xG?!zkGXj*zBR+O%Rr1oUn z>k{3=V=40S_2ue>MGs{7Efl`#%}Y2MQM4OCSFHt&>hOM8rX=xldkUHo&vbM;$=!PYT7J<17cV9hzdxS( zB{eZuL@oJM;USQB`KH@iLP`q7hxhM2L>zY6E9~8U2qkAI&~;aSzN55r?$liFTm8}F zJ|DbT4=tk)N^eyKMsOVV@GrUAh!q3T%lbp zL_t+rS4);xuL<()tMUG%L}Wo1YZ~Oh|FD1@afruVDYHheqvoTo=AG-Fl3i-^T3}ge zn|EEzeUk{rjA~nghZba$Uc7WN()(OwR4*aP#;Vu^sc51lIKL)ywaNk8kv~9dlNef4 zRKk8Zl;AIpY=#f}>lyyrlFo`E!Lktv*LmwT!Y3m(HZ>Mj{#{#7$L)#!zrAVCyu#L? zD=VqX{74guBS2v%_s+dP#-*UEsA`70R@3?0e;#Fn2;Eg$w1EuqpC~ffo@gY6h;Uvt zx-5ckd4KnDv`-K>81IS7T6!h4EHh6Kc(=y1u~a4IsRSz%J5OmvXjzQW0(rCA=72!D zOwtq<2Y9?aT{;46ZEddVHuQB~C{}a-{Fedld(h7jDK78)A#kI6MqJ8tWR<6TEC z$~0u#{Al%vIEVf4(e2jqe|OHaKHmW+Vmdz)Nmd&?J~B+IYAdh66sObLsXQ|UCR;L9Z0josk>`aGzsV{3cZ4;{<-Um zp>&_7D8yQi((hI2YI;2Ucu+Nmz-=4&MC@_ybcato#s9443C*qiiYL7XnFPQ zqazWG^@|)$-(B2BRK$kX9>CmbG#YU*H+ee^%wi?Trv6=TR#+=>@8Xl0!3%AGEr_F) z7Xs{*(KfkZm93J>ke#n|w=QB}CWAHr{I{bo#Uv=bq+%GNNbVuK&QOqPj@(A@s+ygC z8Rq5oc(M1@rwZ3=N?Xg(=y8D`1E-{fjtePL)V1Zg-~UU!mvfm4F8WSjgrOKteBp}R z!kq-&ri1`L#sK;#7TGtb6nnzq6nPhS;4}6?2+i*#`G-u6h=Q)I(0&SEZxQ}SwYNv zS^7yopm?{PRj5^1e*LcW7j3m!OvfoR^P`Egi?hD|FW~L9=UE5P*?EV#X7ps(V&yV< zx=efj*Z2@l=_rx#dw_`_DmFm-Pe-;s$%c26$tr^b|6<}J#ALNZPCnAU{D2^58m?)x zZ3Bf+ZVKN7=iL$}&f4<+{kuRo&VnlDRo7oqQLH8_9=sM^yR2^k007&}F^2{#kp`!} zJ}nU6#HWuTttf9t>0RIcdGKUK(#y_60Qz%r6L{-x;^F6=zgSo6aIbt(PGozVZNd2K z>5_la8ZK7kj+$BP=S}opVN!-6+98O&(JFXno(g{550tjzQXOLc5t3|nPD;+-zMj9o zt~=tiO2}y!-MQIEbt%=40S{e@E$ z<5)-UYI~h#DHb==)pPOom@a*sx`Z>`Cocr1PmX}s39^8T zu^*tfiy|MM%4KZo0EuRJ&2Q5j#N-)td^uRplG_ZG{AOClCKs~ckf+CYXPaHEMefRt!R}X;Nr74 z1=u;%;(Y)B+K6|ew+RR`L^8`L3jO~M)8KSaK~?5uJ{hf~@vn=SUeN=1km_>RYITnk z+Z7wJ9iK(3lk{EJI|>5K=KZ9Xr^QGaTy*Nq$X4Nedci8`f$YJKV3+ zp*ojA$Vht?Xv9+4Zs%~wN4=8uVZOZ&)(^hi0ddy~vCE!E1jx0HfSqBP?N>6fF~9Kr zqC?QAId0h3h;KgKBU=Q!+K0J5?M!hep2OR7OZ7Dz3}z=1V!2a?>Ri4QY>l7S^a9I= z7oHW_u!!d}JsFku$^}0}!%~1V)4N{c;R0)ZJUp#9F=UY9G~7koRVT!tOksgaS!yY` z_SLRd7H1bYPauyut?Jybqj??gK_jL2`vm2ZW(`K=l<=PnsX?h)&6RO7aipeMuk{Y9d z4*b3EBBgcGAa&Q{_(wJ>YO6AP7R1E48TZ#VH%CJ{_XfQjGp+~T+D<4^6io*N8S={x z&1;P_YG>CB->Z#5C!6B&E-+G2BY&UD(pRvpC++IJy+5d#!e#Z>)HUg5iO0^=J;~z@ zep9~_yWRE`-$}7xjfAZeOQa0qWwZN<_z9S#2d4r6YTk8I_@l7)+%A#jvrYK$BI&@7 zt=54yhP2#xD6WG*6*K3RUUPFFgtRC-kUe&Z5^v|lHJ5H&GxU0(bz|#z^yQRLmhguU z0ar!*yYSQoT9VmsZl^`_-^Fy((mD8?@V$n5IhFd}>oV|nYd+pb8<=Mr3|5=1?1GKL zD{O8I(Dmw^$#J>(CI2v;Pr@>2%0-#kUgfrbnDX{mbInh9|4!DEdQ(E*_j2xayJ_pp zn^hYZIZ+8!EwzhqBx1K_p?eh*%s%amCbuhHaA1x< zXhv)uL#1sgvydw7XPc4z2E^jCl}(@}ef3Xq!y1NGI?v--?Xq2*wQVblkZSl#cA&|w z_kGX0&^x(^b0+*unGzByidyr%?!=F>c)Q+gWN_8lU~jev@>UpP!7fN-yV~-4cyS~G zaC%_yM~=5b*(TxW2V%Gk8#?Zs=KL|_t7QB9~&7A zZxdVUU%l6rHw1kKegTm6$=vond75@?VAl zA8;$loZ#P1$F_fdy&tV_Pj2hB^lw;&7_fC#WN4al`bJhT)QM3_gY7nZ6GnXYS2~Tb zN>3oMRatHWp;GhTl{<48$c8mtjfN0t>b-m}+Uh9wkK1VDt*M9H73L@|TBHcI)KsW= zbnoV1FZ4*^vdJE0!Bx?yjz0f2|1a*xQuh>W6ElYIwW!}kj3>vTttnr6VIpG4Z#kfq5# zHf?s;6%|R?-2@E1asSN2vrH!S3>WeYKcMDJR@CfX!$_WqEe3&*|mn(;anHb>3FTB23_iE?aEJPj z%|B*X0)m2eEtc zu}#M6!gma&_D(VXXtn%Q;Dt42Rm5cWG@eyuS-qt3kzp}jTn5Gg6;EFYzZ<;TNJTDg*MrZJTzB4#cX6OD6o?b{|OFZ%0I3TMU6aaco zT_!two~ji$(2n8^UVC1jNu^2_GOXBDH+#D``V>vR4!@sb9>2_}n;nheLJAn_G9V<+ z7S}#sDzdFkoVlky`JHomYwc-$DjMSc@=O;g8sZTWL5uw<)0~&>X zouRI;9Bd}$$KcmEf55^@vS3kN7)#-b<-?e~*7E*<84D*K(NQdb>=vSe=Q- zOn+V6amzG<%X2!0o6tK{8&47GrK034iu+jF*Xydp4j)xz)vDnUu(y~}(xmVa*g3gI zhs&ivMWT|5{!%vnsrrihBDP>fivIEf1<5ez_uFyad3GJe54Qq$gC|7DAkX!7Bkr0p zLzUCA1cL5krh9ycH5}N={kui6kXRF%wxauK{syRt)qF8}15PM@5VGhJK)^k;d?7oL z*#h*9pQsz56JD7p_8Dz#qp}^oqnvOhlaVlumuczn2_&Q93a4u|BR$HFDn?8-NFK8CP?J~g zv?Y7;d20;y#)i8+Ral@Ey+%`HFsGmVayq;bZRg}>xe30$M5gsP6>z8Zcp6Sf3#JHV zVD*3t2V0B}Jnod&tc>b31?*SF2FZHr&)sars=$i&g zQrckp95Tz)N_E4H127e3Y=0R!siVU~G(hPeLL55<)L~b_IDI1`CAwc?| z$qrsRSZ`$sraE)_#WYo*sip1(1g$f=+*MFrg-1lI6^={l{D%eXAcPjkDRV7R5Apa* zYXO;&nabM6YiZ;6`(GSX%eHWTq$iu65t3<)MiEcJ7rAJI>qzG9=zLzPr z^XVlIJ`P&^VCK_{^T_`HDTJqXzfnXO8x+XN`}rsim@BXGAiYh1Cfr)=P~fT$@!BA$LaaT=*AZN7^N36UQ?= zFLOx-3(-D*g1^<{zs!If-Wbt-iK;pYe!GniuXMU5i>b%cYSC4L%Y6%pY2kZ(S;Sl+ z#B_SCr98fE;p2Jk;E69|mTEABO!-yoy73Tcua-3X?4NF*-K_E_pH1evAmYdJ)KdE} zAh->adp^w|W%iV%V>4Q>heqZ;e%3!%aQ6G_2L(;cTct}0@~Y-CaQi7 z!VgHwxSjOE4q^=4V3H8}sJMfjUO|P7n1^gjig`n?3jr-DEf+hu;h>>V8Q~z*CD$7u&b%3P2v`UZM`>@n?0CgnLBT zCP5zXibsu48>2@oP0>`nj8+kui5#6G z{#Gnd1F=Dyyc~wz7d82F*Yj^2(#*-vUc)v~wBvanUL1@Zlced<2c6vFkIL+VH$c%x zY4O6ow4oAX(<$B^PI$NHgelI+zg(&}Wf3zYkNpu-W8p z&#j`B-$LRZWe(B}+Z9wLLd|~;k+_kzgH6q(eF7h`!&Tk4tq!Y9`pXWFx^L$s+IMoi zvoPVuoodV3CjY{FnqC%6+u&usDe_`JAyL}U>Q>A5lXWa71Y$94s8y3ItL#R#*%yG4 z165C{jg3vlT?krnK00W)JZ>V+f0z@8e13%EY*lk}|_gFV}xI9{ExETaJt2x;>~)pUs;Y z=e;<+o`!l0OZWyQpZhuT!)^%;A4#3^F)KFkA2wr(eLg=wgU$m~PC-dj5oI!ZLpw1y zpClMX1_Mpb^R- zy>^C5G{q5TrNn@`?RS+W;1yF7=tr^=U#Dm(t;7z|Q*$+9#k|k^kMJW#*);-MSZ4Vu zZzGf&FR#6y)7VQyeoLz);!rkkzFyZIM5tApBGg1aRNx@%wAT-pz^yRL>8^ zueXt<;cCTLO_EMyiK}?Q(UAx!AGb>kG*jw;VtkNl{jN7B#hW(AT#mb)!QZpd3t2K( zqF)CSx{;*FgH{>c4k)e8M|DE#0Ll{TUnYu0hQTHH9ndJt+$LD|P$vbj`hL-j#R35BX#HV>pD=xR`^> zdueqw;MTj=W3n2}RH52iNPuodYYLy4vMKE%yphtTMT9VZ4Bp7Y6c(dJK~Sl*O%m)6 zp{;LvIb6jic^-W+#LGx=BK(@O6#;RrM+%;`x#gP_U;3PcvbjO>Sf6|IsZ^8&w6Wyw z<4D73rKck3YdG-1wkq1S7BvQm+||I?>Un%LhB&uX?VZw|i;pr>ztD!rtmES|y8FLT z=~TsE^9sbesi=c!rfL1AlG9lEe7zQDwRr^S`|A>sABa&R35KFIlEj(&sEdsJnetj) zYrJjyVuN5#m`p+6*evpg@mn~eMui-=2P|)YZdXQx+AXxHmQBUhAy&{k}Lto@Oqz00bicySwBkH3Eho>wnOK=q30vSs6;5r=4EX5@UZuVh!bq@6!PE|OeXK@izf4e- zK*^FPZB(G1C4MC=J?+q(Od;&QtL| zJCZ&}U{>szdVLmsCYc)i+7;*1eg<9*ec`LLJO34))6SmKo@KVVk&G+ROm>f(F|zvg z-2c8zW|e7w0ngjFMOS(i$ad7|>P4WQ$YqEB#!=gBv`~a*vgvLzZ;rDx@waBo(R;}f zLOPl`K}p>kO0agc2RwU@#;NW@PO3X*J2_d=J(X>rh(pdaHv-oRFhK_k^iiaRXO+c3Mfd}VTwto3Ytl(ms%-^2cmd=sYcnW0Xy?SS|h>(@`*IZJj z(y+)fEeidBdl`6ok$``d!O5X|98^u+_Jp?U=JHE#a2~#~W<^=IiYck6Sb7r1u*+xA{W5Y5q$JOwHgYXW_VpuTq|NaC(olmvt`?>WoCNO0%Eb{(JQ< zFd}u0qCWzvf2?@#M=Y>=|EvbBLf>x*jiV3pNr}p(k{1!GWzv(&adJ9&;oqeFQ1$|5 zKb8h);o5t}E|gcpBcIbfgp}tS$k=~;6+Z=%`@^Y;KKK;$r@VViY8%SZDc&;FVJfngzIxb%H>14W&qGjVue+o>fJ0C zJ}%UG2N%5?u+Hk&Cc4GLyrO=OMKF#&=2sFZouwRaSq z@$x!jIbE?AlBr400`Wl_xb51ejEBjMHU$P3Jn-;YIA3l<;+*Tj_-s< zA@bqdbUUCC{UmhKHB+~RtFW5KQYng;*5!dhBHY;d8VzguVh%v2C?}DY`kDCPl=#oV zZ|xI2Lyhv>q|4tSj_fyPuP^>+w;P)b4U_n^P6z#};>o+h#SBzJG|=$n*?t)r#o}pD zTEoKy;qBg=dyUJBntqk0o5}B)E<3T;bad>+ed2f%E_WU6o3Jv-ZAfK!qgoRypDo!t zjHFrlefg{?A+;%lC9hKvsuCkoCFw?e!i_9l3!8Wcltb)Q=?ycapek*8i*lHTb&2sB zWrS+gkd|Rsia_?#zO?eTQSbgX+iQk~2ppH!+O1$i*fx+3)k?$q{_D$6KnL*MyS)R2 zMqj0w;pqsYKXo`(rjx|+`vM@s@-_cB@u7+9IYAuTMBd)vBy=E`LIPGK-bh>8_QHd# ze-qPs6vuLs0u{62@0NkEFD}8RZg#&+iM>gR@RtHgCzKqIcNw(l>Tlu)duE$usXsfQ z30le2Wo3yQ)P+S2i8~~t{6R-Ys^iBCo$16OOyilOm7ZJH<}*aP&5gWkV!SR*X%SWN zc!4HJqcJ`!&c`*Kdmq4PiKQ+%$^MZ9KWh`>7~8SC%r~{kcj?(6001Y&RR?&FL;!$^ zujE$7EVBz8Tu?{qyJh%mhU2JaSZ)D5!%2R>ssi;3G=&LZ#s?I3( z++0=vs}Hk)Hy&GVPKdlAi$#?tX~p=KyZ8e77U4+Q zb2Izl+73^C<;}eI1(i%=#_X1LkmX6E>1QZ*>nQK$sb1>ps@aRZ-ikvdhJ}9Sh~m-g zQSA)Ly~38UP)Y!wjXzb3#k%3MESV}kn9EqOf1^+E`|=+au=gWT1Hau?#a^Pk95Z;m zKq|$g#VSFlQP9wE)wkrGMl<)=yhA)lK*_meP?Xz^9HCST_E@hAdiekV`eAnUfn~-) zjco3J-rIVRJP9vLp1EkwDaW?m(Q`+v*Sr4x;+lP01KNGv(gMM;IKkf!mJ+H$;WLVa zB9Viob%w94mzuaQpjj>efP;G%DSB*HSL`?98o8>BlB=sLI6+GYx(n&0gByU3ruySa z0x|&bL+YvwXk}F%K-m5ru%?>U$bcZ2*$-GlJo$hpy!HL>t^fe*F)_xFg*0wyn4c$W z>(iV}574yje_y5Y!HEBUv&JGHh`5&U(FMFxk$EXvS0wB#20eEgW~|~OTxrc(d$$tb zGcFCs9rq~61c52{#NSrtf;3qi0JU^Ug;v89Eb$9=d)##eQ5o(7K zJBOXM@$Jb8^-h_3)7OcEsU>8eL}cfMWN$I!!P($8B>?JW2R+_+X;Q?PJ`v{f>Rgc>zYN{qlPH1Ns&^Gt6oNx|_@x!yiT1cX=8)<&x`M zWv>g;$)LIFHMBH#VX#-XfbYo!uZw1+Wb(i!^bRb?6 z*45g1_PuAUU?W`oeJ_dVz_0WadlTgN<8&#kIb80gbcxKHQ3ZO(l{$p@Hz?h=i9^=r zwa}wRU-XB2d#|@}$=bvOs{Sq#04eo22;EuRVO@S0vG98-^-;26{aL6Pb)*$%HT(X0 z))TlxLbY1nbN&ecXnYUFwCI^&%x$b*^t(WZ1xt1jjWn@=PFubb@wC)r>A+#%)VNhf zaDN=4ZX)aLb8_$p?THyfZsG&=lO2IKjxhCZW zD{V2P4Cy@leA}`q&Pl|Z+WQlW2;?|TB$FjzQH6Ecq1v>>J2D}YwcQ(MK85JnG}}yr z`uqT`jbS4|5p!2b(Z0w|nnvc>UB!3Xd_6bwGUZGBen~>dh}V{zmsK4ESF`R?+|5`} z(&)_%)w}jZt?l9Gm!!22Hq-tto8t@`|<)l%)APIF&2;)}R$i$gMk;Sj^`ka7NzJB|DeiPdv$MF&S&%od_wnhEN9@F9wKhCAmUrf=+_S|qGEuw=o zk+zS28DS!@eCPJd9evdmu#icG2UszRdLxKNu`Pab^?_(}PrPKqr6%sr!sRF4r_vpgIej?V^(T;(;r)0h)TYBe_ZOIDoH0}J4nRndS(IYsnj*VT(0yN zZ0|Ggq%B-6&trS~4$H)rhSi7pqwU(;waT-MWa#8^6)h?t$S79No}gYp#o|G_Y6>-N z*%>B7e|3FHd5kvQQ5CId{=n{vMpsFKr<>Vm)nHj;OHNVTVjBTIISoHGK}}X8lg2CU zIAr0_i(J^aRx`xfBy{ftb1!15@io6!ri*~yeP5uECi{vIi`Q8-%Y3)*V0+(z0>jZ} zf`wyxq`<;S4-Y?u>E$50hEnwXUg*OR(3@%#5;)9eVy{o~w6zeZd6~VGEhemSgX1Nf z3SO(WrDtYw&vWNqB>RS!`uS-5cu!N?nItqg_lUJrs=sk)#QBu@;*QsC3KEdlU6Xz} z;z?@NX7Bs@Qg<{ec)=m47F(sM7uE?JQ+TmAr%ia8MOGLXxtWK+RCk4k;pcZQxRR2^ zET(Rt88yY1Sm`MI_VzKG{Cq;W77i1Pp{qgL#H@O~{7xoCQRLlT5AxBBx)U-H6MzHP zzdubgoQ;giB45!Hi8b+I0Ju++dkstN_xle2vz7iY+hH4Vn#K;bdCngd#Mjo=#7AWu z?}9#sSizaVOS zxa{-t>H?P+EQ^R9Jj3+u&?lq(V)A#XI5Kh8I!<9%=Ib|oG#CD_6m>{A$*Uip7tJMf zNFs$ zRl0i4`_Mn%-&BVfK2-2H*@;L#eOsrWvW!f8U7;a zSmKMRNAW~_4NXDR&n}bcHi%588BZ8+;zmic2dQXRMYy+2yV%)Y46@u{FmV0QkCl|v zO)h3xBwUNYVVRjMUyG1=f_itd3=If0PS)2ot_IxgB|k)Ss&LPNAXH1L92#*orZf0$ zswu1N2EmGSLmwGDOME1-9=DAsK*5n;#VCaaL)j1E7AL4dzkedAR;vycEpo%h|Hv)Y z3|GY*KUpzSYZl#pBuAK$`&CMxN6?pJvhk7YV_;-nb!0SpQ_R9CefP-L2LI;Qp}|{C z&<~0nTJFe6rI@FSdaU03^S*DY3`&)_F8h8ro!F_1(IRf2UIjy?;QRs88!3qEhWpFblOs=JAoCMEYu=S&y~!CW_rPT#XDn4;ft zg+EMf{wu`CaQbdG$~e+L5gjb~SILj6!ZYaqk#vrpb#aZWW1NpD;BgND6~`778DaF? z^o>}$$(ZPu7t@LxR1Gc2v>U}C4%upa``=sno48=+38 zov&YHuVCQV#Tcq626BExd#K*4X&hlDpV)e(a1pMGd6j@Cf8NQWnP-d}i#=fqFw&O> zO{offiVT8y2NX zcQ3xz9uv*HZ$Q&REFTlyQ;(1(2*hkXn6f3kv*+(!moapfM(cS}((?;$>=MnOBFoTO zy4xAWsH9tXKu!5rJGC%af?Y562+>7UH{Y%zklVbnJ`{~}SS`l_8WsKI8*rnE)*C86 zF6R5$?IONL?R>36X7tD)H@7U4vs}F~skCcSP&kB8<8tuc_7U%*?6a2*VtR8#>U0kh z%~T9D?D24Qu9b9D;&>_Z2;LWX5ykF=-1()}1XYO7vDUTel9#PHXKdVqa4lKm^=48$ z!L|&$Q^agj7%$BZcTe%l60}Mr%dDG6%CFx1&rz~VTarmxw)ols&EmNFU`=5Q0ecwd zGRz4dBJiKy+kjG*t&ce_N$vhpO9!@*;oz`Ckg#LFzS_1&Vg({Ky@^{AF zPhD-x+@3?+N;)mP>99ECGV}KM*HUs?$_(JFnTw6!2)rpl!fwH$Q?(Z=TUo`9F!&c{ z@}(kBJ7iqATvnSIu$IBY`|#*ibCkqe_PAj>GWtxf=T}zPf)t!>$m`wLHC~9v|J%z1 z!pF95bDGVaQ03&5RkcY4PNfp^qIbbeuIT-CBy#DJyy+6pDspe$v^PkU;NEu_f#dH- zJ|IN!H##-0J{g>NjstzxZE-`Q(=^-5>zs`c`x9~8g;|L0J}`bQN;WLj4jOI&P4W*p z=_^>+YqfiBT77A>rqAQ%+mW5Lh*)zM-ea^XFW>ql`5mykwmx7L1M2y&)dZ8?_hjC- z)Ws_1i^i&9#l7jY9a_zyZ7CPu_;_{@{HDrQL1Z~={Ng=xn_D_iNhUC9Y`WS?u{>V0 zR5itX3@3^dnlQlcz$VmsWXMy}wFKWlCHU2|Hmu2i{3)!{LAHev1nZp!Nc+^QL&EW= zIHeVv-1>>FxEZ!=Ih$>ymvDv?fqs}eSd5-dk@cfHG#a=Y(iP8JHctlLtEr#tvCZf& z#rT4QkBW*XRSiREVwHwZ_(yMtN0-HGYHo{bRshd+2=(m5Ct0%X@=K4zOWC!X+r!#q zar)WH!HgG!o|{@EX(mkC+k>~Urn})6zA(_V)E(Yls`_V@^9Ekp*o!tTPFEvaAAZ6} zZ-Tdg(Y*5fWB2QPDG#vqQ~kWvh9T@gn^Lfj`#Go%VqJf)+u+FG9}@&W*@s-vGW(8A zYtj2LyyYsP<#y8Jh&(+Kl7Tg+JY+%^k>nV=eNt$(cRlp`xGl<^dr&ls`TrS2T6C|c zX1GrEv)=+_e`LuCOl{j&CpygKXUQ4a#>KQQ2ZTY3nAe&C&fp`x86%P9o-u61A;>DhfQ=Ec2`t{6I{q+S__1Nm`9tPBisf_8e+H>s{ z3|d7TDp5SSF{oVx=H93kJ}$SqZTj%h*6fO0oxmidx<6gmU=`QATy=lD@fb)jZ_$mv zzWa6)eBQ##-Y;SMYE{$~->>sWg%=L*`2Nw#_st5-=qn%3PcH+bRv!pR%}TpJKKAvY z;-*cK*8ZCKrsb|D{4HRdaZoLxay36WwYY|ycAd`wd_P6pqKomUS8>x0?ATAMdn-%N z9-a;xIpcCe!7kckn*PM88P);?rtAhj98lVvI@bvOOgA@9hYgg~i_eUR~j`q=v#+~##xi;7MVdF^#>-}8^RCwEd%I8e&#z$GJ{Loxf&RqAK z7DWBHOF?uyTt-JLz{n(EgZ&kEkWHwbaUu09a{#yA>D-uGuI5z}G8Bz8a(h+mXTMGE zb7~*|aqPS`-bSos;;+*$>V)=}gNR@38R_}#UNqD_IeJak7QzGE;THiA<9;^LYCXdl z&Ksf8LFinARq!5^Qt%aEi_l1t7u}FU9C&#*o1x{o=S z1%Lpw(7p#gRB8P4I)Bb_CKDzH>xJelpe(8O3nw&r zKYY^C;IxtG8#0`jULF^&cVOS+t{2oC7_8=6-hd{b83d!QaEAP-7bM%&P*po%hDCBW;Kid< z=j<(7=Ha4vuiWStHzhRlxaP|}NSk>Htouxj53t2ZYf9ejQ_H-huz$Sk)7VF#=gvW9 zx3gaO>mo|CGZFvYDMJF>n*vOnuyQy^bEAzv*t~Si z%jhY?q?AgEvg+I3f3Za;6?#R{mNNC}jabkJ3p*tkhN^~Ut86^LY#P>k$Mv|y(bmqY zza?UQcP_BFjTYF$9!9RIFhHZRDMGf=>qG|1W3v&6vW^Z&wr^R;)6>DHKTmLn%ijFo z%o92Wlu&eH@z}exV)OfKywZ#(`~9~GgsGSz8ifp>_1`IbYc+1BQ_dDmg>SzVQ2ftc`p7hwyL?&f`qg4?@X4L zQc+V6*YhZI-oFRwI=yEK~!xq^DvIDXFd>~%J|%$7h^mZdHs zR$7@6(meIs6DS}OL>;R$j~yt5!WsM`V38AoMX9!6TYPx$vBmqnpbs>I7cuN?{yh+j z`UA9kC}3(#7Yu$M_xUqf9u4~bk6<%4IP_-!g{tdWi>o|A-b6ThwT{zqtD6=I==B zyxZ`mw-7li_v`Gt-(7h-YM4-;tA#3p$Mn;B@ZH|eQ*;e>cnxWH<1l?+!S&E9ySJgh zk(-A5+;5j&kNs8PX?aUaBd2s#%x}IPq>-}7Ybb7wiKOH^@tErTjUN;O&g+H){|;3m z#}3-;t}Xxao2;f7W}s%*n|A_p_mrXvsSZIBfN^}DG%hjG-?UN1jkVKd3)6Y*jN#-V z$9T4Ghr?ALsf%-jfws^*UNcLwuaN)c7M_)#psRc`!`retoN{e~ZrxE7B#8kdZnB)z zc9DqisDUj(Sl4ZmJA*e03V>8oe6{Z##)GnzF0OOJ&2KOzn{2f?gQ4r2fK>d=8a{!U za#b44>V5&NT`-4(Bo8{nz8iyEjq3`k6hn2*l>+_dDAu|p%1AZ%kijPz<0fV)5EZGi zI$du-J}4LmpFDPGQcd&_PTAfXhZNjy=PO|XVo>iyyI_lCfrlx@Y8de~x z##il!?)=_B23ekeS|$wn-vUCH)o1EL^)EWKpX&W~?(m90DR zr3wb2q3x0f{06mEd*ia#Z;~+|%~uIHB%RPc>rC4nY=nFUDc&)K%7;SRru1#MYs=89 zg~}}Vq84*2_Sxw4pZ1RiE9x=1X&oo4Ff6&E{ylSnknV|aA)CWvj^0TJ^I8EB99r%%#sjj zO23PP%uNiJ_#{M;>UsKI^K%Rot+ALz4`y(l_q3Cu;0^&tEEB~snGI#*20mzG4xfIp z!-^)`L&#}5te;DPbG-3E2Kfq;L-6`TZs>XiB^)aCM7c|9GadJ3O)2YJEuNst( zunf10U#^}^s(q%CO+P96MU+c@DJeOlQ`R~WN4JhmE+amt{YQ!4t%{P(h?+6^r4Aj| z(LA4}UCR0f6IeO<)PPu|v9_dIz8rsmUnQoYMV{Rl=OU$+g-Ak&l&7=Adyl_+2-4V=qLOd*B0Y7OjzigljATtqj40hG%J#S0e`?o(c zr9Ee~&wy<<_<3+<+h2dGc;J3G>OSrW(6QYNZEAGo$|)0~0CWWYe`>*3{KoETs9h&- z^kE9UYyOl>>xc$0;O_b5vHNa01NSYUZQX~Epo5Q*X{FtAu9S^_3*>cwt#|AZufBC4 za^!Lzid$^);DK`UzoaAw_WL#M1iO+0pSeO$=dwuan`MK>JjP-g>le?>yHz#F zb%}@1h?gqfN~1rQMJID=QTfP<93sH>4f4@w=YIu)0sU1lqHlM%sm8rG`^stZKZY40 z-z6#UhK9l9$LBYIZ;dujh7CbIF+pP6KZjIM+-iu}OAc+=SQKa;=-=8h=@R()2EOA# z9V3|ava2=TeF2FLtljt%Z(~r^ZGcnJcggRI8*D^<5s@ALWio&VbR>VRd!qyQX$fcm zH-K-)&KVYciLaD}60%}nf8$=^-%nS51c(J>HMqLlrs@-RqHQro-+qkjvI!i|xbS;j za#PAB1K6rQZ!=~3F9sEaB}TT_95HaG%5pDyZD&<{de|0+nzm-*`AacLW5F)S05P*N zt{~`Y%`_xIBgz2}5|YTJq?3g4wTOdM&EEH4Mc(hRzF8qCJkTsUT{Djo@zsKf{tC;% zSs$%Cz-xEi579Tf6$76QP0qap;z8n}IIJbmeF=&;g?rc!%CU=tq4#vqZ#Sc+pdgb8 zio*daH#ES{7A`ffO*WsgY@sL;*ejTSON?okrAeQxfz0PtXwD1W_Rm`Olm*{?N2D_(h}hI^6X&y^$O?3+qE_MkO{3;OghlV*3>y%l}aJ z)=^Qd?f>uqQX*2)r6M3LARrA=(jwi`-5^~mAkr;4gmetu-Q6kO-7z%3J#fx*&hxDA zde?gQpR8eK_MW}(`?@}LJ-3#@zONP%0oK%mq2kQZl@1k@JgreTe!1zNF_m)*BWHBX z)AZuWU=7y?_eVMSaF^BbL>5AG{$9vSMVzoAc(UgK>6yZS^CDJZlRj)771%=eWHLVA z6y0VOucRVP7Ap!0ws0{=_+8%uwWNd-Aa^|NRBc{+0+L~J887}N7L5a{jAyqsto%U? za+xOuVyIm*@y@r(#vQ=72Wo#v-(k-FjRpJ&Bt)3|Gw-%4wtm}-v~lJ8^zbw4QDvbA zlR0Ep$e^=#A*|^H2TlO|MLw-gPIG{L{SsjwxSl0Pa)GJr$Rxj<&<_bct{%-VOe0@D za&k)LmIrm1nOWRynriB;PDvVJaNIdXRd=>z+v&ru;NB>{hpx`}F{|ax$hCLPC`ZHf zgtTxMXx6)}13EwDS)Owz%b_+U+QH7dC4YH!@ev(PK-;k-#eX^h4x~X85F$p&Pxxhk z{_MPioK~f?t(XxW_a8}s|&h@-F5gwUGBf3;VFK$dD=aNOXM$NaFOql%uG;ePk z^l~|JMOpy$%+lMb_%G2LqQAt!^aN(X(OC0lA+6B zw>n@>-TK)L;vuuZz~^bTi$%?yguW~j*%I`Fjq>>~jAE9s2egV+>le}F`1s`(UjjCg zHFEza%+{s+frgP{a-jbMP5=--;f+-C>x~HB+8AEWNM>N!4g8Qw#F`DQ`15iQ zwHQpoVJ(>Y{_`;2mSj{_xK5N5fm)bR$&yOhb2`dN-CpqkB`O~j44+HgwglYGq3dT6 z#Y$8*tnyacu4@I~2fMG?sw2yjlV88kjQxO&Hu*NH?IJRg#FR^Qx%NcbARe^Cur0ENs23*VDR$k}!o|psXT^a3=b#l620CkL|tYmN&aBe9tat=V*=U^c2V`lpVDp&ORF4JfteO z261rb`|F&!{GPbKYzR$cAI0lLqr;g*+j_E2ulGyx(Hz_S>V%%2O?Nc0m(*s zm23qqKWEH%)z&gug7AIQ3p&<_Q?$88#1Y4!NM*9Fh#%j+!QkRyW*Y}mE>;AiVHJl0 zoEJw2)6|9YWj-4V#K|#5eW5;`<`H=x*Td0l%0}0;$0j~0-ogo8j@?YY2Mv}?X5g>Q z-|hQB06N89dM9Ux!^n?4w_*mG>zF;=dv!dQepsC2-eYK(^a^Bcl7rZ#;fH;_yBye6 zT)-2%+M`P2)_~*uC3-2R!Z7Q?Wicw2WmtAT{KXmWt-=#1g*0A(5ljMSYDUY;) z*qlLa4y3*`>aRRYcvtT@qOl0+V0Z7j)#tC7bDMPQjkVfa0@HRncfC8vB<>r{SDQSE zkGJ}WPea}}S?{>NNYOluOLA+w{0SbEiM`51t?7pymTl8yIDogKk{G5;gGQjUyQ^rHz}j@UnVy^J!iIrxbBw{ZBN7va!v#y0`+X`hw= zdKS(8I4<^282vAUkreR&g$#tMdX}$I*8+Q%9=6M@V-9NkcOSZFY(%K-TcEGk-PJ1pr()&qVsW$7sWdA+x=56L;(H8fNxA-viL6>|=uAef|z~9pdNH zPf$T1b3!6+J-%J67=i(?1N7q|L5H?9_<)vE&9=SVC_Ga7iaNvS(6a%$vjfe_eFihL zb1dr%|Jgdq*`C4N*U_50mX`-$<2aJBlT(CjrV7@u;1E<$V)HyVqxc3z>U%)tyWsb8 zPol#CuV^iKY$Er*yq<6%X>y2pA>6pc$WnQv(-O2fis)lw(!BG%cqu{p3O`?}2OEC= z=%(E=a&@RWiLUbMi1clL&tQIuL4FY&8svCIl}3ZG%;Pb)wEU`yPlqY+lErkaDdPrgA;&jK+avVn(kBMoI zZ4XXgW@Ff*+8+9aWZu-b?__&TzelL$e%k=M2D|SnRTB`=W~Ei0;vaA{u@WnVLq&ZL znF<=vmJfK2QJ?LoSWvp)&UVNa>mU*N-$qidwXrE~71_XvNFn`H>lB-YK|E60&G>@k zvl=46vhXi2|ke#{S^yFrVA#GR!k6_UwPCola$em4! zok3@A1`Q7n55Q%<0D&fy?`AjG{BcBO-59OY7#+~vv~WcjDCZN=5{xTpuhMsLF#Jr% zi1z`v&kWeH*YmingBr;OtE7CVvHsE~Bu5_~`YNMd>UeIhMu9T-6L|_Z&a-c&9TT$J zZns4a13Y9LrdfI96FP{*clq;%t5RBbr^PmI!lYcsPEAs@^Q7?}VV?PKNJS49!q@R- zLeRsUZJ0*K6S<2W6qRHs4@qt4{ZAPOG}TkxxHvWXdDUx=S4-!~DGRs*4PP=%4Nm`P zC0}lqtQ1?sz)x`cbzUXqdAr+@5ri7$Y|~OUM$PZ6H|2{Eu4fzMY}0k3;t>|lwyJ$= z&=x?xXa*0e&Um{%l8U<*_KWUmTHuCSuom2lYw1_J(#;7Jg3vMT51;gM7&8|Wx~ zZ&;rnxn2(figreLFUv-D#&Z86^dJk%`y$k#8z>De$-N7D;TknZci(k2qpg1TDQccS zM5Ct7zOZky$$iM9p%=6)0OD8n|2Ozz=cJAM$4hM8uT~m7ap0Q$#!Z)S5g?EzL3TZ4 zXWDV)<-}}j)W}t3=Hosp=8Fq-CSG>t-tDa_imI#NXJ)nVd$25;3;%Xl5REka4c*-! zR5|XdD>8)9l`@IA+GZlG8t>C9$Q75-?lA^2mkk}~iSum=OM&cq^@#bG3vKrvvfyCS z<(rmGgq;DQ^|In*o8xGMR*B#)4l4HKO?wXy%iGx8Z$g=j>hV~zAdqJ3saJ}>c*5^H zZWag8NuQnkjc)jafDqkC$a`_nf+bI9Y?T_g6N$dpd3p4Y>QRWNqKsE?`%`M*?kRrKvx~rRL8e;n>&Vs`oSOC(TTz^ZR#f zi`{G#9|!j~Ce-n|dUkscgM~2t;-sKW{dD3Rxk|-nv7mxZMm{qrn40c=Y9^Ux(cWu| zYK|8Pag6$gsO~QZaHJQzE-=2k$|>FkuquW!7Z9C!z+OaP$OSNU)j$+?@Oum z1dqe)p14LqCve=+X%&z{|F7_D!Q}uBh zSceDU7>Qrv%tP9$XK14>W{Ef5jRqIg4JuCT`kKwWv<8PII<`G-uNKYbBiUc@A%P0z z`>G-Ry`O^DtRP}D45zfm^V34zo5OmtL61*f=FraAe&ARz)(JYheIB8^l2go(+8{b` z?cN?Km-3jO2_on=fTg9iJ4+HL(rDfrH)2IPhB4(@YYqe@s0#cpn!=yM}^|adgjuAilFG#cnuYFszaMc0@R-dDB<6)^gHR8epXlC z&b_h@(EvFv%#hvzAkf?#RHhevzOW%?gr&(}atAGkUxMUp$K>cQrbW{lBUI=;yPj;7 zW7$rl?htm88|i`$E7x}1#?|m8a_K4`rl%JjH8RV#$!A!;?2X?_t>G0!-eY2%O0G|#%7Bf)u*0{wNnW;tj^oY+<$WX;hr&-W>TM}1SI zFY~slZ|#@jbnr8Nf*ghZxq7;jN!7<80XG z7&_K+_8bpCA}=WMNA|_g3sCS_Vl9~WUqWK%S3P&1vU;j}?Aj1pGfI?ekGivZ4Yq}`nGq7RyR@5@MetPZ~ zF<2YyDJN9CU;~x)KjVTn`Q4e8^PRN^arck1xza2}WXh1JMY{{wD^4&l#P$D;1$1%n zbA571)CbJre(1(5uCqyjh>&%BZOf;X6gpd_kJ25F+giwZV(N=#1Dflv*E&U{OXy#} z<$3vb23%?&Hs|UrGX9spQen1NciUh7J+kTXgaxN{kw5(2FP_t;jkQ|Mc}-TCR(q*b z(w-!bE0DT=J#!Z{KMM-p!gB6?$N~EtG_F%;qX`bMr7@JQ|#TU z@eG6UzFxR(Z|;jbV!A6{Y740hF40{#VsLj$uqEKHGu&;vW{_lcJj`rF$Smko;c~Zc ze9nH|y@~Ntc*N6uBu4T{Py+O9E}eF&&7JsX6k4XJ^~jJ%!?x!L2MWjAa`t)u2GyFA zMVm$2+jiWz2z4@*^1f~XP>eS(jIs9OGq4>$NBQzf$?gyY=Vf-aX*w$a2Elu2HN%tu z;lw{b{6MX@IW+dzt4GM(7RL*#rMIg;RQ)@usLXMfvS?=HruWkjLDSr-y7lA=%CY8QKb?HfZD zfNso{2nNS{u5B$0X7`L6w%X~@P>a%8Mw9>PK!;GMo|jTW=oftHq7CX!0*2tU{se3x zXeU`d26J=HtYOoI$4I57&a_ZE^u4OlJ3EF1F8iv*@3)w-u2Fx7=JNNW^Ir(@d+t%S zha$cJZK(Eis-~P^0GFY$i}!48?xgV2t_>gZ1ck_b;+Ftn*u6;-6{MjKfMJEM zhUlT_bt!cZ9VW-hc<3;sB7&H!(6T9*NHa$1P_^ccM->a3H|n)A)V{B2qQv&%CX}+I zqli^g@k%R!W(LjCc6^V6YP)&l+$Wos5O-f9zGI&77vAk@<25t0zhp3@s$(SkY&@on zG(Jm79@O7Fp(o;)90gi!yRq5s*`(nF5S+Xn1IO zB$SYfjLc*j(yrp4K2zk7Rzhj(6qnQZnUhd=b#6WsX>dEr++GD#J9>71Z{J|(X@D*# zQ&3Vpqg^oz{lR#XmX@0&@>3l<0(`R8jE|*v7Mk@Mr(aaZz7Dbf49o2^S1CN#p7#CU$t=fJt+Z=UT9Vr+d7K)6<6JO_S$P-ZBYjj4fS zPxC3LF7Kiv7XTw)tt7<)m>>-H6$b<|QxDRZ@M*%B!^@wsSPIaaE+*MAjdxr8cBlGu zB4xL3meX!RM_aG?tcR0s7s^8V{e9UrU%5KV@t@VK_4G1m|MX?o<`Oy_uyQhSd8bLk zXR}~$Fx8YYqAmXPm0uCxe0Lo6ux!61nBT6u(t7L!S3T8AJ2uJTbHJzoyfq=vHG=c& zf-wmjrSsx44h|077%Bh^0oZ5(*tEQH2pSxBcJctvJu4pq^g)RhUZceS$lta|yO^=G zM(vO4PH<=3?bi!rE4bFYw=OAHNL{Y%yiZ)~Cw^R@Iucw8^U29Wt&Ig#6a*IAq1B_v z6Uc32lDbje-RBwWtRZnc$Diyg@lD3|GkJSvlN{%!L$<-MvqwHw^3b2@xPN2=Kw#Cp_pC-~rSMhH~@V6Ub6*cTT6H zVrENVH+KME9N4`=+X{oL1|av3(a*2MfG<96<(m6s3T7H}VU`+ZufTK`%e>`WKFeri zUQWw%q|_+cweAwqk?R(#*a?28DwSdbvq$2fFM(&EO(`cftQ74Gef>#JrN8S{GYE)x z9qe8HrQP%(?OF_7f$SP)GCrcagg<^F%D{2-~`Ez<)tHL zbsgD~lNL-{-g{eWuWYH~va^zcqdi&1d!aw(HDr@^9bYx9d!>?SW!kPy{g#|xZBCeF zHBo3v%)#B6FWpe`$)mymKl$~}qa_O*3Rswi5+;rt6 zRXh9J@~V60U?qdya>gd3U`i4T%;T^$SiFOvI4+j21}^f@t|}MF_$Xa}Cj;4c+_~&` z62Iumn=iOI7{%RP7@S;L$sWOu{I<;$y_w_!WXUK_0L!_x=cE^v=|Hf7Y=iqksxQ#i z{o-P{xMjYzuS%$Kp1pLO<90}|(*C?tx+31vU9mz8v6!v9E`yG;_1*Z=QooufA9Ax! z%TF(Ktyg4Iu1lqoS;qUhEu}Z#zJ6zH@Z1aS@V`hy1HV}sl37(yrN1M-aB)bQ_HEzo zT+zvYB!loE!#gZ4Ks}zc#^gh@QX%%XP{6&awvZ-X8k>1Q(3Usnfv$ z9J15X6I23QGZa^qvjN%uRjX|Y^rO0lH9oqhRvPaU$EsulC+}R!?idhc*7MTmPTcwr zHy5R;Ic6Gq>jzXp6D(8?ZGB)XlDjL~8m-k_f-#8pU3-q*3d(KE*J(9%Q##2`!NEjC z*s}?6EqY-}Lwo4-(?kzMi|WGaqW`u897$S!S=pwoGoZv|O61*n<~V$Ouranr&t!25=GFq)=aQ(aN=dx>p1D&vK02~|uQ@hj8<$PR+f%?`+b`xuS zSZbd2D@&A2ZIc_!f-En058a6|S3#9zfMe5&oH-QKs9E~r9-Rjw9`)Xyp1zgIeymX3 z!O-`ohZ6a&{;UlH*zYi^OF&+8(3(2Mezz_dln*R2A)Loumwa|1S?=4zXE3-A;e!RB zoJ_o5lI~yPb?zZ|CH3P8$PytN9rl+K_pt1M5#5y?@A5ZSK_vEV_fNRHdF;++?M3yXnhS}QQ&7#Pqwdk*k zsrnMLsmTPjMavmB1&>vK(#GQ$_=S@`rMadM@0ygO!39qQB5}#A*P{!#AqHK!HaXBD>1?#nv zrbZYxzf=?YdQLdn_oKyc@e?Q*U&v?%6mqh>M2fl>e(I@PDDNn(pz7FLd=uqhI-|{Z zsWDma2V8zpVkapUf;I-WBgbp}cvc)t45sQ9ODmVehzcSx|n?V4&U@IO@ZICa8eLGI%QX2ZvWsAo~ zu?w+9?9)_-Tcet0=9ozD5*99yV;WWv7_br($T#ws$>@Na_0!71=68g{IjRiJW zMOAwPIWt&;q=J^K!!gwiHj2FK*k8$TP$>)K1O6Kd8AbfRBLlw5%m{@T46-Qdrgjwq z>M`rDmg1^t-^$h-wm-zQYRLBlX11gDI#>}?(_%g z#Tp&>CS3q^RmHUg%Lt%k!RLaEMiHQK1kg@5&#MxfwS1!Ok*onefG!|Yl$0FLw$=~LD4Q*S*Yfsp*L1_pYP-_7hU&; ztuR0Tbj-an1GAAdSqQ*rc5l_$BF24gZckfmw!;W8EIS#aAd0mVYryD@3NHcparhv3a*Pxn7N-p5{X6a!?k z*}851W?B0zoK+?xRcp2>v*Bk^c`qTzwBLYRI3Ws1=0yv>7+$WQB@RD16~fbb?Ky^( zU17AsdD31I*X`>Cusfx~HZj_I>Ejaz!m>#-WzOR>c34Dqq_h!GY@b9bI!1F2d14UY41;+u5D!vo{=HxM7a{sPNdRm)BF41Gt9v zxEzT!6y(4|O-GPPbobdYnhboQ?=IORZOiD!uX#m@tuMz=QXd+-%h3B9=J`G^CT^XQJ`f$-8-Ha^8Kf-B0^q_*oFS{NpxQnud=?S@Iu z1^+=p4>}3aA1dFx(&ZulT&4CdRim^gG{|~oFcgkco8<}Q%;-~Q!|n6cY>Wwjz^$-Qxs%JXWZ(&?H6P6 zXci#{|A|qaB1^$};z;zS>r|illKV9Zv+ljL((G1^G5hy5n_GyA23{3UnNla60kchH zonaZKuPyZZg6<4V2jatIgU4;lR&srkGhiH>Z8B;GTf%d2FNX4sQHC~{L7R82FtJQIvVj}NfTol zC7HwPTG;R-n2IVo?A)tX8n5YuuO!WJDP_8cR`a8CeRvnRP8QZE?ZxxZ@)*XW;Twf%M2a-;tMXMdbn|Aw0Vy`hQWMs9mNu= zBIrv!WDf1K^RUPdDfBKsZ~L&rNKo97KM}*$VA#Pima<3x((eL$EH>oXD$_)c<2FlE z&OW1Y>Wye2@{Bfa$9EH=J?l5(gc2`Oe9clmsN;*jf2~M`)OL-1t-ZkitrSoR%|@41~gT+Ia*gPrG4 zE?e{OQdAr>h;dmOPTG?w;=tM(MYRebHG;4HXG*}sNhXsj6BF~h-I4EPew$S__vDy1 z|DiftMA?EeUR|=`GHXL^Q)CjzT(L_M;Ch&i4TLGu&(Mce)<6&eoAhIRL8=H$H}!^} zl7K4_7gtST;*4bM>&nNQW?LXhm!Rxq)Y&4C5fKk+ZrI_D=JUyhv0O&mTJ?@Lpr5hY zs_es@XvOVG$8ox) zZTu7<^Sw{v>1qsj36hHX_4+zRA}~xDPd)!lz6C(V4^xYm+z?QA{9Z-%6yY}z;N4k&bP@aMA)7?zQ>?i zsOKQ6MXAj`*DtVv@Q?(fw!Zz41OuE_qslM03e)S+z1jgS$A~?gsrO_3WmBauHQlV{ z@)5W4r*3;je6~K~cfD`y<4{F~F&le-{r>khQfghfJ=WEsbAq(Q{{IT#FW5DAyhOSEx z=p`!s6-S-{Gb$jf;~$M~HvM&Wpy3Q-0QzOQ{J!9-T>I{n#X;$BJ?gH@Sm%6yyjJXI z_5%pG4$#(QK4S1VU+W{r*YX@FP^QsVZT5&Uw9M#_EBt=dd$A@PfCAEp41o*)b1zEX za=!ee41}-jPsGluK)(Hh+x(Z)Z6!`>pN7K(3r$5c#Y{88^~R=$KrY&%rl08Y`~M17 zcTHe|Rg$Lv2vl#mVHrRh?trDM8pwsY@0>sz@dfN!HwQPKJQv!V)AiNo7O}q7jbec3 zF6Gr5tY^=+%mkTdLQ2ugM%Qi7bXytpmMLF#TK0ZRAhgVbdC3M#;K6RX^z*!PtB~8A zTVI#XIt-p?uB-9^W^1^-@_6T~ml+!bAiTpgvQ>y%=WVvP(GssZ-H?56j z@8`uL8!e4Cv+sEn2-493v?6Ar6FC{j+WIA^KXLa&6Tp3AzSp?a?3@iLW0g>iJJk>#dj#uZB|%?!cJV(E<> zLZEcqQLB)=l-hl~)1U!>6D~8VgZ6%-bFzRcvld$AL~8h^GZwEd6uat+{tQ6QmP7ci zRGd1GnkE}Vd@4t{UsfNE_ITzmA_WuUI(vs8rX$T=fhLv@dM;;~J557YuwKsK+AXPpO8&qjgXs+jbdFwpnSdMaS*q#Cr0g3F$%H zj*sxw77w2&o|2FLVdPM>Ewe^t!9~gSF>AcbiZgd8BjI8H8FYA{9veXahE-f)3KK2WwglP_2WQPat-J3JB-E*{sNN@?Xl{P zuC$l@Tpetv0I~!e0LSKt_rS4r$1+U;>oL4VirQ<6o3-mz8Z3I1(zwq*x2?0YX~Ip< zS%!PGQ4g1*sPZcupNB)xv)o8!S~G$DgIX#KM^@{gMqCE27#MQST^ zM;XW9EwuuMS|(fZ`^)qZU0RV~X4LpkB(yrTqX!J?IF?DnA$jT}a& zDbsHo@yqLa5#L^ptxK#5>ikxZ{=m+(-(&~=@rJ(gmLGs-D!;RW^Q=s&!~yv55dgcK z)B__8umEl?o;W)WJ8ysgE8REBE7dPCL#M>hO$kd?!|o*$i$#uV7OftS7rqkck?Qez z2FOpnSAPG#j4Ryk+Y$i3C^ydO*_i917zopTuBkvAuDk3CROkhnFajFr2iP9~%US8O z&PtuA^?YCdFb@_})109Y@h{7gzT_yS1n=Gj?uT%EiZaeiSbfBI(JI8mJ2-b19=xUFjR>y#cB7Bp=D3t@;OUc2tBqS+uBx!7; z_9f0;x(e8I-_;k3)wzpOVfRJyL>o^rkIOiVt0?^70kf7anLi=?T$G-=JCd@2t9Q?~jLOK@!FS*(Hw0SQ-Pa^+-*)s} zGAkk9$=kHt29m)l){#n?zVgHa|RB^Bmi?WD}Y z({D3zF|_%6@P*<g<|LBW2|NmdGlF=M#2#6mefZ<*^6Scto9gN6Ux)QtktG&CclG z@unLrlXKqSP1kA6*LSf-lxD&g$4J1Xj2+IF!X398xF?iD~?Nf9nGKq!J zmtARKOf#zJ_CSn03`|QrCvtGg1tmlP!ZTnCPJQ5NDyUpn@S&ovk`7Wk>@OA5zQ*_O z7nb-E0l)9JQqz=RhvC#HK(~m@0d4N;98QqVwacndSVOyDia|7DOdX52;ldLLe~_~M z7go009OMTSI+Wsg@HIhUIWgFNhot_=dXGUMN9_LqN*`r&ECvUGG;X|2TTfV48w&LG zAf91En0y=bU6W%C7>KQoy-CAz0e^0Juz4M;RG+0viNmJoRCSY|^XH<7i(V73?ajPvC$hK+oWwLZ^pFsLyeW|TtH_RZ zM5ryl7;T~%lf}fVYl|g2zTRx8yA(55Z;Pn0tf8P&7MFnz7`Rerw)!^~AS({co~8Mb zJH)LTfc|v3)m;@6KzBh#)?y3KUt$2TF&otlF=$e<+7G~NtsVG&%8ibX=!E~}A$9wj z2mDep5p{-U{s&I&30czIRk}Kx9>n_hEfyR3>r836-5Cua9jrj=_qQ2iS_2eGc2&mg z&z_IA!<7U%@6>^DDhwxaqr-!32XaRLgov*Bns(Bm`X|*>MT+yBN zTT2M5rAqzXZ+h9R>7=4Gg%D`f2s0;Q5w9J=PB|P`h+yp&7xjUv& z=>V?C604!YP+-n$$yNUDgy8Y+od0h74YkPDPaOsVOp-eTphPs7D8r7YPy4aw#oRyoOKH2c0nC^FJ?VC3>=$h9 zrw4H<{BL%!EpSfqN(u>N;kGCuB}Tb(xldXOnEpdAB&{)~(-&1go# z2_QSWe6|Co?yV0UyWikGduBqT(ls$>YfzXIoBcQ-EQgUT)a_R)LIhCxKV*MuY?@_~ z&eR$@cvSUTeIG+om@pAiVeI7(1_0Wj$i$p2^IQeAVbugR>rkf6sJ7lgX(b7u_puf^ z6g0~(egPQp$Gdw)a-IhD`fX05u~qw*U}|JYNbNLk=hc6U7VR|nUDXRI1w2`?=QMoH z{c+s>$sKQ%67~33@)9`aJpSfXTn<|?KqmzD0C~~BXV{_$?^O?J0NCd(Nr0Nm_&(j; zcBl!Hg9Fw)*?~Z;B$m0Oy#?hhr&;SDaalAx3E4zyz6guV1l7@)_n*eQU+HL3kJaGCdEzl%wsmMekGmuV~}CBN7kvFO{m|-sPCT; z&CRRu+MfNY@8L^Kp!TAEij;%t1rg31zI(%Px#d@*Q_@nB5+G4TS_R?0ZOX!qMe9g2 zsQ&_OgJbOZ-yR0*_N>SH%=K33S{BjTeV_5Ob_!X;^|Z)V9zQR0;Eh&sdK{>642a!0 z-8}nW!tDs`2AxSpG(@<&oazCx9>qMX=1?`~oePm$}(wtfDOR1Sc>bKHki4g72& zj`?UL?PQ2kRyK9zkM&Pf$I3JNirVQF4M&r0iQLNg%`u1U7>v2!BJ^|>VRu|Oj6GD_ zaBnBy{KK#nM#f?)Q$77e>ooN~h^41eaNWt;^NP3D3PaDFOSTn|XQxG=LQ+9K&_ zWP>TD=#g{hMj^}@$D$9sr?m_q#W1DF^m$qJNvl*W3z-wy4Txn^y5uzQ@Wh;a z0^}{a*?>9i!JTAH0vo5(A|eCY>wtOBWFJ~p*m;2p;r~tm0eOvVRI#kIRspdVFT1pW zxw4`b+_*1x)5?Xvl@jGyM zxR7+l^SS~eCd9rzDGt(-Wpt3WD9`{RcwL>(&gFRA9M}L07;p#}@u|)a+eWy2_#ok? zbGFr|s9_ryime)|+?}3!PuXomQ)U8`UCFbk00SD&K2yyEqK>-%IIu3fyD{<3)i6W> zi^$FBG;3v`3dD5%4;y#Y6e7tO_0X%8W{QFBcKv(_u⪙xplFvXM0?}TQUha=iM)P z#kdj^hu{OQ@f={cHkXPX%jleXU&N#Q>m9k2tXd=^|I1Ra>%~{$GDc>=G`i^POGR_6 zWVC}h-Uw`1GT|D^*S}EBbsQs!+Ubi*;6O6S|7vvZPY-jR$rQu4Q|;6;#i<+?$jSLN z>+Nd+B^#;Nx0Ijom4CNl$bU>_QOTXNBy&-M4WG$Ykn@fsPFb29$KGaz`P*Te?USgs z@xX%!WwIfoK8@TZfC`K`4gyBE46guG-o|^Q0wOMjyH7!){24!(?rfuNXwgVaE7(P73b>L{45R)Hs#D6=)ycx?pL%yTx-U;-e3i~s} zE_HAVEC0@4?041U{rjyqP+>+O+%}C?Rn1IvovJtdZ)qv{gn?w}pTw^NDg%f+9q@E^ zEPst6`aEMdu&f1FCkv&#neb^ldmp5sT-2ld8m+C_1sE3sO9~gtEnROYQK}*VEJi^Y zYWx|NSr0{A+}L#pUrxc8u}}nFcBf4H*h_I+&DhdVbb2Pn4L5|w#B(G%_phmdx(*wr zQh*%~HI-X&*Q|T+ern3A`&|*p2fbK7jjDCN57|ecJ1z;pHYA{eah2(S?>%|+EU@)P z#`>7$dp^ByFiu8Z4T2VcEe0rUskJHYuc;>5iq8%Gj1q4)qZ<|ZgR=a{hx>FBU&rh( zk4~&A&Gj2;?mAPI0z5kOP&MU19yN~8cr38sGyn=@E(0MnusM&D$oB~jz?bw=rnK4yt$8JJwPF+$1#OoksEOSo^@-K)8mH@R8%P+s)26Y8(7 zV)qzWh@QO#HEsfG0=EY%9Y>LATysmyaoG&f!vbb)HB=H^R#t}RagB6}dY{ASETf>% zb7~_!TmFB|z>1fDPn(Yd|qjku-Op5 zOyn~A!{H-#4B4GL;FHh%hsOV;Cr14PbF?}`^|nem#Lk^y@4NJ88xTFnDE{BZCa%BM^8cly^XbvIja#d_w-73 z0FU+9GcVV$Sr>JQwWQ>l@Gc-w{D)ih7SRJJsIL?!5IicRmU5fwvE$I%0Y8K0z&B0WUO9b|792ekLa!MU1KEpRrvg;^un= zI4J))Hvo95#BZC$0Cv}!FX_N1%Fjt1U*_-UpNA{tM{~SCOR3B%FS%dN^1g_FrU&H zg{97uh35A+$hB-7Zf4bb?-MZYrtIsg9CWa&i5^+2nj;v+b52iY_b~4B&RmY5w=H8} z3v|rn6Om(?3?n|iI<}nA8bB|^czkBl#P>Fhd9}Tan@mj@w2(T9)X#2LUKBy` zir$H0F9w1hi2>YACdKMj#JtN>X6S5D)Mw#1c4t4L6Wxwa1cIL|d_{70u1_N3N^V$> z*B%e2F!nH?O_(QOo-u3d1;K-2TG%bhy-Q$s+Nc2saoHI65t0v*FXJsDoV}O+Ec4GW zO|q*WpUI!y-)Cl&O+{qfibYnuSMy1bg*D4 z@kdm%+U*G_K_ctUi+>=9fbflVPZRm=eyK0X@6xmNCIp_0kmILE+n!nHL~vfYpHXdX z19z77q{&IX2oM%8xTs1u2x{FzJDw8ol^(V>pfqxkYlCBM!L7`0`U;~R+;+gkJHdsY zDqnf*hbOCNT4JCiuaK+MG;Mntn=0D)d7O+t?#5aHYzd-d=(mpaNYcvaCWso98D-TSA*$nW)+?DQ`dJf=MehXzKLXVSrHVblmG26pY}nlVIFEUCLy#%`sa}KX*ILeoFp`|j6XzFV(IE}X z?HK5NF(l>R5E%{EcEpyJmMOuemBZEch!6WT4iGzEIlopHIM<@J0+PA19FhIuGEr-J-$o7t&iMVkyp5cqI8ifA{Tg_r`l zO={Dz$E(MPV~91FMGhQ9vdG$yE3nD{Pu43@x3K0+mxw7i1W_}vc2v0_BjS~X|Gt3$Daoj zkz9z%Ykus#c~LZSV@cc>oMQP_)5PS~AC$aL@cbApP^NY{MLS61*1!`Ycz4;r&sm71 znzuRJ)-z9De|qMsU*W5eV`mY_`Ie`h<4y79?{Bj1a01ld=UA^y`!0Ig#OT39*qFoP zwk2h&Mi}MDgAlM-#p%-a0p^t6kB}BF3{otTw&;2w4bvh1$_lC){*HcfK=S+;hy>y1 zG$>|V3nad5G(Mg--Bp-yKZJ+|`VPHOV+vjqykjVL|&l~)kf+`Cf|n{RrTP`|mQ#m%94qfTHe z+OgS|5 zBllg={d@HKD}qu++-0Yo528${eMY zm5-A-ih>T4v>R?T`^|tf#ltUKY?BkNP6VzW&xiKu8v0hT!}nJB=_~Wg(#wm#ll-K2AK@KEhKIm^0`~#l8JY`jVBs<^{z3-hV#t*xYamRs5ZrRk2ExupvSlkpd zzc3x#z=VXRRGdFO+?kU#23dcPR;sq%T>rWcNz69p`(#^}F4lj80-egeSU>~USdp{P z>I*F7-_-eG^09bv=m-nfP&^mjJ68MSaS2UtPZQx!%B{TIohpw``e0jiwX^GbWxltz z=qPo77DR@PxkMR(!{-7@&{8+s9*aBXPr9v6sofZc`c1q1{f~||wL~>X1T@$;>D$iEQ9e{89b(~S&iYFEyTn5u4SWiI6MWt`PYr_D6YWL+yDkrp zyxjBbqToL+E0%4by>=i1)T%Mm3-he-x^r*%#?!HjNZyg z|ABK?F8Yvrw6fmn0cyN)a&eivuJR%22y=gVH*+oDaEpMFJbiwC{#Rhi{XfyEeCSD4 zvn3Jp?c}i3I6im|f%}G%?D|_OvD@Qb+Q`(ws_nE2nExh1+ zG>G0FB#*vOg#6G7z&->|g{SAe8s1p=@;eq`!~e(JTZYB4b={(!kPsnQ2yOuaBxrD# zAR#z}0Kwg*@x~zu8Z1D7pdCnXcbDMqZjHNJLqm6;V()jq-@ZTYdG4Qko>PCiySl2@ zs;VV(jy2~PMZA;1jK_X@K?joY5{E^1U~G;2bA3?DT`Y60+oPRcJ_zm>)2r`wvIoO- zu3;W#8 zehEv^eioVO8^0YTG4y;t?a`+%KGU>4-a%#1Dqibak6(aO7cu+rCkASD*=jN^DHU;v z-k*`Vt-FZyR{ntB$G`ZfT!=1S{u~qTZW|jbH$dN^qMC=Y@HfqVgpXC`j}O%GNfDKF z7>cE(As^txW0SnTBr5)thD6|tcfDXxsyZG|TkyKA-p{TI$2dH@Le|^XFl;sb%YV=q z?P`yjKXF!Tiy)u!t;@2rYm^nqV3aVuqw|6&0%D`ZQiYx2qM01_mXcsnP3H7T=0IvP__LKa%6y~NG zDfW=}8u1=H8|tm_&tITnx_^=;{@V5NZ`|`ge;51@k_YV3_$#OTpEL9Sj{5)mr@#7t zhRDBuwlMvx`GE`x|0fs(Kob9=nfm7<|34MykCS+^gPLa@ilGy(T>p45H9w>?bPG3# zZytF)I~0{3?EWOA1i1HZ?|Jw3Mv3RYuphd$ELU;)4c$F^3{+lZKQ+)%?XZu3@*Qy2 z5(3EPUAS7EUm(w#rqC4h^4VB$GuXllI-Q@#r8NXzA&$pIVKKZpMFPZDEcVw660df@ zC{#BYY#@WbrxKtLVnyNzOh*NCZ4@9<#Hf_Kb`$xa9=C&{U~p=3il^44_I7ODsu$5u zWyz2yMd3bH5lA`Z$l6A?T$sEZ&xR+q?UctSJw$@|Ul*GG$W zJ#n)sGhBxoSxkY!o6akjU)bD0yiF(v6$-KYNY%t=_6K_pxn%iU*USYG73sHWBEy=Q z0a4HM5X*Q^<-MD|NuE*rs* z9B)j|z38%p1me|RKjCn!YKnf#O!Weh@&b}_`CUMQugsBH-m^=Vn_V)$VE^|WART?V znZao&Jki9>+uHZV^-^CwG`l(`JX>t$Aq2YB0+Asm%YqhM<_N4!AZxwV=$l8vF*;ys zL>q04%EbrddH*N zpAk-wb+luY^m@n|EaxHbEZ-;t>#7c*O5vQn_NICY%$)vB?BH!EuG&1)sS+Xm#oJ3g ze0#qyl<}u+SHHISphi*7Uu(a7G~$!wHzvN&nRei6B3KFW^H6e>`Z|o!Sq?5tQb{J@ zn|Mv>`&d~dkJ|D_*2iASUp~o$-bewhImf4DS#vUvIrofZ_xVdPFE@AIng<_ro*UN2 z0%0Z#eUR1<+I;HX-b{`!u01+%7g4I-=Dm7IHp zM|)V#HKefANfG?+cTq|0T5AP;q%1ctl>?A(l%bf8w*_#U^##3 zX_Izb`~IVnUuVTp10#goT65&vbt|c@sy?#Fs9aRE?Sc8w^$updzI{@xCkx#ERIrSO zAV8z^i%~k}fVLX7w+BQcg*_Bgd&bez-22`uy6LiAlzraRA&%DGUrmj9kZ|P#QOsk_ zKXQ~kzOJ=aOc0L86R0II`aq^2k15~2#@<;-uz=$qr{71|#+9OwULTX7@Ae5hB0VeQ ziU_RrnbPPzA^DXO5!&sXVPoYUoKFs-F~zry%Qs#aC*1HFH+fZ$?Upz|f1 zOy1HIF%=r%!(wI z_Fx07i0ZHJ#6kTo0ng9`Ei$Dg5^w5w_a3}o88YZ)j;y&}Fi^-D>`0&h`Uh6sf`Mio zb1h%1k=HX9o0_Ym7zFjzPg~Z&qU)B0E?$vZ13Lc6D?V^_${4d?M>D&S@4k z+v8L^v_y53saaPjk0|GUp#?C}rhGe#wv`pZPUeeOYKVnIzm0-t?>s)93XVIBRN`IQ z9TR&yI1I&ZH)5ISV(e>jtxSuYe(B;-xBG}xy&p=as5=P+iu@MEsacK|jY zlxjBcO5d^iJhX|&uh1C%cp;9blau;YMxWSK+4EKGJBxyjD4U|^WL>whZ)_4S)wE9+ z?!U7J2l$;$dUd^bX^Y?^?$t@U-YOMnwGUDWpig4qD;scneWBeJerDFekg8fz+iY^2 zCkCtblkN?A1!j%2WtzgCVp_)ZSM0(emU%)zBxl|A3cQ{2a#B0}S(2@K5ip((mAkzl z&!up@6p1fQLwwqGyri>N@q$sN(s?Ux9mrXnx}9k>sRSIpT28AVLYg;R_2(}OX_D42 z4Z~;Khu)n<6nF{nDK?GPuENSGXq55Dg2;D2{Y^0o@@b#kquhgkKC4|zs_{*fb&0*b zHwRuHTX4Yck5MsAZwya~O(5-PDUt%#_;#wsv*`}oLm>E%KkQZUV2Gvg4gT$j33B-U z5>3-7CWVCjH4{!_+#Yggp@XwS=&G9u6f=crKwghhXh{N5F(zh`%pWxvh z$Mww|l9p59<2E?1c5XuV=|R~6{|zAl{j9$^J^JHg;B6;xbEQtYB9<-UKz)aw`F3gI zKijZg5ihTA6(nlkar&Vx_gx2#e;HqOBNkrh;xIbU8QZ^%u!Su^uQro@vj8wejp^|G z2>>JGzn5#YJ+mCKw;%>J-YUxdXA_45wx18#|Cnz62WI;ho;8_NXOI3}T&tXl#?)>M1TaU~|w14uMZZZGs$g_Lpfd7zi|BUco*Ys?icpH(9 z_|cr176fq2{_8eGUT7Nlq=a#rZV4d>{<$#d?BRK+37;=Q)?FOrbMId@PAk9KuYUMb z`OUv>*4aT!WXt(KRd5*x8v9=cJx>jk>P(=f^uOK@2=sBCqn*+JzomdczkyNn_U8kD z0^YY8v!?;sfcfO)=;%2R&OJLjn*`RK;^pDKQ7N2Q(Kfj~5Qph6+@P*~NNlSr8iK&4 z{6ht+aYSxHlYw07G9CM4ve$-qUc(NV7cgSfgo=UU?O6xdf2ZeeO21U6F@lurT?SL< z@o4V&qOLISh{GU#tQc#O81{`kA(=gdW-utb#_glU#Bl zB5*XrMvS+zstN$;?^OwuM<9axCXk48+{aV`mLZUZOkfEf=xv&RKYl?viClnF#BBX) zHp%HOh~@ykz@Wsk`-a*KF$nWsM+t*m=%d zXUP)|H8VsE8S;CfB48@$PTSEP0=>giecj_Oh_8j$&EQIa2 z8hA_agpBN6yjm>T^nqHW>%LrsUR!Z8PV=OS$*Ovg*uYzi=KeQT0 z#{@7b*Tho$=HTE&8YVYu zH;lD^r}#^@M28tsp4B6KS~$`?Z;KeuFEATW_<6|M*|oUwE8=LMolFhx9$;DV9Ht{I zA~HAYGBkTlM3S*)ty<78VL<~SlCXF&2m&d!9!@^L&~eU;1hv?eD5e3_ou`0uFlbmn z0@0J9U(WCT%{Sy=hE1wP!kGD{Y0^PUSj9n4Oa-33QMTUKQIr@7HtE+N(VrLY@9h4v z#xR8QG{Tl7R`Jt)${?D~>dv$lZ3>RCF^lr8t0c>Z{EPwJO6_=f@>y-GZQg?Xl#+P; z?+t0tgu%@cCpy2#QCtFVO6Fp1Sqd(5!991Le5%A#CjV?j6U8f^p@qqMZ}}@iytY`t9^}S(ScubH2v-7 zabht|3t`l1nk{>MM|pM+emg!_qWKuh58i;0m632#7q9JdvlZKrFDC;L z=St)_PMEWil;;sNJl+ z^Y=%>4Mkr0P*x0MNx^VSk^=k{bH~Kfg=$A+O36=wZa5J+B_xE-T(T+rsX53$lAIha zj@H49f959xT2jjR4q3Rl%2w-YdJY(xChxum>V6Cfonxit{oNAe@{t0YjyOnZjF>&S zYanQux9+4Q{6{pq&^?K@qNgV-$KDG4t;Xa$9V}Q?;i1v^l5mdS{gmN923pNt^npN= zz;N!M3=9=5JSDo*ctqDSPR(}Do?%J1o0D?fHFm%>|5Qd}R)x=jb>*b@DuIDcw+hH2 zxi0Z{_x_>$)7w8{#h8W#RdvEB@&1WY_x7OB66Wx$2!X$|h@6YTXiDJxLLhgY4 z0lF0t0BrCV?P6iC($KPvf5!LSnev(LD`y4bI#8no1xIxfNq!7ZGIo#*;`Xp~uQ=KqF5xg=eXdazfH#wp?)A8Dd>W5*ig!_1%Y(9` zV@dR5nJ}CCH$Y46;8PCYllWIyB8Gbc>cQfqp!wSg2!1=Er+iDFoaW>q8ePrZ-;bue))(%CKIVnke&sdhIyis-nJ|R>-&O;6j9k~_MsX{ z5Xt9NJG3n@ErHUNW>{3rtxiJA@?R=rf)apw{rzMY>0Q)miuifQ$>?|h4pE6nTcVd( zguR4a@G=Fh8XG8s-8{)&m@0!LpYJ_W`ZZThvqZ! zi2`UjgoqB}7)`r2vE1-L{MhNQ1uSi2zA+fsTBjU@Gw4tnwJW43ee+c>+1;kqjxSd}oXWbuN$6!|)>IOLZUjR>g zxa}@&r=o6Iv@Hq>J2O= zdYx~iu%ZE#fMKp_qOMy6K00`F4F2jvhL`4DP)%tIz^%36ecTbcdcSpOnVpZ#4z0er zQ8*KGy@)|kKu8^EkA#Q>EL8b zXO_J42^;GpA{}T9>4#1U%lb4YWAOW>uwri%hI}Qk{0fMQp@g!N&XQ?B{$5xBQCYm^ z217t!V-gP!@lb+6!Q?0?5pvBQoyr?^fj6c3@A+Ug+${NiO1#fr(yQupv7qteRBEa!vJLbjt1}MyM z{7f=B?w$vrH{>VcWBenWdROk4Z)V8iAkXvFb$eU-%~;vA*?q=%t0>}>dOAl6v8RDz zx(+Kwbv29}ahD^#7m><(GH26S>*7S7Yp%JyvfiIa9`gQ(Y{p%zTcT>-{?(|gp$~(+y|(Lcy$g~tIX?dU$am~uP4LPz zd65~4K3e%w4fuh>ixvhEvnaGVGx8VEe%I3(<(uZK05%&}^&WX*QM`jpktScn3grz9 zw|Kzk6^-|aiCT8iyM?@b7L+KX9-Kw>tl0+)W?hyhuit(6A_?E?{E&*P4Ks198cc`>Z7!kNKum0FCYStY^&~ za$=BwCj+fJ8clXKl{TvWgFfldKP2G-L@`4E8pIEkaf)|(5nEj|9#CgYt!_k z@u@Nv_KyaKg)$7Q-QeDup%Lb6cq;dH(#mYD@lM4Cv}lK#%Fdei2@ZS{@y^~}kPVp^ z%U*`#4VXx_KlqY^pPeqe84*5 zzJ$ZPitnr)AALag#z%@y-vAbchu<+8uVFrOE%=MbK!rqXzWJBo$Rhij+W7mu-UYgC zRKE~U5&Uf)&n|u-aZVbgbA)*DP_gCXK-9GZ;7PnH1L*QJMU#-NnMu@1jbLmtCKFO8 z=PML0q;#{)Emc9ASBcl$SLtBwu&I#L-Qw)WbHOHk_w-@q(!HXyy_6XjM@_`3P>x(% zhDGtw_2N)%Y(-#{*vPR4#loe!W2>;)5_0$AfGr~)S)O3cUk$YN%avSrD}8> zcV~I^!1!8psxOM}4bVXxIL$wCUfXql*P=DgQR59)xS4>@gP*Z{seiiJ@#Q9Hn;D__ zP)7!%xwOXFfh7oUhDh&cKjbIN6OM5cOvDATvDtzvGQ-;JDSh7_N2{5GQjqhDJv7&` z2gkLYmYKp0+WNvEX}zN|LYG4n#f1TKnMqvT+0r?D^8J1w&~f-|`jmXj%-S%UD1NxH zcp=O?nUxiB;J6WrIGF&MXqy)VOL?S6k(!$H^uy;?F>YPo%p|n%;>E$@?~)lg55pQh zLO+|SLA`+>;z`fOI@`&YiL?@v1+Rwm3^H`13)tTl5DA(aK;bz)8Cr zc|@`@7U<;z=RsGy6l7;_0%i+C7e~GZ12YH3b}T;;8v%?4lg1#(o*`u!kFle*2e)s< z7(-HDVL{ET!Fd_rZ!JWZE)ye!XyIbYaFlqn!^I9|vQjhZ?(DP*npgBsu6Ks~;B zlT;==i`{CjpkY7AGQR1NuP!@V4bwD_#d<^Uq^YIB9Q#tsJgQ^4jHZ_B0~{W5@p!2wZ_X(CIj*{EjFQz*ML#(6X}Vmzwg-axBbNU~n&a`PuGK?Q{Wj{?nl zauVYe?V{P#YXc*G)=woQAkY)eMKo&nM*YfNeyG^ufIyK|`?Xz;#lk?PwrI)K`!QppGidA?!d_^LDP7l=o5|mc z!9R6HiZ4Ku`X=@5dGX1)cBHG?v?RNk1aZJJ5v%A;~V-T6LrzvO4LQDV);O*Ap zLvr$FPOJP=1jX3!ozMFm(zV$>vK1z-v?Tj-#=8y~T=$;VHkeW<5CyI+k>SzSb4m)% z{288!;ZGza-uasEBAXY687k``kBiG$h~DQWSGBFIOxj`yGqw&pv3C4ZX`0JJYxq}D zfHDjxcIy4q?JjYOZk%|gDjqz|4IIwG<;r8Go=`~i{@z5<%GNFs-5gufTU;7L3<=W` z>6C9@sUfscz_|F=wk5mrdYqd?da4d|as1*xJ8wV!jm3ig4=m#UjoQ{czD)>Q`r0s& zJTVSGirA$3emFGkIJ~mmJ>FsE+cm3I@mGouQ@<}=_SZVnsWcL%-o=XYfI5Fx*OaV$ zf>NH`zod*DaDntbTh^_m-_M*ie)mPWczfU8m4n^G==89SM(3j$ckuqmRJQPWQTrIW zay@^&v@B~CtfYHMb=%$ihp5+EP3)pxq*0oLIQD4A*pT-DqEUj4{6I2^wdj4EZ=M>C zQ}37%Ka{@_Y??D%?~Kk-y^FYFpse(>b9tA+S#z7o(iZ-K6Uh~ch;no|P5L77^6%J+ z)Gv6Xv+3@Z$0uir$I+-o{dwqs;yH2TA!?4AcR&9pTi@kcJt>{EA0MBtFv4$6-$$gH z80YsUSom!2BX|T<^*hc2w-Fs&07sjVm@h76$#=kPJy3zT`9Sh#3r?u>*xNK%TC}Dd zPS<8h>MvyzO(FRVJ0=c}q(Si1)U-7-*}4hghxdd6yy9x<9G$;zIXc!MULK-f^P(?lW0 z7zXkr0-EIdqS>7O^*RZgcB99yD>Au&NgKC>EO>4Dp_X)-)Mu7sr%9#ETjoh&p|*RHqg*C6eu0AoAM?%sk`^A}Ty1za z-X-zCWK33$&6bs?pe4wJta&6_3UW0;$amQp+rzDRZHJejS-z214bW(sVVysuY{ZK; zE$d{#Dy_Y%-?aW6kfH23a0kwEbef0ldM2Y`42G&soYjVQC=6T2`UO1Z0f%$^?mpw20efWNi3cTG6k zQ}H2SL1=Ozq0*Wv;W-~+6TDu%?$rEiaGC+SIk)<{3xznhIHf%0&_fPTEZ^u17sHT8 zxj*jEbt%&CWa3>R;D!RcVdYB7Qh09hY>3crU+~b%o#SFCc|d71RbZ3t2ZqZ3%;t2 znLrJuxVV2;t(07UoanngFsKqI4^>cNR20bDEce0eT+21K=4lK0uFmYlThu+j)tD+R zE>ZTe>bY}Nt;6j3KUhFKB#I)Yl2JLc(O54_gS(JnP(kdbVW@7;cUw|&p>?uNhj|4v zs)p(JN`1jE{9h}ck5zXH!C!7TM5Ne@C+pwyYdm{xn)5MUKn7Z5_c=FGN4nZtGr?S> z`FWqb5EeL)o#p9W!>Bi+^ycspaR4I{8=7zcEhaG!6|(XBsve{#%4KrFPlEjQKVplLA|1Y5wvhYA7| zj|Lt=*?c0krW-Zsyy|W5)h>$Jck#Ir8sC55<)N89c3#svMIEWgDb!rhM>}WRT=-q| zsyQ#RdN`RaVjNQ{dtc4zP@`3#$CH|T{p3`>p1b+42w*B0?&rCVZNDmMladJTlP916 z6;G_`Lk;_SI_+nLzTRps8mC8%?P>33cHIZkh`vGt19E8wx}^0B9ml(=k>NJ#&jdW`5<*}Hwm z^~h!$tGzUWxD0TVZ>LIQIs2QA`JHAby|}j!69EoC13QU;et(~UZcbamz`l9&>3Zv_ z5D&+C)5pRRm#ov=;~2^3B|5Gx!`U_C%U_=oDqx|t4wxNZ>)`&x)tn3pn8wYQ>->dd z>)U4cYH_DbesVi-BAA&8Ln1;oQLc9Vm6Pz~^rwr}4D9EF=B~Oy?b4z8s^uG5GsG(R zxv8HneC~q8^M}``Av&L9Vlc&yt{F9yeLJqJGwtC0e#^__{DgPA4b#_lo11UOGltIR zgip)^sIRs9oo1j@^`8_sKp>^?%Q=;oWA?oD{iHTe8lv{-KdEt zKHjY=S7K_6)y?NyV~2uoPmsw`Xx@`Anq8gKMShn?$Lr*+2CCIc(Ce+CqN@8l9^KX8 z#iLZ{y8x!+e8{LnnI+hJ~IU??IbiEFu* zIO_@IjHU2pFKSHff-Th%-VcRK)lG)2D%ic5j30zYx@?jA!vhz#Yfhsaykxz!ul1*iiDF?1&Guleaur}y#$$Ts7XL?3oBq;hwdjkL<15-j z2V(=(nz}oUaU$T^`1drJL6cm(b_|@y-;MH}4Mij+nIO5=x?*JF3HGc76hD^sN}|mu z-MW-WgjCkH8}lj|XfkAbUK-fyEVNv7(H)Qznnfkc%=1Bhw7mzhJ@ z^vT89?>F#%pQS@w&67M(qnrpeL!pJU~D~*DCakHww@3S&h z%ouyR;4>;ei&qa@dEbSZw~^x2r({4e&del?y2wU5Ia+mc4SufGN5kl%d22ses6Tce z>t06Ie>hw$t1aO@%KM!@GE$8^fnRGV>C?8Sh7qHOl3OdKDPzOFCcZ8!_fT=&V71&8 zeArTdd9b6IJ)1|DrtIUbo8P!^@71y&Dg^EM_NeUT)n!61B!b*ev$K)i+e2c!gICyQ z{ec$qlyB2z)l0{X?&d{L`%0A5o5@p1=Dg#Sq_WuQZdYgHW$~(nOVi4%SHQ=!jteeM z>;q_?pAKXzWb5I6s&>=5BGgS@VRLqXJ&Gc`kx8;KPrhFf4%_&sV|c!?O~=&<<6%=L z*XgRsKfK_n8MszXQ7wKP0#OQUa^8)%(?Q339R6(ix(^n)zODv*NNn@4*GvZRl$<9d zPF(@-(E9P!jyewWA>DsC)SKjG)KkG9PA+|m2%bS19vdi^xIl%Qg!XrKJ$V7CE^y*x7qN*xjX$wATRKx{q%j*4ye+Pu~t z;Slz{8Dp;YZg4`~^U^r-FAPQ~pNCGQNeb($F{5$z@Y!z&En;O$U-qG>5hY+B!$CRH zj(+5o-YTyZeVYu5o-V<$*?Vld#n}u!{w30}dhxh7o~%}nPmFh=w!*)g3ee2h$9VKb zMoEti{l`GWZ>2lji~4$EFFQN(`pe_Db<&J5W(%*A#b>0cQ}({=zCNnWx@s<++*x&} zL=~WD41K1aDl(0bR)wQukNCU#ZFf`EeU;&&Ig_|<{0@uuLF64Ejw zFYSQSr8C}+OTlJqzP5hVai$0^Iz?T`_R`0*M%Urn(1`2){ESbGqlpOqOED0|+0sf} zo0zxfJh&NHWVr0f#Q#+OoEyAdcC;NEo-`u}=3^XN7Gh9*JX6Ke=Uh5lxF4>)DzZyt z2IHGrq;LCC@mE>G%`7<$S$~*<&I{W1T7`K@_52=g?fA z8kWKrfBr*27OI%vJo)&mA|!JRad+2CJO~K-E30dr96tJpFEBk4POE4a$O}rWVuXPY zId!{oMEl4EWvRW_Bkdk|3NS|lBB?@u91TQ=(kCqpdQ2NMWe?u754mrL?#aT2JSAhk z%$-=9Pz(h;;l})VP_^lcCAw+LbQV2 zXpHyxvVq=t#Z%9I*Vl@e?&RC{($MM$nXxifzpMNaJwzYOUd&yl0_n$j)2%Z+iBH=m zH&RN5Pe6h<-*%d-LszkM{&Ab>$|~8))4GOfR^VC^>3J{zLhx{&$rCuN!yK{|$ad=!;zQ_N?&xN9^qngByg1SIlB+>(ZGGA6bTQzE z+VHh%OmR>8$nVq5@75}?epxh{-&3-A&%q+%wp!OXDB2y5(d{(%RXyI>GqPREF1O3X z=YvPngC*U{gf5RoZ4dXH^>-X@C+z4bY^vENiHbU} zHH=Fd-HR&fG(!oW6YjK@m2y;d|7nww0w=$`-fNuqPtUDt(u9#|po^F6T!ES2hkH48`^f4tIpvmGnBDt)7P8pQ5m7bfgz zYg}`qto7*{EYLNyW#C%#aa{9Hurt*m;ni~67vs(wLFWZMXTC=qs~QurO_t}Bs~fqH z93%FE#wh`GOtG7Cy{C%8JjOAt5<_2REf}pvE|fWh=BT0RBWzMM_}Bh44(z?)~m~;{Vq)}nN3Va zEwX$LQ?yl7S*64`$E1h)_4kfrSZC^$f6v%*Y)82rvPN#&Pv0oHEVIX8!ex_OW*OQ- zQV%Ce24ba%8PkMFtlCrkV{3^Q|{AhN1My4=xjGLefJesVm z+nJRF;TD5devmziZ6pq9L~8Xtda-M?iF^bp;{ z;#cYGJDO?Cr-DOA*97BV2UBwm_IW@cO9rn=pBlv{Dcm*(!siwrEEng%-xdMC5z{WI z+R|=>)QEf@mMCGnsF7#wvZ`iKCrgsS&~P>%i)i!n)q4rukMp)ssGjvP@MCGWx$2NO ztGVc;;xwN|D}yPSve3)QV_de8)WLJD>Q0VI-D}3D`Q(oWAL9QwYqw#ach2Pa@|egM zBCjArBPZ&$ioIOD3m%(%f3$4VOxEmDPGQ#7{Uf=dzgi&s-Gb;&uMV_ajQq;EaPQDm z^}wlE9%pA>HIPJXOKgGL$lt_fa$j@C@k6W2bw*ILdttee#xKBf%gm7qs>FpR37M`_ zAB5m2xG&Y>GTbQTa~{kqp8N5uoy;&Nrd5j&7T>^%4;5-?Xi=YV`Q{x;B<`I@;^cK9 zdWjmT@Zs72U;!8Evn$cj$E#_`tGrwfv#!IC$PnuK8_GE)e1KL)ya+a)b~&(j$bh5t zH7tR8i_pg|s=RN*+)g$*`q_`yS8dpI%>&H!rb|nIcuPIfVUk;dOKAmhkgseDcFUg- z@l;n+%*y@Fui7UM@4Y-r>((3Z3bjF10C`USL_J=s0MI@>3Ao7#<;7awI^V5EPtdU< zY|Dcmb!Ib$ChcK$rzxzC4lYu;$VEtBHL++sH-GQs)!K!f_F9sf?uB6kYgsaXlk1nM zBAhoPvJg@DawAtCP8vVucpDLZj@ey<8^!bCms&S{n!ggnd?lsEyU|yJ@uZCEvME&Q zA%GoO>(e%e&thp^)g*8X*-!Xs;M1q?_P9Viruj zRjAd#2ljply<)pnG4GvHO*FmO_AUh#rL#1AOb2~2IH|qdJOuTH#HNTDZVmaJE(NnU zlosV|=q7|PgGqd^jtj1!lrE|8!U&=BBC-3)@q(*+>rsjx2dk}L$DBZtV_*^rn#azf z?r7)b;>bJEr6U}rRV{-V8}CLE4*R{HFO(uIemXDr-iZOpkSDZgGitZ9~7rhQy%`KDW@1^c6s|+vXEPy=}=MO zU2(5bcKf5+a#@bI;_Xu+5<=H>SCbeohcV+ey7K)o|C;?74N1?N2!qHZ@3eDiNM)WLJ=3SM4 zBHjJn_6?68ML(q+le}J@-%iRXMxzi|VWPzTfZW{eTLp&BsrZ|dNcy`aQp%=(?*aMK z1M!7_Z-0OFuQKWW?oqxWDlAnYH@h?ey?);yLMgO7QpZD*ak zEd(L!;XXP16%(wqI4a=5bU*Fc^8O6q&>45ZPRHBu;RA|Ti=}&&1tQwk!d4-cqdqy< zR?bLxkipybbEY1>IvGEfYWad8qBt2Zr9?hiQ}&N_Ku81##0t8NbQk9$cZ4B1t{2nF z?Tam4v~lKMUvTr5XTF#%dK*si@}jY^>oT|MH`5D__j|7*6vJ~^a-TT%S6M7%2<%Fg zPuK5M_N%YlNJUalZZ2gJb}Ex=n%Z+a-JClUam$)x1I2LP`>UAKuZ=7pLsl2l9Et7< z^~S1Er4d%!q`n)=wwd_j*_hY(T7!@$c*DFYdNaq~joof?bQv_&VJ%(kQ(+b7DGRnW zhWW?1>OVDQXHQrS&z~m}9&Qlqy8DY^Kv4xPl0!J!tGYnr1n>Hd`^2G09#*8Pig(fGl|}2@iPkWmjLSa~;lNUsH@c3G?H8Oa@CKnS*6Mg;##f`9*jh)Ph0Q zA84uZj5X&Qv?O&@LIr_)o)x%lCjJ|hkck~iF2)5haEQ(Cmt&o4y=GVygP2Sn6tJOg z9k0xeLTI^R?jnn^gy~>jgiw&;S8^Js50@kY{0wVLS;9u_;p(S9G_yLA;%2aj>|9d$ z_b%t0D-<+3a<@x|ig!uL-s?C{`xNY~`@q?Cenv!Rel+>D z<_}%b;8x5u6Q9h}GJv&`zc#RF(hjPmwRa?HL8*6P)_|#QB$`Z7X z^U2rKOFF;#%)-9U(%`ySGOtO4L6BF`Yb?Ow;oaQ8;w9n=API5FMErcZbHSW)RGu@- z(#}M;0b@TifzW24zu9YZ=LWVqZ)TFy=Oc^iXrI3@94+{Ck%%)JFg)5hYe#;5`5sF) zu0Zj@Mkh8t`RTm8mdMt5M;NcC!jp%pud0^CmGNijK7>4unwq1%E9bPIZn0PtbC#a4 z4foEUKZdYI-15er?T-wR?>VRl7AB|KNb&t6483Hfr9=U~DLRYxV6%~p0Jpe^kvbe{Asb(dS8jpOZYjzE_ zb-#8CrA%u+E3y{{n+?@pWqt_xlztwrnB4P?pRh{s!6dNs;goPwIv#~NoT-Dw{D6U9 z$ktRVc%hV_6X%(SXR&pmY>4H1&R*gA-kbk^MJNrf!{FOGJkJcD$miH=JnpE|{ z?Y8rX5GBd_CMTzG?2eYtPpk5zp zKP7U7mpfC-5`g@9JBW-vG#v=2lD=Mxd=!mpWC5>ME!E&he=*79y|zux-|_wx(*<`< zb+2H0Zd5SWo-%Y|{cAM4M*TdV64IorKB#dp1h1@XBaFMl6ngRh33%EB2XH165i(wo z6?0a$t4vI4KPm^G&uLm4CR=$t0S2|Op&ymq$H|onPMd>h=`u?R#nZ5cD#Fuqu_5y_ z;zFFOw>DDroJ-B0E3vYQ34!n}>M@b^fbBtB5BY>ZWvJlaR`tv4rrHZO($4k>=EbJHk!#9AuCvL0r$IAg#BVLxiNJlY@V5sR){4WFxcms1JSYu|9pRJP>7SDaHA$sVHNRa~eZ>rPnk%m3ee! z!X!4ruiq9yW~z4p&E~7X6tEk;3EzsTOvEaamjCq_4@PA9i>;3Pr9=fe+MTIXG8ktS z{hmd;3*GHe(;3&h&T?)ae_ySuylMB5K@ zm1^Z}0_K(D@KAf&*8rOe`WoNr*7G@R>wROp4QM$fGvWF5Xs~xrTvt^feZ5_-M}4?z zEt~HC*CRh0jM!hP1&#CEA8J+MA;lMG%EiCXLAvrcN@M<&B4~&}`F#H!STP|ff+U6vGqORSmmN?YI&)!EPx+=#WNGkBJJrXWU8aP-@Bh_2QB<^Hj<%^o8v5jV z&8V|+*{kNn6O|9a(r6UB%|2L)AUd!`As4|n^UC)@+>Ob8r%ej4o7_FVS~eD(V0Rr( zb%MKQ!N*!zdlJ`$x=1EFl-~4JUk$yOgz34B`yWAkc2teRjd9GS{%-@VpRM=0*r$qK zeYMJoR2*&%sH7EXqe%WS`sg}d8%YZuVITi~G<}L~_u!zYhGB*^^=K3#VPjERC@hd3 zl|yIf2xE)t*wG3mYoj5=Ti=W9ae#59vGWyn^Q4_T6uIf)lcM8A)V#L$w$4=$6Gb9Z zGqBU2)=EFF*$^y!;aufwmC4ww`CN=T`M0qe(wWHvA4Ti&t1cx4s+$Xmzy8H5csi*- z_e-HvKcY_bnwYI;d{Jv{Li8${4k6_9u9b&y_UZWb;lgn((P48fc~kAd8|13c6^@fx zg|}E9T*l2O#-z!^`C^WKkX8M#;Y3mFbjUiBX{1TW_tXIAussbGuSTj`tc5?yy%VpH@_E z{y@ZV&>=B&$WRVLoEx9V=Y7w)-t!N<`-i#qweQ(^?|bjH z)@Q{i@Z_INMDP%(SL>$HnB!zVLUb&%=ut$6&aF+nTWF?X0s#hWY_OOiLt|+N+C|nu z6#b0`Q}Ne!;M#2A=gXuHm;F_KH(ABaUz0sftEnt;$~Tvel3=n7-~6S|rriK2*?MWS z<#iS3M7faYc_+T2S5fMOy3L*kaQljMDYNTim%*M=aY?t84K4 z=JEA;Dz3l#pyJBB(`uRK%djZ=8SahOi#Ws6M~(xxmGKQl_JUKFhQ>ZAz3Y!W7g29n zr;#|D1wbU4xkJs%HBRLXEXjSqKz5ve=`@laQF__n9tkgA_Gq!L-1UXb8BTu;4Ga4f zaMK$)-Wis$rwWcAc(2dy{!W);U~7GbUO@hr?ojHDW^b(!-Z2XymFREgX#|-&tikoF zJEPu*g=08MA8~^ZqR`62AXge}^kTs4X62f3H7IXC{36$6P80N37NCM&6iUTDZbW&D z&IFb&UTXL*{pp3nk4*%}(OQkbOQk&0%6y5HsOc@V1?&Cl_5dwGtz17+iUu6?T~b&{;h9t(?>=?~Xy9gtKKIn3(bo%Z3}W_1Ep=|7QEL8FcC5E|MDU3#|q8SDN1c@d;& zTDHQQH#N2)SORyh8fTOY&YY`8{;VEn+z2|URJq|6e0nA@v5!+mcRprifjyOJ89{wd zL!!KMR$P57_nyl)78INL!Xeur?G%5YPtAE z8AoE)gQzTCp0hNr)!Ir}oB3=o%TU#2$S+r8Ud9JLxs2`lv>2Kn$2LF1AMQs(WHLm- z(~{XyHAdxyU)az=cx@EIit-CMvzHO=)#u#6SqdZ_2XMgw;=p8s$V1h^>RY~|k4R&A z9EpgqxP%1Yfu;Dv7GHU>L$>9Raeb>(qc0Z^jTQ}q7ERd&UsG*RK{O$W=cDXdrX=q; z?Zq)fd_OB(50y~hG22Xj!SCM`wGo=VuO#t*)>_PZb*?)-TIyIU|C%UCD4V$|&*Jt) zp)mq10ULPH3KE{`n#_&nZRb9!*4DYIA9CgJq_#r+Az?k^!GM3Ko+XEdLynt!j{Zn67E zShCQRN%rLD=PZP%s1d#7XZ2LZ0d0U5&W=t}FN(wxmwJ}bk>{FCkRe6M^U%v58Pol)Dy|V- zCr%JW_Yj|i$D(z+7&FLqd6VJ#P3DX^^NH>`V-zZ%tEvtAG0mK>nj|mom;}@?C6pyRZG&*Y3VlsZW&~{e zdGMhS>$hBcWE_@PpL;y(NMo`I84F7veH1Z?QhmBSw)ihVl)ou!uC&48y^ad(cI-@d z%@o6GUy5*;&}47-bbwOR3(jdL-7aJoGFOCq7~doa{#7qm{Uzdkk(SUl$JEn&%`w&i z_{?bcVi%m#pG%FHOzw}fggUH4MlqJsYf5n4K2%Y+gPDg0g`L%7_x$Nz6YpIUpeq$N zJ$Ck0PhdOY@vf>m+Gn(#^X}9@PRfhqeFrGun{==PmX5)hW&e9e(0}f+59&jaqb#oP zf;|D5U;=`52XR1K49CFL@LdJL!($%B*H;%ih0mbM9lv50TCpQhnq907VVw#P z728dk{j&va$Tdp877+IW7K!BO`eh` zQNf*5(0vR}$pqcD0L_6j#Z$9hgfi2CM->zB8sF*4C4!pSmk8Z@v*9<3|M<^{>NO;x zUDB`ANYpU{6ZgG2t)UT$zWc$v4AR$hYr0BD8P_rK=J!Z0iAS3G)uW(%G(QeA0Gmi@ zfxIeFRl!WDUuXO9C0xrRB)9Xu$PD971xv1d*3H#>!zMEweXx$S46~qjYp_Fx59$Qt zlBVcw*2R@P_07iRmVHa?t8IL#=LQc@7MgXlu3^Fbx9ko}K&ao3DKAJYm`%LBq2%&| z7&NvlVn;6*;#zN@m412*J!N+SU7Uz!1;jfP3nK=>b{6*ioRwKISD%YO5x{>%XE&g0 z_CWL~`S~umTK{9no1;)7K1xWEvD^fm-Dl5acvbCNxW|ImJ!gIJ+m}$@UMH5ve7Va% zBr{jAE|Qc@uAiJne_sJ_6W9!^Nvo^^973C-N#AnqTdBjihUKjcEAqmYFUAFfC zHL?j^aR2VZC0Ji){XW|7W|z(5ad<=ia()q5?3Vs+u6E-}4#INQz=+z;e6yCd3Y}bj zXz>IrnCaBkI{q-oiq_fe2qVZNo)Xo+zBPZiyHFl{RA;tiCnu3lQ!7Do31})SF3u*u z<+HO(RU~fFlL^C%G~5VlWc51pmE?Q0LHn`7{nxNwjB-cN9$9`c!&&P3)r#OKvM4@X zSJjy=f0**nRWoTH>#TZHVD*qF4n3?j((5zp7V=4nDKD;hd3BB8o3@7w$gY}i_0#Xw z&zF}puu*>Mo+NVgU-~+qw>jGcPHkp^N}}R@TiV(E+Pq())IA0BMajG%@HnSOyu1Ba zvp(K>4TsA13!N^l&V-U z2^CHh4*dQdIfkv||1_KVgG@WbxKR4yV1Z#gmE@9cbH?qLc1Kg!_ULU)7P1(SB|@3c z;42^UHtam!-zf$Uz>tkeu*$+8|2X6{nf8lV3sVdIj-{-z$Xl=vDXR@~p@^IFu`#+F z7UkVneiR>Y^|kASqpkVk>Cs2mO6iKwy%PEO`4dS*|0W37dhZqiK^1LL+Rsqll7t6D zzzxH~L_#C^T@NpToj3QG;|f28Hmi)g1Pse1H`T8F$DfwEXWcF*Dd>U(-?R6nvscOn zvYn^;GVPPAAC9fA7j`W98N0^-L4c=i%_t5%sDwvy7_(FOnC?i;v5cAzeVhXD8Q>)G zC2WX%SZ{cN8l?m9qHaNX#?I4v%(2|f;*K`dt(U`uYAV^pgu)(A&5;<5Tye=l6vq*K zk#bUg_Q=GpmQQ16a^|6)=IGGv>9OXjva*WAy7ESLVRj{M`T- zVHR1yRaW;-EuhAsTzTfy^g-t*9x=)DbmhBxFlF(aM!zmH>M~K$j6e4|!r4PZq@apSD!le(Dd~V~?5vEp5ceLE- za2TcTCTL7U(*k;5wT20V$C76{*|R>3%6IWpjifr0)Ju&7lbPfGNVoU< zkux*}%&`EFR{*yEe>znIytI0z+7K}Ak66}FuwZ)4Jgrl@x;a|vUbkCV&3ubTf((3) z@P?=|4!_PEtdZLq@N8nL16&q(lgP?czT~9qO~PK)#bRwIy zYwr_b8@xTHYg!#lbrK|BN=M1T6mj4lK1qqP+Y3Hg!V6#gsH)!zu-(a_x0}5vKLKe< zwhwSij|Zppm?E02=U{>iN#x$&M%afs5e{hIuZ*1~m=3zCqm%X%df{WmP6W?^%rsFJ z(3@F);oz3HW|jpeM9gmpqf)r`7T?iXd;O8$@}!|5c1V!;)Z@33GEAU>2#vWoBZNO- z5HJG-nAM2+{)B7IKUE18s_Mj3=k?}T!KB)R-!-0h1l93*e9A;R5f$}l4@dkzP0 zzqtD4Yda}`{|?dqUgoanCUZ$=F~f88n+~)eS}Uk{vlGjQ>BS!@u}y+e8rn7baBaB0 z-;?E_#ceLq9GC&y38if=0FD57(x3UFR4yr*wZ&*o(^Wc`AvPHfJW)6dBe!}%Q(C3Lte88c?M6=>26=LZU3{zm+tzxRnOebq`fb?Af?=B zJy;P@DL^!HH{)z*I23-~u`9dO4RFlh`>|)J&yGY5nL6*3-OTl#Gj*1no2&BQ;S4BEUqwIG5@_wXN<0V zq}lPR*^%R=onY4Yh6x@a%BsS)EeyR4gML(zHO}DY5N79ZalXN$nIq5GSsEory5l7m zv-Z}{tD(6%j)-fkzp{WreJawxXnM6V=Npeu>b-N`{ApL;e;bLVev;lPJEiu#Fgg0R z7LvA1;WawXG2(R>X=j+|!{g1bs^4C#E~&6TcBV{$w@i3+Q4$CH(mIO%NW|o~ncVvy z(PS}__vAUSALyKk_i!UP>A#{c)MEXixbKTG{~OK!KWFc6EtG6u->Y4^2idyoa9~vq z@e=Mig>&bvvtqdnUQCq3h}K^)7%Nb=0Q!SZT@{xX?@Z=f7s~0P`MZhg!QtIWNgcp* z3%@^<|6bv`M#?qcD<%;X3$8P1y^d?ZZfk+;Z{6}TdZDGZ7FpooCJif~6^j5IbopO= zxyMhlSHdtmj)AsKD%&5v<9o^K<}o;osSmVWCr#vjG${+@vITv)Rh=FCRxtgA@HAnNGDN`;S)opf3z-Ac2 zCL;3Hcg(%FGOM%yGp@*&FJHc@Yhix*rp`I3f=JgT7G<3G=4?vdTrR7myts&4O-)<- zp5X(Wv;e>N=~qtRzcPUCz<-7I>cqVVO8=z)C0QlOiSNAbM+@hZ*}(egEmg<;oi=;$ zT_&FwurZbIiUpmw;w_|zbV>0kQzd5Ygu4s{SawP2L0Nq6YIa|r3qAh3D2V2r^{5P9 ztArMz^pau*A*Vf)dJ9rn0;(_KV(38^G4qbLvWUSu73;@;*D-ry;00tcU-&_QAy~Ge zf?LVARfRnuoX>rimQ?#PQymT6y|S1^$oJdtkzHk>o8Ni=86N{l+Ym}aLvyR>m+(N5 zX7{}%n2Bw^g-s;rzPBMXNX`0tR~Q@cJ^V=)M!=A6b_+p)pIY2^#@D6?NhgX(dBlzu znCY);Ryd%rSAV(oahuU}!Tee3tlKrA?a@ra$ihGil`g{)Y$sEM;eXnb04FD$ls&Z+ z&$AkN4sqVDNOE~rR{r{ruSTFpe>1ubO_Ir>A~p8RzNfJ*dZe%J1QA(Q|Ecfh$zLj2 zADN$L!JpCMBH(^>;5U{5;9P&w|5KawXf71(-#QN{EHD47S&si&`+ueeC~b`|{wi9a z)Uo}~=g|LLN-jR^JXP=BuOIsg2;Z^_N6a4o=dJ!Txh`Uo@)A3Yyh3OO6hr;5gAj<` zH@Uu`xx+D_CiM3hOjQmq6+4)BPjV(#Do ztb^Ok--5V4K;G=5+_Aukc;!@DhVedn)S%7g@=iZz=2&$tX;s#6+2utlRzCFg+NlZ& z#}rIu_Ld#yQA#|URGBwE(638=##^!8Cp4v%R^U4fU;RgrP|ZG?aCE=CX^;>&a_;X% zD6W0ktO3DldyNAE>!x}Hzn~4j!*Z}vXkqv6;S9?SRBJ-XR*T;@?nI%Fuvl`x(Xw35 zr?EP-->x&BM48&j_x!p*LZ@I==1Yz6jF6C5+f`+RW?_(Ra$D1`{DR`!{?D-X+tU1A zyQ!On$Y(08*R!?as2r=ACm-4|^W*te<5S;-`wm;lZF|XDY7JQ4FX6PZ(cbhoco_`> z_Arv_Dv{)MTex##853@~@@flh!3E25l|zpp=qqS9(DwXym_J>J<0xe#^Swytk*GDMZ`HjpX@%D^__fsG5v&>08X{sQuc+DIj zc8b#KbmW&rKqB4rT=HA{f>Z&>&9rWyq@8mmonBy-v+#T zymWJB{*0|(#>YoPwj$Ne50NZrSL!z2xR1)F1=6c8ez(ZOKf8NmJVj);s&`sC;?+si zHgtRrpSD!DR__VGJ|OU;kgf$KUD@Zk9n=XFV5K0#4h|&UMx-mF6~_`Sl*zx0RfA-P zU4&Hm8^<67Q6xH;iW&7#!hfP~GZ|Kz(wnU$9~+m-?mJ~>%}D+j=D%1Lw7Zqw&}mp+ zUEZm&)WiA$eR}b>W6rjEZ|`lAP)p^+qW^vqCpjb3eOA&a{XE5NwZ~Nl=i;?bX0l(s zk2?2?l8##xw7{c-vmS@g=7}@~Aw69?&C$t4qG{?XkIJDETus_8eXg&MmuP*-TgW|p zwzTsj#FtGg<6}XBCk1$_^z8IHA&;xMvKN*LZ_nCqYWL3Kte|exinsLjv2RK5x$68b zDXR|U;f`QfC^aL!lUmM$)9VK8v>GwBxR>z(gDT1?Q(ifT6T(xjpTjeov3 zzOo9nHL@xh(5$I?kL_&#foal6=bZbMe%3r41a<0raL};oHC^p=aVUY4u9cBPJ12@Q zc44CScA(F(6wcMHdarFzF1rOq?ybI@wq7jATvxR))f?YmD2lDh1OaLXy3}tUZRCw- zNE6+bj4bU5%O1ZyVu0-RUR9l|wy+ABxqMnm&%b(elv1M#Bh|uJy6rlETx^=|jTlor z_!9g6Rw{a~U~U?V#51K@(dzMQJPbSdDJIgVZ&~$r=s=%@HJz*Bdaj1+6%r zeay+SapLT+YGqlS{_^dTK02{$PbYG<-XyglfW(;AN}FrEE@mR{3-hX>ezA}}Sg+#Y z1~r?{Q+V}Bu;u!d8DAf*cUFa~j`hl=^YU!c>{Q<(aTyT@LzhcxWQ)i2Wd435L6jVH zb+F!tjQZ65kB9lhQv0Fo?Dh0Zerht-pgOR6B+D4KoXpebED5cnB3k`oDFG`?aLhdA zIw?IQZu+px^fj*@8INz@%PE&|DZNwN5^ohoHCs)1`)$!RGI#HLl>(wO@uThWWTm>5 zaf5h2T6p&D1XaiTP5Ua09dWO4<%-A{>I&S@i4y$)(CSPQV60a;voYL$LoM#$ps($V zFf|nJMf4>Phdn9H?XfY+5@)MSTS#$Ppm6LE}11@Zch$d zl;q|Ota7sD2HL)9)+i(L5Pg|z59 zsHY8;OK#ylV~^@x^{p#8ejIyLr7J6$%gAup1{tTYobMLo>P~^%Kunb~+x~3>x_W9i zumphtm{mP@P;2Cn6-q(mc_UI%L&L$=!{=*G!K|BI+4gYJ=j7BS`A7|`!@||DCTom6 zq0b9v7}i{nnMA9O!%8;a#Z`z4tPxs3SIX5^+E2nKc$KU0dEQgI+^BQBsk8fqS7)>3 z2}V$NY6RIsY879qToyb?8Wx&vfp1Je&D^=GF3w>9cEq*yO{y%%73MNQ1siS=aIi9- z5m@#h2XJdD|h13_4^jp0}b1q&7Vft+qheItBjd_pQP<5w9c+DLkO>W+271AV@ z85pC{)>E$e?PGPpdN?fhubl=(78}*lwvY?_Rb6j8&)dBR+vUS%D_t2yDH1c8+I@!G zi|Rp^X;qfh0wsyZ??r^dujiOa?bQRW-k1Htq`p(7Qw1STg6BKgxca!Ua=q6iT{9}$ zD>;slIS*!&pv2xxRyU^?*g`YD=B!wY8{` z;DXT@zayC%OJP+dkC|?m^kx=;I9ncGPXvM*FE4!hj9^iGJ*IsD_iF)dV&(XjPE+$6 zUXH=ryC1@#p+=FZnF*}8%DK>2B(p5+Tk)E8oaPgB@csiOg7jVa-4AiI4~ozAI z3RX+4s*{DOE);G1Fr;x}3Cra>qD6+1Ep_yeyu{9KI6a0>qM%?$-p3UDoW8yi+D8#1 z>|J&9o_RV?BAj|hR@}ojWN62I^Pj6SCQV+ZqZe6eHwWzk3 z-n(W#lJR93U8REw?PT+R#}mM1#R|q~*p)C%n&@!`0xmcCI zlJqXlYARs>EQ{CQvH03kpN#UMaY0-Bf%o3K_$RVk?Ns2bIWEZ*mwkdd%!(t zMY<1pqAh0e!?6SVDJVOj9jU#ApOZWe?RL*d_N5#PSPRROs!=iFw6QOu_H`aVBzZ)fWNX|1@0^$LFX7`i%IAq7qltTMFJ zZYN^^xB#+TXwT4+ER6ue$7fh?o=7JssScJ>Om9_A;8xU>Nad)`hV3N(^v_1p@;=Np zN=e+iJGEf#;IVvQkJ_}f+MyZi4b*j z19RzaB-X5qL4=wEUcFt42{YIggP1DV7IEOhU)0y8{j~enk8L^*wsu} zej-2ERl=m2SHtcU`vbnX=2qn7jBhyqI9RiN(){5bgBYusx>UyuMz)dN#yl@}BJ0)@ zck{JYPK~iZn9{ioK6$ATVkH(_>I!-Xy^ycZ88UvA8|afELdcK)Qs^sAo0ojL)p0+S zP{KswPg4~Uq3`p3$;Wcmij4=>!<02@=7m1)O$SZCCO@fJto^Kuu0yf7jq6^ShCa6n zvjM1;_F2^fR{~TUPfL^P8%czkJ^2tZ`QzZw5Z!&ZkPOA3JiOp}F^j3?ahblRtu;%A zA-8Fs=#@S93nR%|m5JvJy?iSc($-FPg<+L1G&7jbZwg0kg=QT~KGt8fpl&Due$K1= zJG5ECFz`dXf`^=ob9FMHd)E>55V6K?0m!L&W^hcJnzMJlIhWp@c}%wqs4p$zsoGES!UGWK$N-)LpN z@?lDw8pvl>(w&I)LG@i>ZhyU5H3)qgc~wIZ2+fUX->fCS89yg+vs+O;{aiu?B35qN z@BoAZ{)A;Ety15)-?Gv*+V3NEg0r}~pU9Dx^9pfm;UaIv=bTY|#05wahA+&%OAeQ+ zs#=bIrD14(7N>m}@iKB!<&?Da^i+oR>6EccbYF`V25kWi>0z|2eqZAkBa!gr(hJgZ zW~S6ZQi6hKAIwzcw^s_<(LOAQK}o24rzvsw{S(H9z_geNqU(#T? zb-{`c6M?+mH|{>rm(c4duNrH5eN=Aqwt(N#J7764T^1>K1`M-sGK10~WfNNBSSRC%nehh8A{T122p_r-fKlHm$1dC6 z6>uDS)8;=hP+NEL<90uex9_){XJ7mT+d|UPn6c%L{{CX45Qt+~tdLcU;$1^a`n<(_ zjtEUZd2{QDN6AR?j=FP%EZ-ARW95yZM$2b}oX5B)ucP$nOEjh6x)-ps%-=uX8(y^2fQ$Whd)a^w84y1BZ&lg5 zTPhfpEMBgv_7?Sv9jjRGWv5Mjx=H33SwRRN|1$9G%bdKTcn4gX3NCeFnW__ZCghCL zEmJhYgr4?BdG24_FzZqJH?+-~=mxABclNL1JI09uqQ#kDlNugkpk zH9b5WT*?HKJtL&d`V2qZ^sZc&IN^|!nF(dIekHN^mHnzRv@QZ(TZ(%u?dr>vQ@=Ym zIi3bNmz}Va)l}!gHgv42qb&C3FB@f)tN*}OPgw1)f0ET2;#rt~b?OlTY(0vh;ZmxM zL-2LJn{=QcnFX(H2`)plbtW+d5*-5C~Lqck??NrKf)+A@t~PiFGnBsWJ6?WL82Lmyuj$ELF}IoYrma3uQf9JLFcQU}s{QrD%gN=`TJR z8`AgNa)iy=#TSlaB(-fi*)|h~&2aRL=3~x^am#&`wRU2ZkbIP-g0Yy?Hz-=bJhWf0 zxn1=6eCxNDxQ$j)8d$YLM0?|ETfec5`^UsP(bcxZw5_rF-JyN>@aN``1F{`GtTL{a zH$Ps^gv)Y~nwN1aXsq@v5vq4p3VFg^&G~SmgWos>&juN9!h^ami+5a^T@NtsDp)V4 z`jEtBgrRfSOnVvOvBv0ceU6lM(`-)z@KnM3E33oiqPnUn9$YXjo;n)-fHctjG zjDqe7NOP6V`5syr%9?%`K_ecno8_?A{F=A$!g{sRecG6`%QSbNcGG@t+%H#Ad#~lG zGN{B^PQcFD*L+oa8;wXJIeDj{QI?!yT1_L#E6IGDXzcw2hC+Hr8!rIs@$^)SUj!82 z@YAd6$6D1GqIr*^W41x1>wHYvG?K{3hD;amWX7m1AdpzF`JI?bXztJNVr}mhkwr{m z7m1dYOP$NA#g<(N9=p6ryCd@h#+#A4Mr8-G1jlLGzCEsRX_8lAYREVp&qZ}rq*H^w zS5~ckV6?!`a|6q|Qqk4NeZOUGL3O1Pe8#5N8& z6-k)ugFPqAv{=Bwj^cjaKVb+gUsqk$AIAXtLM2AMJ72mgPUOn%`MWiBLC|sR&Mgf- z)N&`PV{tVLC1SDv=0z2m%G1+OFp)STWubq%s$ec^n2nWZHrjFWaruF}^>i>-xg#+% z^>sK>8w*Ojxq}N5_!Hlgog`r>2)39flV^HL$IO^!&8ET{TUE&{bX}AtlUT{zSXjM1 zmGX_ovc7QKQ~-H>HWmE^`v@oUTdyirmcXTGVN_DGA3jm~)uhXLfoo(EqQcKN2W1?k z1V^g3(R#sXWC7?|SGn8<%e#~L9N0YmnAGe{oru60*4(Nt_viMfGVX2GF+ZyerhIl- zFfnmeVdeb7gy|x3%q(Ioa~-A%2M(vv>z{}@oNp~8B^7Rv#}7M$V%T6b=8~4sNBM^; zCyi^efj6r-;S>N!g@TgW)1JUnJu3I`*{&B#hpMe+tw7)Wg!s9wg@dB=aMx>-SwM9D zo=exu{hDB}Ap$IXd%b@Ns1E;0okEc-qq#J}fAI+aV9T@TkidUG{r>?{0F*Ppa&Oag zpZqZkVnrAF%EWGuA+<#A2e1BB41~B1d1`5@!bmD_96SKPUwd=acszfx7@gN3l-|FmgV*k9|ZvZz5?kBb;72FXFu(1QPh1b~U;Js9%0X(d%e zY!o?P(C2gffA{Qf_9{@5WNau#jlJYw@Xi>dwvf+d=Q;l5O<|U=%~Ctr-$614bggw! zE|)S!O{hn(R&wrpjfsp#9^1JifLL^nI@_c%R-WKY6_W8gKJ%IzEeAi;7fx z(NsEe!egO zL^sBrAwyP!SP{N{)kqL;=Y;2@)jdGy@`8a)u!(ISm%Ps*|+gtUZMSXdZhWZz?z!Rfc}ZtyJMDZRtOs*EPOF}-(pPv9u0>w<+v z+WDUYyVs%691H6f7QP$9)^Q6PbXA}~%1GVLUP-;hLN&i)H3mq}1WLpy>D?&a=r&}ZX@!U;= zjMSW}*?36jjf(j}(^KUA)1N{a$!@%Vy(TT4`;d}dizCM+cWsD9O;2(UG^sH!Qqw4<4;-FPOIYibR}wB_%>I|0r(-{N|1iDa3@OiIisH2ty0*mG zf@juRu=52mQlBG_e>poo*xn|MVr5`KF9_Q_`CHtN8k9W~Il_!3KfqwNH#B~syw=uJ z96=q_zo9V+9GX7{+MlJ1r}d0=dEWc*;qPIWxomI2o%r5lR$ERK($l~1SIaBaCH(N~ zz6`q->8ioCZd^3RT8aGczG8QEPR0Pj$-YTH5(7%*gR)0)ayHQHCa1_1X|*_h`M=3a z2U5t(&%i*h4vpKv8q0(%JbEx`ryHEtG0v+(+gtQg=6_Ru^k{rUSWei9zV#d7V{X+~ zzXpkePPSXVpf7v3U!1Qy*~EqZUF3p)$M;_KZg)XTv-pc7DHTj=BFed~{R&oVUjE%K zG_>9c=dN0poG1plw`zDTRK9DlX3uU^=>{p6B$-WGu+1kbE| z8TK!N=4(GV8$U+|{c6L_F=A0%brliz!@&LzQg_pD%~j<1weC1r!{k5Rmg>AoS8g`c z_!o=Bwp>Nnv`SOl-2K^(+6w>GQQzeMCr$rxrCGRF1BX3s*%(+eu6#=nLZ%b{B0N*< zBr6{;%`a~+b)+QAwq`x{{iG!iw?K3I4Ra8nC9SW(hQsUcv8`75oq)H7`BH&UqH+rwWw4(u7Cai!ODhZpE@ZZZB|;lUT}1EmP3?D)xN03RnAfb7bi8 zxPl72vAq_QSA{ImSA3M(K)MgI>zz9!Vg6g%IK34Ovz;ya~U);Gg<(? zl6}3)iWnMAO%nTCL!5OVyPMQl5$Da@)B?aAT`73dm0aHhK2)rdA7nW zt?UdByftchb1Z;ew~V{;)fHGk-l#;9rDZVRuITsZR-X)?5#+t23bpc zd(8>(>wonn|3lKGhBZEOSJBRRaPO?tZ?41pq@ycS*V0JU|#df2~+U$0kbE1w&gdMU4D1DiB;B?PF4e-hTX?r`2MzM4!2{{<$h}wW}31Tu$NR| zPt+Jf+uoIc3#j-i`*8T6DO|t9w#(rr)1h_ofztd~MV!q8V{M6PV}9$!9K9X^5zHK1 zrFGUNC3#s7>)YJ^^?0S7)OST}HvCKL*^Ai|^CNS>{np-{U+4WZ3cc^1G{247+S(6p z1bLwoAGr>N2EVM_3DWo4Yl_Hg@3bR?3q2u?t1B}>uRI029Vqm=|GLblHyP{LPVLSA zY9=C3*aPX58EuSv!~SJ38TQtPDrUg{aLV!Z{MO`pk}PVhv?>4%7~1NjY+BSNW3(?Q zklMC5h~9-w<(}g#Nx8(;W8+kRs6HcuRP}(x+_vEmXhV^gXYM;I zM({iKqjxi<;*=BWv;3Cy9<41sm-C$8-1nxrXVq%C8A=-Oz4Ob0x78}8tUYlZI)RUJ z%>OYt#m~5Gz1=vD8Y{+4IfHlBkirc@1}m3Z*8ybB!^IK8nTNg1Oz6G<5?;V*A6fLo z77f(Hixd<+8NhpCdJC&NyNy1+{Vz%P-$^ZSjmx&m-3@iE;qQ$v`p|+4fWIo`>2#}x>sg4I51x2Zcm34sXk@B zeAaJ!DUJZEeY+__a0Mn%rU`vjiVOdgW;B9kTJc#JP;Bv3t<+@)e%5yUlgqz9_i4s| z%*OB4A_XbD@z${DUXH|7P*y7{BKl?IFXT205ZNJ=>1RqmVu|-5p#H+otR7mV-`2O% zw(`{tw)dOyh@6RpWZ+UKnGkZ{oZtt}rNGWWD->e7WexQb6xeujdb^lrfh68=bxJV!-Plm{y>cx3MhCods7spV>zRE{Sk=2pQ`kWZ=eC8g;|8~Jyl5uKAOAG% zFG1n7NDD+iz3KVN=fD^g&3{E(fov|IVL z5d(kb=!p^_RSyeZ3_hCqHaC@?Jj7Zh3au+N%P_1MsLh@qh7{Rna8qqR&`-MUJK>Zq zC15O8K{4)I{n_DpHEV$KA3NNvIQiz2FtDs&L@QXkGsF8o(KTTPo_`q> zgk5*IPBr0QG=HzJ7Y@dT;d~$7KkF62VyL$>t7sS&?z@cOK&=F+`~(=L}Cc4BEaIl^ICj_n90xuOl(`;@pNViFI7Rql(beINCzujD#O=|5iyt9fp z8vBFc<$$4K(^UVT>A9s|X~@o&+*x+x<5qZT^SkI9o+uA(>f7=~zZVw*uArzl3A>jZ zQe`iXCMrZdzbYBO9Hz6lUmT}%3uh49Ogm@Dl=byYmsmoYA?Ek%v9a*eZl}X;QCLq4 zez|&_Uc;aZh*Rq_rU=CM!Qio>KypcEfA{zbn& z@7}7$o;6RCwTm=Q{DM9}sk;%MwrtyG>Mysw2cP-n#ngF7+;Y`aIgzc&0=t6!`FT&) zpzVJ+GGIt0G4LH-QW^3{Sa+3g#)q z=3hqYt|3Rwfr2pYh^^)iWd~0%e14bm-kL!*XD**gUr5lKwy%5W*pzib(K*D2z<^4U zgj{p^nTt+3;I)f$k0l>)`H_9OD!9gF>y_?!hzkL`r{4fhRQ`1Sw2Q|hUq)av=tv=Y zD<+4$y~W5gW3i`~+qXkv`cng@pPpOyi34lG8FJMi$!z5_u~N9(tHmS#2+yAqvRln< z1tr5j)%y0ukmc9(VBuf~==;8JTVAhhnGRd|;-&1vEl!^2F2#%euHmq@H0bpaKU7KY z{^iPxPGYSu_`sr&DZge=8F!f=(u-_xI0~}y{9HbFMtkhXccH|}$)J#*@w|J4NdVN< z#f`3a0U8${X7&$e*$x=kW%!GMjNROUH_wSab8FZSp$py9b>>MlxUoHbR@{(3z2Fhf zlVSyq@;muMATq1|3@MaaxjC&hJnqu7^B1MMr{CB+o{_6idqGF5emdZd%iF*ucql$? z?8yWjtd(zeCOEWeiXW>$XLl}|h=8dt6#u($%TjWTiBJfcgi*gPGEoG2QjQHNl-{hI zIoj7~L30Y{a8Qpd#)kp)k`%tQmn2Yaha~YFi`&fh>z?{G%V-=22^-j>WNDU$Da)})zc6Wu zjCflaB9t~OCcL1m2U};CW35NXQ$d_oqXHdNKYQv{ZjALSLlt^V@YW$L6(ME3(s?LQ z#V*0`qN5_%qSSPZ_o)du_3Dx&qxWdsFM^HNB!LdUUna49nV9!+MG_iDFv)H|7Z zdf5^KUR-}FmoJHkF7%x?#k3yVX z@N)6n#hf6u;DK&W^Evx*Uh!BXh{S}7$jBQ!HpWybFiBkTA+7VFC0-!NTD*e$Tuqzj z_`_Oy3v$93589Lg-(*v~x}tcp7qYW|nC)d^=e;;Zyd(uU=~iQ=_Wd(R^`DK-p@Yah zdUJmp1{%~#k-}xhO}dj=@6pMOwoc5aq|IiO+M^+}F~X$)-RKYTMo=}XZ%kK0*VZzA zu&}kuPWav6kh(pbA`hMoTV;-o{pkSWDq67nKl6v9$!ckywP{(EbTa>tnOSLdR5yF1>sh4oZfbTABaD9B> zzT>6cf$yfG7DL~a>^;xf`pOYr7=k&=>mN+0Oxul?+-UQ0rOlL?n2h4qXc8zlfiPbS z7YaPqAI?nOreC5QacbkgZz;F<5O^!m7v%{s3msG_2o(nsmM?>8-t`lm{CFWoO}cMH zmRsR8Vyqi7p``FEQGu`3<*6#?&Kgxr&q4XmAs3NYERK9$RFFhA8ZH>o2fbbzs+BCkd6;peqXbx%!5 zz@t{f_JT2$Q$&Zki;kx8CH^ZWlgwz^-_7kI35%`dHWH#rx{dtFh@yt0ngz!|PK%hh zE<<|ddoYDY%f1Y4_MQ7uHyFsLerl0P8x-Sek@RGQq#Gff1bD=~cntIh#CIXlw^}Z? z>z@iJn1Oa{PQ~fcp{Oo3-+KE`S<88*@*xk!K9i>Ehb4fH|HI)%A4!I9>huU3`sg!p z{aM&jsbc29uRTp-g423n6TP@O5)T25XFx?e?m2wpi^@{&Ezn~1f~SMcTatfEF_lFQ?|t~A8K3sG0pnQV zvVRr<6do+mwtRSTGe5b@4aT3}^aDIObn;_;xiDiz3|Yk3qqBNaJE3t3H)VJx0LJ7C z0L9QRlk!0JiVRL5Z?Mw+Wmm%8#ap<#FBIxH*uxnewbBj8PM@m`Rx!38Tx9M!UJzY6 znmClVZ~ly1U#W^Y`SIV*z5(<#YK15_%OlN$as0SFvr|;Ug?!!kiNd`Jn6|NZsyfBt zTGv7JshH$szQE*Fcqc02XVe8EfIg3Wg>%SWI;3S~f?M0MuRsLigXj%q?S11r6z^KL zwv*`@)15W69>R?HZ8w#M9}MVBq3S&zns*OQ7nj?M1sLkDr217M zyZVZZWsJss4HRv?d6Yyn1D-${iav_z`<`FQ(e#S4A|x*Tg&%Lni=kF+#01YI?Dr>= z({2`dlLE=Fmh8XJJMDH}jJ>Eg85q8&N!CVnM6{de)Z%?!+>?sN7v7VQcDeSR+7}Pf zX5$_wEgK}Je^SAL@;)%wyk$qND9RTL4032U(_Y0?+&&}_Mi%pRmoLv3WmP;wfMY@o z1`*v!hjuqS=M26xgfZ|W6~Ov@1{A9ypFcz^Ypu|*dyr_0_@ay8LpCsET3(tHW# zap{T>*xj?4;%A~pyUKpo=jllmFY?L1!W(V)(NO8agk$Du4>s z8aXmC{Aq9evbH4b4v9;NBfskB%#!8k+_tD+=4r$t7(y>nx6Xfxyz}mI1wJ*6bn>}& z11BhE_<3(pmA(T%N~L~k?GNQrF42nHx1wU@#osQ;%a02@8OFPe`=Gz5C%WX=82#rX z55L!upQ^pT27c3&;j5!vnx*An*K)$zG=ZMe%!6bb(ULI>HRG@9@%Z@saQc@tLRDKn z&{nP&t@HZzU)FlGQpJM~n;nR2l^raPd$Ud_JC=mOPWR+CW-vD!e{BZ3nYM&jj@i}( z%JDp%eyn+Qt1rBGk@fw|Gf4#$qB3mVUIu$lt!2blu1*3+QCzh>#GH>l#JFjLZs1aK zp6q6Qx`$OU_37LQ&)8&H!0?}BiO*LAe%ap0w*meVmrtTC7YoXd=4LEu#J$B1%N3v1 zcSANSN_>R8afB+vXYJ=3UXN9=WxhI)&?*&_Fp_N4|GD>mT)W!KPlU4bdDz!aBuL1Y zNqL+61(p|CINS~}2tiS=JXRqh8N&&jz1Xjg@Y$x6$P@{x?Oou_3^-jj9N4FT!3-F_ z`Qy@y6Zlec9DXM*>QoZEPuJUoH4xc8lSIF-`23zw%V~VX>G}5WjUI`{5*K{v6zA|S zw4dr*7#-9CZ;&@{OO!A@Lz@5!P@ru#n!v`0DHCo9!QNIIfErC~GT*_%lkPy}jJk;t z%1>9wjshTUrOT4(YP}^5gLts~DN@qYVI<^7i-|ZNdJX9puIQlhBiKfQXCSu-2xYZ_ ze$u~sYF}#JyJ-}~fPaN-U5j7@+4LN*e1?AedetL#($8&sX^sbBZ?MR(!lV^l{lWh8 znMQefeVeCo_x4m^iLuWB6MQhjvj#OzIeu?x&vUWWN2cx-A3hV^>2Uh{R=|0v`PXX( zle0R=`qagkQ;dVL_m76nEEnV3^mCeK%KhJrLcv$9pe|-5-X+cLnd$@&Rtp;e@q{+E z3lnEYQKRckDhrseUkl=%-sQY?;pkU9UXHB6u*e~B)tLDFy$uf^qH0@Ut+`oGch{4- zR4U8`C7mzo$E|qQ)ohuGm+#-}*JnZUv>mC%CyPg?I7bCusOw7%D3ntalIr4a)%#2h z)@ila4J_Q!3=59jQC~!*TXqI|e^3>~Th;DPIeg5QFTWel8I2)lK0%#Y{7_yU7S@F~piGeq?zVYtd4-08~(8$lTbg;b`(HA^<#GW+r zQ!&xv>}`NU%f7z)v}h!#(g)ITQu77PrFH`zYq%D6zp0x86MLGG2?R&GDYaP8pN<}) zgbsXRxFTak%DQT#c5i~&Ye_ZfW-@v-7cBR_^v3c5i=p{mSl_687isHB)Ekp)MdN=d zupihlMNlnMK5U2V+bk#6gbyFzQ@1Dfk(axVQ>;2o$1J3s7x#!*6 zUHh*~{eOhFH6srM?@8S-x?}Avyx&_o=8z}n|sFof1H1LTm3+VuO;MtRpQO^;Gx8gxHWS2QIk$M zVzXni$?#%R@bXzx?@`_dXApf#Pf(iL zBDCevy6Gb7vXog`b)bT?tDUP>33k}LQkv^2;3hRIZm>f@6)ER3ddYQm81^1xIgj-U7~*dRFoCx$~dK zGXUH43kuZ2XCceu7Hl)+5JA8-QG3tV#z~zE;i!2itJ#ml6Bar(|6D>;ka~}4RsklU!p2#Qg z8?!x2$&Tm*zUk4(ZaCsa@yV&;!xm$Qm%n$CUx=iUfKp# zMs41DNB%-Lj#j!*`+hUlp%QUFc;9>E_cofT&6xGbj$FPE;CHmBomPzp9GG;FjgO(T z8in2CT+A=N;w01_$xd{%^z7Z|5v`7k-*=zj8!_Rx z_mnM{00NdBt8}57gMO9Ft8Pj;UD?`G=?ri-JCI$b`Rr%l5JmsTH-fjR;$=FQ)P8WJ z)npM7gQ!Qg-Ef?(er>>GdU`~f!p2Lx?YRaD&Uv?N7efam$JIA4eawB>gVrt8M(t2b z1b(%!t3V_(3iiN;LcV0DsC14xdG^h*93jh&K)$Xy>l(0zXe%p&eRfSgm-8n!Ctc#2 zxdMyG#r8WCDdz*XgU?gF)xvT2kEoiWlAejZ{c*q`I{LPlXy5)aP18lfMDw#qAenhr z{Ka^V>Qjk?Yz5}(kFE#r|6JwSikKLh!i)KK-(LdcZzc+pRgpW1to@L^4Z{}EW67b` zKbD-y8BL>w9j2#1de)VXA9m{g$Q&v&SkX*|l_|Y;dWjTm(R*y^>iG5c)mq4caxXlx z@H6f}P-+$he4A<7r7OVuLEEu3hSXXvT!tZHV;LM9J^X9Z;Q@>2iy5P3QY3kBi<{kT{_OGf5kyEwHB=%^(?tvHd$ciq?nnun)j9<^{ zU2bYKvV5By58IaW_pPjuDIbN8a7)~ETnyO$6YB#XD~A0J;4rG32R{qHIlPMvicB-N z-T$xn+p>4J)4@Vrf9y~alR#XSY2)c;ojBv31A@?A^bo!DJhSglw^xiq8luxvTexX` z-?je``NU|}am44Bb8OaMur{X;%Z);wqZr#Tc=Z_*FaZ5jI5VLmjPhGDw{Id)e#DKz z=n_w>1GR9xaM3wP)v6$oeznUzE=I7h*+-Nm=_lkKo3%pOzCS|z!fIFaBX}3%U3J5Q zoZOY>>U}tn&VkbEK zhmK;tLB}!jC!_N(yJq73ewKgY+`2*xJmE$r8oA##sHtn${~LRK^eC3SNCJW%xT;@9 z17T-apU{?gE}$MbO_9=f92t^hf^Hr%H7R4z1AD;qJofoPWtr|=82eaTqY`<)Saf|ujA<-N1 zzk=Y_giXDLI641B8UAzO+)e3jq9L{YSD&Gwafycf8siDhiZ)r}}rtu|N^G6FjF7GF6!8oA*)*_7Hjjb*Rod^8eCq`3*&VEpm|gOeEkkQxW2kt)34)@y}gM) zy-kO{4F7MJ{6A+oiS*koG1=Q|ilwSA$ET;zm&Vm)S8Z0^KPZL0PVg+@>HjN-rtgDL zynApwtIDU!&mU~Y`FFvj5py6=W#a4ZehOBI{J%KwOrV@86b8{?+iPi=%C#AZwUj|_Gn;=KKho% z@nzd>Rs`j7xmhYu_`jx0EKZECL|K=W7o_4U#C!7YGai550=D1PkIu17E4^vmrDyN0CtVOEva`A$a#_q%S zHh~5WW}u*Zi}OiI{O=z?o>=`W4K5b=$%Ic<%Cp|*fs7DOBBYv?*BwY~R&dkh|Cl)h zqD&V|GiZT~S={4DbiRa>&FxL#?5|_le zHGF^U>R`y_fj?ZL`1E3$tP&3E@mw(!e$WJ+r@wtVfPY`LrAoJjOH@=-&-M9p7|)c$ zp2Jn!(}DTs(lG7X(KM#dZBM6vpNf?GeE4W$<86^XAM$5*#~WSrKj~`*hi6e^=?ljq zij3iHq%m~?+lGvm#i_qHwxQG-BldV7EhXUWiV;`u41;-`KY4&P`u9x1-mYY=zHg(P zmR{hD9!{#zOwBN@`mSyan~Kc}68jJ=uQP7K#Bi5nv#pH@o~BMZEDTm}T6)mjvW?Uo z;j7IaVX!hwuNem$>9DJD!+lQL@R*r2uX!wn7HbGE1SP2ClxDhaiYPpNiYiGw>mef_ zjTH`QW%O>w3+h1&`4Jn3;4_lWp`X@!+J>Gcuoix8WpI3)$Uumoga$D@!sO!*L z4mktvt8hViSCImNNw{!Pxk$qyy0w?pe55k4))>S}UBtdiuJAW=fsYNztX^TdQqK$9 zo+ZKR0`yeQp25;9TV@MrbWxB8@p31Z{pxw#TR3c@s@V2rQ{?y_| z>@y9#SFwrWU`-^6l%90C>m{NEExc*&J3a|n5f?*{Ji%P>A`>EWmvEa4?NqjgXpT;h z9qjx`4ggHVe*e@xqKU{Fzcb~pd$-@oTIU^u7<%yqqP=oDUEy0&v#o00}%r` zXjqDBW1G@`fkQ{D+`wqkc0ohnAwllkDrk50pk-$9Ofb1S;0khdvNhi85Aln&W z$Db#5ck~oy2o6zL%*9v`*%KLM6wTvyLooz58!m=^(Vq|;$S&7LKD)7o1T(I?6n($<{9&u*u*IyCbA>0XVe8I-IFuFy6pd`Um3?wM;803mi%4`600K5xqv{aJ6>aC+^c~p@xv?7Nh;ToL~X; zK~544L~c8fSf0y;^Je4QF4z)^jdzH>(eBORY1o`Mt#b zr4;*%C8^uAJJ`bw?^#R1OvBczTb*fKPbiIK>rWkvVFcXzd*Nao&9kk~x^>>%q~FeR z=#`xvUv58@#qc5f5QPAb#l@@xL(#E(YtWvhd$r(k#O&ay@!9T7k{0KCwhgLoPLH!} zQ22F%SY0fxJT@QGjS$*KSf^=Uu16pcfpj>C=4cNd4!L5jVI?EoR_PV6v!2Z7bQ@$? zxmeVlXPdtEaod+J3FVQS?O(~+o*l2YGoGDt-Fe&*R;#Yl(kn*Gdhw~$;&CKM{p!nv zX%4IF76qc``gEvAKwlQoHG^$CYt2@iZkwf{HhrJb{4tXBndYDzYXe6# z&CM?P>}Uw{pdfh`|HgRAm#r!L#qF6gPLfWvkeefwytQPw54D$vir!?&`)%@#2X3^-Wt7jJQ*&^PP9S0=k3tJtuZ^T8awZ5sFAB5^UeQk1x|mPcD>o#0EVT+K6&Y<4 z`*^&h-X-1d50Gk9Kj;%7P5DrUSR7lJ9_`BFj~L+$8Y(;!IU+gP%}&D zi#JVwhh?bDZMQn5bEb2N?Ysf!NK5$b8AqVMGT+HFjD29mNnVy9Qz7!03 zL@fDMKY0j#LX#>o-jd&Xbt_#5vL$9wT;RaWSe_xy&dWW<=DpuRCs`KwHTDl`f%Z|@ z4c;~uSlmJ|bs6{MOS?B5kpVN-A#Z*DB%KzBKWy4zRnG7jzQcL?RX7Z*;z*{BS>h?K z#JFPfb_FWk4xf?kiA&$~>sx(V{HQY9xs$qC(NIzsZ_>h?^xD+b2^8F`c4f9f{8`?7 zo5{XiK21E@g(ni%FSQ!<-pUz2W_7t#n7#fq;cCQICx7n)TX&+x_t+iV?za%>^!dJ!^47Myq<{ZV46#+bRJ2@qu&aZvKOl+w z&4?XN>%QDKLWtMdQfEW0hfPQEj4(yhk6Qg9al)oV6zVq%Xae<-nyND-4gPSfG}7-|b1)k&8CB(VOR%@1WMwo}OA^ahzSbL7Kb(u( z#c2sYv)+`rUf#EJ_i%U8zb#q&eCbG6v+YBsWSEsajnq8prjr9C(?|!ol9uYj$>=Vy zmlt=i8Ev)wafx1@s15mWNIa2-`d_Gx04a--p@PVx&EJX!9SIizi}q*SUzdOd4FEdb zw;)<3Wn)G9hft@x3&DJrw1siOj5g7VH|Od}JR9g)TXse`1f=IvgX#EEaJ3Ir6Wx$S zc-BL-V4a>(YP$_&{ zBWz;N&X%inGmuP^c5-7q?!wFRfb_xchY_u$4c6O`WW;q6E=E<%Q^jV%$S)0_8`d0P zMY}TWa3`V+T{<}uE9F*Sx8bv)#y6l2BG`4m%(%_s02@_WAh~ z)8Eb395cx{MJS!+x!c)6`Lja%x_ry5o|y~3fj}p2o|X8;+-djyrhiO)X#Fc5aEWx1 zvUgOIkNdB4sgXCw6InBkqmZmj&cK(=%-@DCwAG%++`n_JYV~Cs-NT>G@NZf>IWu5f zw{12@VbRJaY-uUggXdhMlErR>Ig0hd73SlNzD9pqaz$^4p1CQzlwZ6w-^AzZbKb>oA8&=dd7b%!&II~8=BsE$c)xvMP5 zRvE9UIXiZgC+<3_vGOMR@xXN9?(Aq=k*siRO52hpHv9-`b32^*o2oR;)_f(Q|KqKF ztExoQ-cv5h*LYxm-T%R%u;3 zeR%^vTsajmv|7w3g3K=rn&p^Li%JAnmJcs)l?uDmnqs0NR@kK+e&oyZuieTw`I-9r z^l;Fx3nj6QvR`vre7udJ;aP%gD6b}?qAS|cuFJHKKS}&z2H4BC^{o{Wz`wcd z`g#y@N~rWYDgu$;AiwdOa2KW+8?p91r2XB%n+V2!=KG_!l>{HNpm8LkWRLvDYbzp0 z1)y84(`B+#mFrEMxmrI~4}PhfE~HR-o*le;Kd5uspx>RYeLEP(AjtwK1xb+s(Z-1D zg*}Gd)upgZ!S*?pLC(YCk6T?+EI?H2?a9t1HR9Tt(y5z$yzv5}fjy9n^dEPl;du>? z&B>+_E@nD!9v50e5KMd>w*2U6BY=-JE?C9Ml|jg#rj9K-Ot;vVdY3SoIelV}g9dWC zxL2%0s9OvjZrt9?YyUE31q%}6e6LLQr|4b0M&%d8lO~j}?;s40$XnBf0<8hgn}iU6 zIu2k&AMUS4LgRSbO2fYii10qr+VOhK?|R;+w4#$e-!73%n!x(>w|I`|nqd`hQl7=x zAISKW+d($XY$J{(Jl6?s7Z%!byE^u4^S2<1>~R(HfxG$?z5_7qPx93j?pJT8o)ygY zy@IL@aI#zuK?3LxbJ&LS!T&Y1Pl0)cxS}z1wA}Pz(Vr! zhOy3vTE)a0gXsy-e5&hNV+^IoRPdRzJE?uLqPo$E7RSt!d!t&$gwarqFz7kUy zW%NX&qGCNe8mAU=t3e=+qK(~AnC9b0{=Ks;UqkrS%aqgWkjxN;32d@4#6q(e^sxLf z919_FF!qZ3J#&uLQpM|-4qf~i#Wd~`HZ(XcHBUgxE zeDHLMp+EhuvVO_*B19;@^oSimiVxbZVX1m}Hxx`~z^q0T!R>E4YHor&GnP+3Qv| z8kpXiT^GO>CWwEEAdk>#UiaKxlE7oQyfH~5puPy2!(->$QZMJaFehWn1Iyo@E`VM) z)*8HT0W(ajTVuEvs~@v6Cxkt}y|j?+%H?-SrN$=`U|Bt?G%ca-ulSHfEADLx>NIqV za4ckYik|J9fo$1)dSPY(?2Cx*H2^w{6RHwZzfG}UG3u$PYSOCQFXwx;x$>!4^t15t z5liW>-R7_rJ_oJjlkOfof$#i`On?9mf$h8xeR;9q*l*&5 zdC}i_gtD_ulXjs8M9$|Yt{J$Z2*wY!L|@T@>;}1o4g|%) zppR;vyJ5#b;t){#DDM&^ia(AnoF3qJg)o`AbrNubq9{!6!$AaYBu(^)jL-r5!L;!S z?86^;;H|#F15>C9z<4ixWnGoI-oaraU5_!5=pW-2yQ@f$HIA&DN=?pAfC@@{*~`ro zF;#l)h{48A7xjp$;j>XGSF8>Az^<%TYot_sjr!Cf4g^Xh<^r@%T3L;k*-!m}J5zak z=k{{w3dy_QEbjiXCS6jK)O${6(cF`xj=}obKI}nGV~*FVyoX6B*KP!rr3Kc^EiCNI z7~uN!l!M~6lN#`w0&ub75PDm1bgaRN!y3 zMKkT$=}k8pr4_WxsSu){`N22xjG zx=L)eI3ibeDptdb)f{+uF<(%MI234c((z+t@MSfe*GE=bW1+MUmo;t#NG<5I>T0Y% zWwuTM+iy56K8(1a=;>$u;;PQU6P0%ObG6agni*Kiw&}ykgP3)u<+B5TT-nqQBSDXs zRhp0O-iK2(dD4atm4UwO*1YUE#7n!Q`&ed~zv>rLzy9I{w}1SX9rCe47KV9`pN36S z8rMEF^UcGc5+l-AvMF8%bG2m5c2_d1;ySHP7}3!((=V_^3Fh^iw{?upkXG{AIsRua z&`(=$d`I~waAk#nb*~{J$ZXK=lNzh}SOLMdGNxAZ$w$ztxEO^ny~t2oQ+vN~lPJhc zm#VUp$;c93-JWuJlwFwZGY!+zM#^`78=ht9*UiN0LK7ku{)x7(oD{_y{lRgKG_Z2%^2RU9k*D_UxxQPr zPude{nOc^;EJL8E4}rXO+iU^Py??UcCD8yN7#4a zIieC8_;%|p*BQklL?iR$-m*|*zy=*>voPm(TIes|4{5SqMBxSV%hnPnZp1j&pcn-dK(G$oSt<9EP>MU<08Y>2a1W(kSyf&iMqJXwzw@ zAPrJ3-H_TW7A(!@p~YL#`#4z}!rU-+?XF#OoFb~(GS4gg70_H2uo|Cwk}HUY!Xkdl z`lL#|<-NQbMRfcr9UZ6w-`nNo^jXK9uWI~saX$KF1MajsVsQQ3dzS70;qJYon)Av90? zz2iOK`Mz_`IOmUh#~tJT!5HkFwfCB<%{kYa&wMuL)1BWO&7G3#8*gVg7xUFk9_)=a z=)vzK5MOKQ%eL{{k;7yMt4^2zNGIlWCzx+DJc8}FrNf4c1w*gMz~f?{%WVwjN#Tbn z-9p_)dC=}0f= zdg@4tS7G*3WfT%!-Q_w3T;}HHL?V+%?_ZZD)APl9y=&;e=bYh;8!aux!ljE<&&uamxM54hzxQ>pI0cgyeW#4uU4i1s}A)NPffzt*~6lA2T2UQ3Wx z??;3V*gW{hyYiRB>02?#_4dYSQ867SCycWdRPPJpaPkZ*V`I{2KfP6M+{gm44X*rK z<}aU3=GOtv&~2NUQFi$HaMzO3hY8!E8h~$wM0w{my;~0moB2fO3(-HJy%}r%bY@3k zGr&u`?Pytb{fme6Xr0YCR~o`Z-Sy^wMJ zz}l#M8%QL^C6@as|@Ub~8)KE12|`|Ba%v~lF@7(;rt zuVM(zI@}I^BGK(+l>B^MbRGm0!=ZzR6T2fTUobv6iD2CX4!eju`y~?@SOg2(1FejN zcTIfy?`qN(w&g2<58LDH5(O$=aCtKKvMcuZQle^YalB4%3*NpLr%`<%>{vg?biUV7 zX3}wm~hN&GokPWy-_X8=d#Ir0g}aL0FR6MlZQtZU2ONz^Gw8 zdia8ds>H}^jo0;}rO|n=Et*?>m3V~d3VqCVt-`R@>fMib(XvNdlPy42B4pTWb4*z= z=YZqIi~fo^T=5@v-wB8{fJD;mG*_#`ZGEpGT$&ACVMH9;tMAI5Vr792<8=%hL#f=4 zelu~l|52jmXlmHKCWYVDf-QncYP|ZXC0)hFEQy}W+8Fx&1e5wN8u#~(_e+c$ zbyy@EULsqLNDLd?6)DlMMmqsV|Q>ru~j(xdd zEf?AKMU`AxU^KEZ<^>US0$r^=`{O1IAQo){ zS*pZ&tM4)|y0;TZqyl0h)linlQw& zx+PSUsF2}RQZ`!=A=O91#$BZi!4*JMF_FS3PMcY-y!0j@97!O*>b|oeGjY>CN-#aI z-L1g(r{wl*Z7z7}v^u2rvFGPp#o#QU(+n+305@<8-aY*DYUuX7E!0QZrEq?-F_4C+ zUV5_Wmzp*s8@S#Iv!U6}As*aUt}vA8f(41wv$f3fsjK5u#wB=ZaDQ!3^K|z)H3$DW zCm>p5C+9UE0_0louKev(m2SezR5GIu*Ek8pHN9%{aAg|Bk!CM+#yLE6uGYF$A|}ez zj#zPY_sX}|xh=CfiGUaTrG)G z`!w?d`h!6!IfTfRRY*Fw2OWvuDC`2>`MNi+Q^5lTV7W^ksclvm=(aO>o6aI0& z^dGV_R0a*F@?j5`q(|?LYP{*K6sz(jE}-USfZJ%UT>l76v6eSxfI)n1Qd`>VnBS}w zUlc?*fUi?-EfzA8Sd5&ILsN<6ss}ym`>luTc z+=y=xG;7su^Yd71*D#SZNmgCIX;gn|pwDliHdkl+d$zb$g78t+_jiPazmooC-A4<9 z3oD}8d17-07Z}M0P&Lk#J6&RYUf}{y)1cXg-_4h~4F2jFv9n>JUu+I94%EZm#Pn%p zU#YPoz}U23gZ?(x`(T|ypZ|>FL!VY`%T{@mn$|;uDtXLI=O-$PIF*BFvkzt$t z{n9^*S~2=A58r(MBq5_+)pF52u}cIDdVaoW*SuP+RWF!(!8}+U;!du6ZN5p%NUFMtU zBDz&6gCV$Nh!)!+=XXwQdqeNHNL!-JP2^oWFCnf^eZK;nCL_PV4 z>+|l({@V5;dkhhFk@{E<#ETwE4P9DP;CaAK^b`hCQQmddXOb`P2<3U?W209eywnkE zyZfR)X|@#r{@ZL)1laW&cz0}-`I?%*V|{ThjQT3I3DHJAruCgI%VroM4#Uk9{i9huMTHb)oj;86EfhIG=YyB2~EEMMJY6e!V3aL8HB-Znj z-Q5hQy1wNlf?xmsUjWah_41e)GRtl&zf(|Ri%hExD3#cMxz)d}s$#!NNq*PI_C16pCkFTc z{^;88VgrlPSZ#;Pi&<`Xktq`XhnkojGn7ql0q}LMkeKsoHkTT`-QmHZ%Xs%K`)I$S`!rFK(AJrt|e*oy`TPvQ*g?<;j6g)Ojs4NPP=`V{nxs-;{6i0Z@|PZwEMD#7@yPp zwS}W<0FzVzPq?v>Zq?gc2phTMOkGx%lzKrOk{?ty>nHYo3KRWVT&gcpsbg~?9~{QX zek1RH++(@_Ny;_X3kjjJ<=R}d?1_!HJE?@E;XW+%}j7S=19{K>-oQ{p<7TYhKA- z(r@E@*5A#`fC-*1F*fV{7Ob#Id=8)_b4m(LyypM(sG zrnzbIFm&G4nEGaUScejDrspUKPB(AZJ-9(%;o!gZbkflWGvf4I)C9VgF9Wq6basM} zx|=Xn?UujwK4MY)R3+j3^f5V{-)=rzL^HQNp6rnw65@8l%ruj@mNz*${%XSXo{N^+ z_*tH{=)872&IoxsXkezhFs59Y)ku>O7*xFc3OW*6|Tz6YElIYo6DhH0&piL z6VKpDv1q6BSD=X)Wyhy6OyuNn!;W16>(Pluc$Nmg5prZ|ik zfx>ynms!<1`1&>@X}CNICqxoX{((nDY65`gF+QV1QlIXePPMrN0t)Q}a;F_XG0uwb z=a!EWUKvRLocV#KDu09dUW$jadG|da{#9ls*}@un0kIU=PA1ft5aYF>U7u4;6E}2zZk`}# zaqgM8&-^6R8fnly8XZ54_yO$ZY3*X&%<^CXtQ!krv zy$&%s+4R)1Yqx#m3_`@93)J9^$UbSMA;U{`^n!!PD0!jiq!}i7Uxa%%wzmqKy$a&M zTZ>FK4=kjm#sFlNSf(eF`on=Cdolhwp&k%`k>v0c<`LRt$%dmo^$DX4z%ag@4>9d~ zXoIehXpIB9n%JpaWI(fk(j;Nk+6eyy<|QxQM_6pw?pNg5dW6PImdGe& zN%j&}buf~Q&~jr#7xa1YlN>h)-V$`0J#(0Ez$=RuGFx$8`0fINjlfdgfM>M5e0;P6 zG5!3uJI`#noR}rIE&|i)@&K7-BAqf@NTbozvsmn0o42u=g4kB&+0ssaI#A9OXC5@t z9{XoAM{c9M;w-&;-8|o*%5*(PVlFrnkn3d~=VC$#LmZ>^`}u()PIBHNm=ROE&XY%W z2&rB|7g^Pj*|+v7Z`BfQ0L&|-eJ|4aNUHkxQemF&s6|WH+;7z4oT8A0uIo%(nZqcM zLaljT0{;kXD;R{Ou1%SX#ViJ|9wCo7PX&gcw$w6G-O%(~%<5dVTJTk>r4=$o{*Ox?v9^_diGi^7HxV2K}L z&k^aH4?M;my+Z{Lu4ZNeb!WPqe6r3j#sBF?Agkn%exz6S<KRAI$yF9;s4JDOKTfICgB za^_~!aCrCiUhL$^$v&&U(@@ntv|OOl!>zj%i)di z5(tGbCr-hsJU^dZ_Z5(EHTr7a3>dFQGf)iKKpw|>A`5NgXvjoi-7j%QBW5t!rJhk&K=%ndoT(>FfTdgPM?CNkkf$C`&zc z3uymiKfLnRj=W=*zv*2WCDDWzJ1{NN8DG5Y7AV2r92e(O&(Wd!B+kcrY3R}D6{7=l zrAacCNV&f&sO>B)@*j!ry2wiZy%FTlweFmsUj4}Ni&wFKI|3}u`%1=vqkf=mlsLLa zQ3z4^K^9hVjGvx9&r=dMfqLz8DK~Voh;7+18Lk-*(e!}0=bz!{R7aWDc7YL7QJ$A) zW;jKq{sY@`v`kP)$goqIOBD0g`NRIEXWD4)>gHhl=n6z{@<`^0-96!-fmOe}cn6X- z9n<}*OrZ!6>~86k*hk-0xPxx{RTru@3Jh3WR`$F>zk1Z&txseGCR?wt?34c;z#`Q`L7wa3(dwmeoabzS zViW@p0(T1YfTQ!PKJiFc93o`HKc3tsgkn}c(7sO;YIgSf^dOjB(?DLhWdi65BEFG% zEol8Xhx5yZ9`7mS$a?fhP|23-vDl4GMdmBNMglM#_eN%U!9If$ZWa{;w>}V)1ygLN z)^WMp$oGn~<wNG8wOM`gs!oqY|J~lI;ojEJ$+=|vUemxw3O%%gK3gNr5F%Ky2r(L zN7mkf(iG3*O4Cz?cUozbG>Z!xzcNX*)!|CA7E);cfMDwK{U#hYSQZ`f$5~z;>d=mt zUGh>m@#=seQ~kB(!C0c9l;{9V^#UhNG2)nhNaHB5)fBP*gHlbnp0H_TwCm*6mKLzM zXtNMr;o-z;dWH!JT@KOGlD4nFf5XhKU+(?sn(@}g4^C6>`0RXXTp#u{gE=fII1S?< zoWl^enMrT0t7-}tf?IElIF@s2MZD%hP2kfD(^WjqzN73LAxNKWQy!#DT>gG8+rl*M|>py=mm%5>Wb6RM?oiuX*77P0bVG zfR*F9dQStLf7C-6KkQ~*O1(?{h~!|hJbW-Y@EQ%o7to-Eq7fAxsstiAI81Q3V;55= zhwxC}(E$ExG_Na+UUaLhIic1m_ldxny?{`msghMRV#gFr^gd)~>XT?vNO$%-5$i*eFnE#1fb?ac(#42Oxy(05n{Xh zPV$(XmCM+f_7K$J#qq}t38!GGtsF$7)Y|0&w)N8nvyVA!N8F4vnY55m%Hbih%Z|o! zd(+;U*cdT1BMCePEmoxPp^Wr?)Rc zinCy;$+jn7Dq@Gq?HbJzB2kA`+Gn)DqMzt`&ZZ{5x_yPUKngf=dY_rgufpxB&pn4; zT*KCE%rX0}^+Dgehb!9Qk{2KRHS?OgnMgCUFMyY-q^qXYD-oeCkYrAG)<6GjFtcSDRRBeOQXnMk;{M~TH zS>?x)&*Z4Z7n_q{z-RX=OF}y}!45-{YI%`zl422yGA&;c-=K7qme@!;7f(>JzHNSW zW=h)God?ohVNH6m%$(gBzHuvx0k=AG(->U7f+A%KTUgL`EBD!EZ(wC&YtvQ`p7Qssa96cbFmxMl!QB#S zs0LXWetz`=5n=aV_N&s5hh_zvv72X6sI7WINl?upd<$L7F%hoq)Et_r^cH=EpC`FT-vmu zpJ#9pu?2@B^KC^EZUA$zHp>k5{y0kO!z&6o(5K!$;2bbmXhnNCJCg2BsZ)XJ#|g_r zFC@+AX%+>NbR|gC1fXp#x{f(rz)UN9S6YRquxkdFJ;BOl0jobLX`RZyH(#o2aat8S ztg@r(?N|v%M;qCW>`n+J>EJ@dPt$%;wVl0I%fb)-65z*QRSbJ~;>2OS6ZEM(8)_-{ zbLv_PZk@zh<3<0DzuHA~Kpugdy|2;R&pRHm9{s`blI`SiM2N2wKO3JtRg9ELFF0x^z=4t{y*Iv}>NJ)yF(Df#lkVF8MtA^?RGO%7;}B zrb{du|-XTTj!l2D#s4r)|2@t3wwD!Z+0 z#?|}-oXwn-FMC4iRXQSLF$yR2Q$&}K*MIL8&*9-?Uiqn1Wf7E2|LaQ0Dlv&quF&(i zmh%nn!;Vnl3%9&JxOgX6&`iKWi5SJQ?`JY4Lb=G`J<(Lold1YHEuwo82cevs^ctV( zhnc)e5fTuqdkoYpz?BUKJG08WFa9{=eSJV7dm3qIyVoDG{3I=K$t9YMnsmKh%viAr`4%6$ zRVYhe+%xN8bgv64ZZ1&6kfnRekUYNO9orupUWB9fJyaZ>{6YuW2qb~ntck6i=$-tp zg&)*t#Qe{NuLFN7Z##jLxUa9_Wj3J*SspBWwElH$dFXsn>Zrsy}3@N)BmJhkOE^w-{Xm7pDi z9f=21Ijc3@x|is}DI|7miPj8R%!}o|McUO07|d(1y_=lQn^fb$qO>cSkljwTLUNI< zJIvVmixXj84zDCLYe`)J2cw|Q?{DpJ{lR?F_|7TCsgbkLlk$ch8j$zf3i0aE*15xS zi>t4bmLG}qYTAB;3G>0`JW}Z+Yp^iKUBPTC`3!{2=Ld3?2ALn}I`&;q+nP4NPk=oo zGVVpP?qiYq60t%?@$GW63|8{O4Q}&J=VjlY-&^rDJqHDN276bxvM%NTrjSiCBZxGr z4+8Q@kw~RG(lu~-xQExt^t9d8h%pW`%8lD8HT2spoK&LuW%)ifrs@hEimgZl_VTlX z+?Cr#r~ABxJJ;To1or{5u0#6B68pN3$tW!>oBZ3al`SV@e`vRL3p6FEJ~l~*Kqs0M zG)+L^C*tJslhm6_iPx`W{#G+-jL&H7qq!`58+pli#xY&QK9bQ8`>9Dw(+tA5)^B;G zlH_!ZXDX#3$W$-6-qK3jc$1!GJ_t!ZTuy!tdX)t-yEzqrM5->{!uVUfc1}SWdYJDOL^I zaC?V$x7p-oGL;saYR((^nvrsT^LJ7EIrqq!@l+n!{%fp-w=^@FqbgsJNIEvwJby#< z0X@eY@4@2Y?Q+fz4?DN99_Dc9`z}9Re+e$uh3ZGjbH}XA9X-)kKAs8@pWo$G$w@vl zY1$c-8=Gh=)j~?=1w&bsP`MHVn5VDjM#SIPXPMq;dZtJ}i zjhB(T?_;T{)@tpmU|K{4y1mnUWpq^oJevGdAF@lveqPHTQS1?DxL1J5$}(cS>-RRo}R?)lbE`vuVReck*dl;{6jh?+0&o@t>*#i}n6SHgM4c*N6e}*@+~X z8vIug7xR8R&hFr#5KniZ#YL+;H(>|)2A9wFpx-@xU`joj4?W?MSScWSgr1z-fbJ>n z(A5uQH3a&_?{eXJQFdKtOE^py4HH5tvhz9-DSim$O&TSG+!y~n78$9YV0pl}gu_+L zFu5-5uK$Rx#-K#Lmc7ZeczQYw3(=+M{QWN82;Gbm{R2d(XLJuh`a;xT9Z|rf0Z-g; zoKt{9mdD1+lVLTipf7If#P{zp<9W-08E;01Ijk!jJ7hL!>Nr;$8)G-;DM5@!dd3sB z_;9?F`FqqziWYFy{>RfM=!j{?27S@j-qCnVoh5`l5Iy-^JF|Z)cTjpHoA6;^TY;*= zDmmBz8F~ZOz90SO7~&U~LESt(DL?dF`K6X!IgmpKWJP^Txi2m+yxvvHDPPH-58Hb9 z{ZGA7IQ=8OG+TmP%s~~ofh5ZztYQ)ds%J|Gs;Ym+Y5-#+ucgeFv;na+wf-h}hv)IN!F zD<>;dNax7!eBt)MD7xonGIAyf4Y9JaXa1+Yzg5p77f0Kv`C3Bu0R0?Wl1fUR%}T=S zp8SGS@#_b=hq88NL_pc71vu%F<=%zDC6p*ui zNzy-+E&Is4^Qg{_a=sCvH@ABjg|N`wKq`((<2jW@oZ(pW8=%w*IDU8(20B?58t zOX3rfK|b+2ZAUyvmlWg9SX-%u4(7Z;X^0uQW*l&*?@}_`iH2G~e$O4$**cPUc|{DZ z_>|saO??svFvbnrq8}?Ub2T{?jmw>{LzA)IC*G7WJtt1 zCWnm5@)>Adr?w_;d)25L4nK|#KQeI3eJ82u`E$}&QOi5NJI*#0A&cEGRr08zTf|B1 zc!qxSNW>PK3cZyPCZ@rySu+vFa@%VA{2nT&+EHz5|1xi-G<(@6RH!TU+5-VUV^Wme zy}v7%2bk%VQfn<>~V{@>I0$ix=12-T8g!aZ5oT!_w|KLTm#!y@A8nti0adZQb~G+3I4D zsbJ40aiLTWFVX%W?%&#e6194Vb30vYiZ3#>Bq_@!qqB`i)TJHM`urZYHKQ7>f#`OG zt$n{Xm1pW`lEE34GE@M7e%^5BNYk{5A}rXBn!_ODfJvMxul#t}9x*COt}lXxGJWT4 zO(0Z#Yc#bhKi_X8@MAC4iLH#F5Ua;GjdpDRvo=cc$mk%&)z{bKvifo!U}jsS65o;Q zbg8+og;4pAOv)3yqnBF6)|u^xLY%8(X+J_aWn|v!#$6~T4!{}2n8kQ72_+k%^SL>6 zQ&c}JZw>es`C3@_l<@kio(Q58Yd&04I3|T5>0a2&tmz5r_V)`P-1jQOJR`axXXIk% z)0ra{py=7~!J<4VwO$IW;##7&v=M9pA>TA==S7dx7M5IQ>lzP3?`n`=avX zqbb0ti!846T{*%|WW6zyh&zi||cGdp;{ELmS% zSnfy$HO0^3KDb@g`f-K0$2;xTf;+#pcRnF!?cS4caHh({+-6yREJMxafBCXmP}vS& zM)u@7y@D;rt8|y|!8$t~Wo>sr6IGEhiQ*NG0!Y`#pKYiCbMuTd3CL9%!%%0EZD?CT zkFj6#B*m_aI-y5sA=c_h%e9I?zpOR-*3LmiP5qvajo-}eiVE#xR zr@*gwUDPv3d`&IEMkgi)p)cRAhzG?VpRp#$zS|L?OiCZF)kJ;s0-gO6!cymypnhoaqlux zeKQzLAUP>0jm`{Sv0N8D&$2(>avJd~qkJ!1L z()xj7vjI=7j;P?9RcUivVDCl?G<`I**q~J1rP-8EUTAn}SCs4%1o#_!sS+ux($rcz zXWoOg0hQL?(e2tMo)Fv;4ahCtIe);pJ1Y3?xMvL{9O?-HZ06Ng6*Vo1p#oL|XZG^p zIGtk7N=IW@R?ICEBV31GgwD^T2hEn>WWMy?PeUrjm)ED)23b3O#I=sWhEsw2~6YbkiS^OuCi*V7)<|od&NTR zUTTW{lpZV|%WyDz?{vCWWr(-ruuZ72e&{QpwDz6y_!Gb}FAjK(cEOwJVj-k$(bM~r z$jvc=N{=9;EK8#tQol*)2jQ+^VcSR zi|@SckW5X@nTx1lsanonx2QFf)>Ux^e9;JbC>-!v>&%t@1&Jb%S^7d1SA5{hsv<9a zTr*#IpMyHlb1?$S_vskcKKyBXJeJ65{Ro`h-E|4f$dUWR<8_x6^x?+!kbM@Bze@5{ zmA64uP&WPq$xkl*dzqia#}UZfJ-ctsw0J86lyM7nM#e&Q(rLKM%DMN{T5kECPq;d3 z{(b9{qz@V#^=y>=i&`TIPX>UzYXP>O+DDBz(Fr zmFcTcOy(dFVhG0vu0O`re{>6ot1YzP383xV<>AT1A==+lv*SqLFhG0HyQ88xQW^n8 z+GeNPq2mvUlzNgJH;b^#Gu243Oh^{AE}XhEKdqbpXv`z(Ru!j#pB1v7f60ZdY`F}O zA*z^FyS8NW!-z2%bYVkfmI4EE81CPo=T(3*c|j`e+<2d>n?DPDG*cOyr{?p~k@;w$ z&Ag_~i<^Hs^p`<%DNN<{X* zLhcn3m;OgrH%XDei~EIf9_%suZla9|a|i*cJkv>q#ZpbsX>bN1B5`*PuEx_{>&4#O zqe_fNY_Pm99n`_MSZm_=ZZ+7AgrjZ!RYu;S0Ow;Xsv+iw*Tew9P}m=gdJ_S%bIYQC zJ@ig^6L@af$3-mS7=KRQa1Ea4xA3pcXnttgMILcia}jqxoIUac`1-YqCEB&Sso7cj z1B-(?+j!I;!rnUO`cIe^&;{7t?NY3Sz`X%37p&D`tK2&tbazMi4`wgf0b0Ap2 z2VPVSf|(#VfIHy%Jp8YyMEBVa5BbUL4f}=sx#T`!A0p$+l3${}9tm4df7JoMc8LFJ zyPV2Bfxe1zQKI7D!wd7Flr)bwm6v$?;RwCK6MuE$oxMycKyAF(6(XhCSYK18UxS%&U^&4b^fb#G^K^I~Rh>BM0sAevu*-YVL8g&)TU)(BXP0xzF8Ej=GF9G$p^7?)o|Xj*2~q=1$z$ zg>+4^A-(~7VzY)0!qWmW(gwC@^7_}tBSrOxfgqxj#0UYXF+TiO{R}e= zgXh}#kq5m9Ol#w;y*v{rB}$v^kK`a^HI@P%*Rm!z*XCx@Xw1t;QWPSeG!wOmhqEFm zmZHJY_M2F0a}sKQSeV-__M;<3MBpCm(2VEsy?K?aDIjvQ6HN!vGf zgc~{gz2N7gg?G*u?R8rStE|nezAC+{FGA>bXr^i1d}AUwbJfHS>7#We^~VM#*d#A~ z<8MEA=0FFfM_G5rDR0$TKe>`4--D;;usl&cw#v-$a3M=NuHm;Jx({W?Jvb2Ydka@( z45bnilhZwxZR~=b-uBwT!3(<^UQ&@O+7Ooo3lflLSf9T;PyJCg6KdNBCinT#(zZxP z2Z7$0`_s*GOol9i%Icab?Q|K7W>#O2S19QgtIJs&-(4}w0>ZteR2bP`7vmkrzg-ON zz}^vSz9tVR>jny3COqtcF0&l8Tb_XWjd#@@;*eycygvjS8Re@7q7ctpt?D?~OD`zu2 zki@qlmp1Txnp38}W@|wr& z*$_#nmvFjo9TP%*G#_K^wq==CH3W0#mZ5Y*hB+QnZmWb;8wzI`3lrH7>X) zQ?Sj{{X z&l^hs74B?|WT=M>M;e9Jr~hOd@=d+*eJl0;G=I0R)@jxXkq-5&kq8}4d4uo(3E=iF z1W+yPM#pPbnf*f^VMmNpvO+}f{TXiJXSHr$sra9eZ91p7Rj!y&aw6ya1~j_5UQV*~ zr@^lqj%;DOeOhx>33;ZiNQ_>fLGG)+cwl5Go7c)|K)GPhEiWvr*>v}s=h`3uFxAY4 zIk{n1wf*{GxpRg$Yj>!`fvl%Y)y!Gro-uApZ2VJ%K45B~pK!L@Q8_5HHU5)EA>hbz zUY)a!_C zo^<@8so!iR@aIU)WSM?DzaXpePhahX@T=tB`SPC_yq^)1Ql2m~F$|h-qC5%<;~X?Nt)*LtQH)HVT=^(s_Hg0x^YzuTojWpJUk)>xeA#lt;uaXesY6(-;RT91{QMDDSsl6pgs;Raj8FZv2g- z(15d%?=5ZBPmV&~>D=$UUS&8MVhjT=FN?W5t}xww*2|IbMd{}*`>|L+ws&S=|Cp`rR7^j@`% z$fo>xpwT=uf5hlg#x2s?ar zW?&<@HCJX`YC;7*#QYL|JWn3@p7yv)A(~OBhN1mB^2PISXPRJ8ZveL0R8Sqs=#h-& zkc&KX56^upAWKIV(inoK5N>vF9b?YL7V`3oDc+42W6>!cd*qJ3o#k{E^9L5(Vh zH{*6VKBYF=4Mcmew@peQP5ZNy*R%9ymwrEU)6Y%(zT?=jV}V8Rd0yzeZ}>WkpjsW= zEM+<#dmo`F9?MslUldxDFJZEB13Rjq5+S;MS;e$u;#!2eP+^u_51BW~8%U>d(nRoD ze)gG6EgdsqlB`qGtH2@e3>_2HCqmby^OzMge&K2CHpfc+Le`N}IIvyY^%L+~-kIE+ z`$$nGg$zkN&G+ApP1*)>1{a?BJHK61)`F%Ken=L%*r+@K36qXJU*H(01 zn4xC9q|&CwCi)YH`eyr1jE$`+IMTzZNQV31bzumg97e)G;WQCjV#y$Y6H1_*xnPZs za#nk;hdF=sCifUmWjp(o{I_!mImPzrz3vFAjl8Pq%1i}om%y9PX|bGa&#PG!dDs_V zmnZKYx3jZOgrB`Q`0>iZ=T7SeO~_YL+ovj*&OEb zGa~ItYp~}7GcuT0mDtVLyAhon3v<8KMxFnBH0Vn}r|wtp&H0oC<4Cds4=D#ozvjIT znEPVq&Kq^G_}h?}6dk?P4rSd9QvWV-Kl_Kbe-N85sBX05s?>F}y-8k$jUU!nGe;uYFl2E<6}>n!uV_YgJc7a!KW@6; z-+itW=*z*VdhzGVd2!W^mWTHqm)#@%_>s#mlJN^;Di{^KL6_-kUeINrDK9K8-jMn= z>PyI+L$PXEVrH5{H25LY!?GI>f8MJK2?-t9-r25ctZ8FaKLQoNqG6o zmtU=wG&B^#oGs)0ZV$Weyx4HXZTFm` z+|Eqcl^>aCVr|2_f<3x#I-m5RLywy2ye{;bX4PD`TauD#j4o}*gR9w6pO>+-lRBXN ze(67NsRA?J`fBM@)Lu)yBX@P=EVlRFoGP0Z1NzSLRR8&s>&5kLSszAy4k630dj_d` z-$YLZN(y|I?v?wrzHBN0>h9Nl8N1tUzD0p(9h_~9#>_z4`Iu$20#`TgAFrl9F?cW& z{Os^e`Ma9Njd>9{lP6{GP6Ln*(x;1d&;NYo9er4Sq;WN3Bn<55pPBgolZHw<#f_ndT!jcLykQ34|Zb&X=bTaq8|?hUjh&JbkTKj%5wE2_XCaFf++n~ zAyvn#%*ImSJ3>64rf$OOkx~ypDc^&Ne9Fi6+1}o-a51U*q8Tr^Gtz3H1pl6DU0j+M ze37AzXEbDG1mAo`xBF-|tj7kmy)(#WwlJKsK1y5J+(XI8TSpred?_5PQad}iUW zu%v<&r-`H+i^*MP*mtWnBb#+zNxN^$Vo35@k%Cqu8*ftGo}T*30-O;e^fIFMH(lV| zYA31DJoP6lzQZ}f5xvAZGQRnFR%&_RAHr*>D znteQzbbe**LjQLw`V6iB&Y0N6{%7g0tvCdCD*C%}&q2iZetu=Nm*-a!HP`=Ryb z2M-Q`!6&#w2sXI8>j1;RZGPvRE6;l0=dSw)tTjE|J-c_;uCD#9Pwh^xY7ud9J++wd zpnD_JP1|vMG|X;FOfE6^UhSTbO;FJf)HQqH)W+*2Mtnq={E6702-=sTf|o4DlB8nN zm?+atS6w0|GJAn^xUAFi@RXQ{bd&)XHhts2cURH~e5_9#A@FTyxHK8)T5|FD#2_Gm|s%odd(ktS9oyok`Is0;z zd!TuH^vM_0?FQN;)G*jk&U`$V;bLqQ@#F(@-gmpNsT}4*v`yzew$%;(LO7XYDTgb2 z-+*qMCu<0kfT7OIF*3(bzZR>i#5L}CqrIsrLKg}EkGUhyOw2nXv zhte(>n=JCvT%}QnU?;yXUZ#m*-{TD%iBUF{ec$5m}^bsK2^uh|_0T(_cv0V*r1`t1AAYw_M| zzw-7vA~yEbx0m}Q7zxLi9m|Gk(unK|GuBztRX+KqK@<2y?NrCXTXSxoPm|a9ZcsHc zeqY|l(W6~%WCAo;rMJ=xSmm!VwR~Q^Wwuu6>$T`%6A4itEc~+jS=YK`7xW@b1+)~g z$i`w#jb$vDqk_8(_?xSmWS)zt*&UTH%_JmawC=&xc2fO^pO?8SEL; z#T!>>@aMfS&Lf!TFtnTY3S^TQVkcIppJ9#4VTq2PIGdz*t~hD-u2d%n`LU7n*Q2BO z@Up>4z)BmOd&r213C3uAbaV??C`M*jmESK+z~$n?V!dk(6=lw7 zHthMSa*geCa_DTLkULf1*0G|gI=MBM0Ex%p)%g7EM(SN!>H}b%EJ4f-quXbpm1YL{ z)74&PcY!OJhoJokJ~nw5n)QJNUqqUYg9N{4#m85RJw`lUFY!df$*Vo6yRDA_S0b6; zcsJ^orgN+Sa?*VA;b68~_8MM8b>69`Ng+YH!5vuLJXho6Wm zjPlWUwlB0c4Dj*5oxI!;i-N;{GRjHAkGVNBWqN|^xWsQIAd$CFOmyF=MCH}UYZob# zy+MXh#_B(B$6;lwrMK^+$jW?@sG1<(kuA?{j5t z+=hT|RA4CMfip(@2V%XVjk_8?@6Ej7chGqf`D_yqgx*^W5n|j0hPnG3)q(2<_L^r} zQmvgwJ#VCI20ZW$T3Lf+LX_9fqi4kWj?VQQLBJ#Xz$#CYbvA7LYg&dQk`y<88oohP zH$y36KizP=$uaZMZDW9JPw2Kbj|LWGD^~aRqAGv9tXf?9=hWHgUiNsiMGeOUZZ|NZ4BSbdq|*v#MxQuw&N}P} zP|khcj`vra&?SA z5Mgcf+}%?If4JN6iNH0_Tzvu%uSo+4WL+~jC> zCUCLm`hebMbCwg#5C|aKpE2^}iIQGSET1=Aocy&8#6Lp`i^?^O!I$d~Mk&ozb{QAj zSQaFOds*4=?wtre(0*lddFYTMzXVI>2!U!slgH*hIZ1%hMYuPom6;P9TI=MSMcSAU z^6A#2oTnb+i5CsA>DWe=pTRxn+PclHJ=X8(LklNB)krY0)E- zc=R$g)YHZ9o?awfYGPY}ra?Y=?BfBKnWzFSDa0DDFD3!;WPP2+7h)qWute41N_f6E z0;9Zc&|%@<9UL3}22>fbDdHpa-DH($)1Fnn9$qEULMUCX6Fr9y?w$#)q_Czx|=vSsx|K{EK zSh7caPK7j0;GJ#XsfRalZ6e33difp*!3K7_PXLK|fc#g?z`ZRQOBQ4Uk$ckp!Im#$ z{!yW`hI(fXtT$t+OgTCpt{}reNPvQrDqe2$)ocHDL!@~eF}Eesv>TX>U`}?(osz`L zG3eu9m2yOHC1>oqa~}55(mO7vE6j4EdL)1Qg`qUV7LUN>GujviE6ETk%LmVw+eyn8A$y=Lx3rXLnequI2^{>OV!6z(q4}09Tu`vb3 z-epClM(-W*qd%ohd`TXE)e7n~<--r)A=PWtoH9RSR-4gFoNvw%dF_4Tv-#=u*fKB& z8WU6`uSR#J(u#Q*ul})7sd&t{Tc7BR2c_<3A&wlyFKW@8kw^t^2dJb2+%xBmZ_xj` zAeC9)DMel%HjQcL$s56H&v|3veEQ-lpxf_}D&Y6)Wstg8E~iog9WBGyQ(_woG}<-P zavvg1j}}llW3#_*I#l!ylc+>LFX|?KU8CUqP}JYmR4~!6Q>vD9#;3ZF)G;un)S4hW zp1dyTwHA?^ZN1vP9q>v-1Zx+4jFy17dTkklNefnJ9{O$gd6)Hd{$&-xT)|Dgi|b0_!ZvP)`2fb! zix$dep+JZd2T1&xaA-^q&QHZM-TWKmlca*n`AcA^Gu+{WS^(@!0KRHSH8L23i=>d*?CT=9D5)xmP z>l#jWd-!~}o)07Q*a`}ZytOwD%Y1zbJ-;UrL`Px@<`%)anGG>)xl}XqsLN&h3l5;y zfZU<%TH}0N#zUI>kc2&!r3+{bVj-+s9BKpW0zi7<~1I>V#g z9Hrb4u>UsAjSi)y7U~pDPN;G)ykqak6DrS@zTy*y^|~!^XuyUFs&;E}1Hz1`eZN5* z%8-)rw$a@?p}^q!(j`>`nifHX9+f?nDpWOm`xl<-Pbb5H$$e2#4|=tkdAHyN}!5_GUYsm z{0;VL-bur|3 zrP#Q9tM;?p7eXX;@wlweCd>u_e1n2P8^r@I{_LknZ@|-3A|kfWVzW{olPVv* z9k>0BK{n_0=If5?X&%R9NwnN1dV!#655APDwGR#kghWJZ;if*NgNc5O)Y{W_tCJM{ z%u|s`s_W8Z4SD{)oJ>JnjWc!VFAy6Uu(-zh(jQ@sRk74AqOw5eUEwU0RH)y5G3Bn)>tPBNUln zDgdr!@kd>|A5|AwPi)sEcc%t(>v=^_qyjyw&*D&LDGzDjKGV}ticm8G;#bu=JR2rL zwi`wt7vLCcu}sJ%uWV;g@{J8bryMG}&a}$7!UC$)LO$AKxDdW#G;rJ!Q%A1;QjzE$ z4>-ydcgz^S2CNqT5YzL~rxa};s4H4xf4|W=jkrysZ+K?$K}Jog)C(4mX^ON*dc^d- z<>rSCr$C1YixwJdrRHL41SN&RHGAe4`{;AWwHC`H>%ml{e(@Ni80^)(H1V(z zf~zuiLqXfR!T5e>*UIF+nRV77E{vUF+RmsG`dYHqH(=!tF0&f{acpT2Yc;1 zrdX38-zqO!rFXv_m5dkZ%mo$Y@2(dbgKX&{C(kN8WQOq*10pabPTPW*JN!EOV2rYCr7$NiVw(`+q@UYD|#s>~)7G*V>rFiIyVX!R`C*F9I`Whu?eB<{()s-ouTGq_}?_zu6It|3Fs=Tds}FX-uY_ za(LtPuw|ci`&}Y4?pycYg{O-0O+sA`KX0wO-zrnm+D{am4dt~SwkzVyNV)(o)^O8p z`^=XTdyK?-db??=f50UQIDj2~ZO6K8-7Jq{GKh8j9!ZkQ6>dv4J3od@LX84TgHx-V z?N~HwehZd+QS8e1-fs8hYf%;YnAEuK-klc&%o&+qXpy$wII!tLp-;Au&W;Ud*ti}3 zysiEcc!sB`MssP?A=4cMtig~K8oK5dJz9})Z zl~o~Zu^j`WPo;dgGXG6Y4vI65L)Vr{MvdLjOe5&pCb|%&i%PT+7YQ$v2|+{0toP-z zpZXlNUwH39la$(MlKGQS8;iOu@Y@;wbHnl1C?4ad0R|s7CGeNYHv;VmwtkO&h3l+Z z5XC)^{NN*rqh8zI7wR9K5exfSZ0O71+Zr_N2)sju*<~@YYif$B(T&)~euL(Cl=}L% zZ3OD}yE~FdPa=wJ9Ca)8%JjVGIkdrwBxaGQN)u;LPOam~8NS7ljN0D)8*R=p=CT^! zh^npSp9_(KDXw@o)xi~w;`<+FYYLwV4ev*$hMTv@Qc)2nN9{LI`f)Y3lqqmY{XA0Y z!9faKpsJuY>Mlf|-r?~o7_Hg~$IuM%0b&Wn;BFbZ#v6o{uu!<4pbO+e=;4{)(sX7pp^C^F}LW zE_%Mz@)*OgPBl?=`G`dcx>b$MYZapB5E0seKg{1QsV%TRjli@=MzW07l1G@6@H;%~ z*vhJr7t{%!7Lzqkd&5uOeQ4sgF4xiBAt2}`5}K14fH}qUdrqX^QG%D?B=YcBH0SmTdc4z27>P%k$B`s#K2DUZi`>nK`!mJ-;tt(H9Co)rl9oGbY1eluykOHDNG1e-_?nvBA9{ev9l5!hTG# zNZoPH#?@0ts3V?VX1gYPizEnO|s{ zOjLb(#tdc^yodqD@{0qO(_q%pJ7*@H7TjM{T*zJeOFbQjkf!{B z5HWk>EDu@nQ@!3Qy@Fd*?<8wcX>NXn9{$l%++W+h^VD=$?hGB>?k1wy#p<)`^czii z;p2*Gu4lB|0m)O(Pmfv>)_2qE?VnH$2;j^%b@CVK0R zS-A<^i>X-1bzGjC4{jrWsY$cRL&364Oe^U@ggwTQuRu^1YbF_rjF67w%^IpIbCDZg4A~u;4%=WF%Kr7^0b+@csMqyI(;X&^X5Y=dR7 ztm8t&D^9?{wtyy~t1=w*%#GuVLNb%>;#YJ-g8{XI*3qf9NYZ|RI+mO37wV#TG+aDeVW$p{wNZFiOSK@D}ioJnG z1{8!g@fo&dnufIW6KWcNt?B57()O&}9~0(2OL5eYXwV1Np7?gfc@m4(a4IJ!@`?w+ z8%UU`z!}`+5^&vgQd5{Laks!8@50^_oWWnP->!LpghXi6p+ik)`ViU|zu1T+8_l(e zw;~N;?H`!Arxkoqr!n0|b8vCo_V=)PeX6t3;UGVNcF-!@*@imfp5{RoVqBpq-{#wV z=cZG;{0h%tP!CM6kUZ8v&Ct$I-nC%4@cJ7ErQBeaRBj9tw-F?SqKPu|Vkz@g!pK9d zOz$Q6U#&0xDCMhr<2b6%wHAbTJ>pS(To1ZTb{j8clpG<-c(# zx$UF7WoyTc+AyQdHv;J32iHYJ*9ZaSe82hwGLoU3Yz(G0htaq`TBW_W(8u|M3(&@l z-8Xv#yhD4mFJ=1v<1&Ssm5E#t;7G#p8OI7Ft+pd2?E+o%-^by3IU<)fz1_dMpM(C* z2W*ix5s)e_UV&YR=Gu!9z*RQ95}LQ-$zp5v1W>W8+q$z_23zDMoQP*-0+nqp&M{7~ zR9%~@b2w*ub6g8(9VTgOo zh`DQv%w}9=d3Nn~cj}LS|2$#RUNgpvib8)G!>++1a3yb4Cx3}GC`3g|no(eGAe0V0 zBF`u;jXTAEw=T4Cg9mZ_)j9yAio9TZI%`&?$?@J-mlooSCQPUJ(6fRt7~!LY?&7{} z5@XoA!Nn0A(gURDC~|&y)OxK?a`J8)?~#+XXW2!^*u;+m+Gt(I2-XYVt=}?i^6|e3 zxqRDcGgYWcMz>9_IXutJ@A?e?MCPhZw20a1~GO^)tC z%C_{Zsgb8Iu#k=SJFA5z=t(5D$)N8IiImGSACMdRTPY%F+}4U!hNYjYd3K()-AKPx3mqiucM zUvZ@$LJ^YTx3+8{_A#bFoKCMYZj&{Vv<)v=e8N-b$mbxD zjORwp1}&n(nIl+BWn%t3oq!N5fVK21Rz{`|kWrr!lM3q?2mqPp0W-L(xCpH$@wvv# zIO(b~9?59$$3a?gV-2@u&x1do&c?R~$}U=8hsA&X&?nZZbNK9n)RFZT@jan8HDXIxvgNTE+Pdr1>SbZ>v8v$ zHhB3xEx-JQ0X8MM9Vh|NVkK#d<=74TX1^6g6Y6MW!`T9)oSU-b<-9jJWcF(gJ_Q0RUxZFs!Raeh> z?a;w(wAz!Ln#U3;@Vg4hJT}cqMjY)Oe~B#rn)qfL#l@{b!RQL0i4kHnzFLY5==hO82TqQ8)-VuBu9i5mSa()D}q={e5iOjZ`>{Z$@Xib*_q=#rt-}T z8#qnNkH((O)IB61tl!o2c|t4%Ge~+60kMTZvf$f3XB5VSoln%E3HMb}*%6t8Kp|Y_ z+r9Xle?wn^8ZlnkJ-j=$b&E;o#x_8|K{)L(r^y*jX7U4vT(Rq0to383Mbz&r9aOmv zT=T`Ap1zcXFsJOpQYbjr5^gEfk8So7%)BK`JVe7pJev6JIj}TxTzER&_VdT-N_+q} zKm@p(nJ6N~DLVonO97ubgq8HYyxNPA)A(8cNqa@=(b`nf`Klg!$NQ4>_hauwA0M#Q zKz=HB@*7W}zlZ^AIkz0YtWL0t-mGHGU)R%BW5ObeLN6Km^yc4kv=tynD;Pg|(Wt1< zm$&x9n|xFm1r`~rmVf!8$K98My}mogqzX36M{OtZL}$NAe!R@{)Ji7xP*&j^*MOkQ zgMA(I>`Lwgk1GG+@t(HsI}4dY;pqnUG|$>!qHGzu9YuOrmR3ne9vW7w8wXKqKl=Jd z+89S!>UxG!^0N$l*o;qvHCZfhkNBg9d#TRWTIN#OQIHni9E>nKs=!~iv=0Zi%7`&J zkT(jbFJWu<6WEzFl^Yv=LAB#F@AHwsJj&b&9xv<78+V`ylPIcdHaoM9D?5_XdwS%= z&mYLGbElhh<|mgPEjCPTb(*8D$SqV3+JE=y#Hx=tZazJ~U#1yy z81NNp9x;xH*`3GB&^){2g5h>Qflok*A3_DoWXnDjfHYc4x(-@$NJZ3rqOqfNQD3m26*p@f_Q zgv5j@=S7@lKctN+J-w*SXBzBbqMXf-vh0hX_ZN0m%QTWXSOvd%QK{qp>Q=7f^CHzh3L-K8(KlF&wq8wq!;xnflOJ9Kp^G$#Ngx~nr1A#KTZ*nCqy9)xy;e)#tm^2E z&Iv@m|60A>mL-^a7At5_g+rC?X)&%r_u%|A$L}^=umziLFM855N@2zlxDVB6N4p|@ z^x}|Ky1i_=(!;{Dfq;aE6`Gek?xOhO;e9n5`D>uIgF^!|m45F`F*xp0>>T}QB@N5Z z#ng`05PD7UO{1x!=x<$6yv9I{+}?LdTK6O$QEBssgqzJm@M6`P7n#Mnkkw?pfk&&b z4HeG4BhqpJJX)|tY8J{HlJ134U^3%Hp-@=u;V5;qIM3ARFz9lv2)xd~U_i4;4?nr^ zkE0a|YzKFJUZ?d(pG=?@IyTi_Y&Oa@Z!6?h940{=T@X3|4}ok>42H?182J$(0UF7_W9-U*5>> z8xzId`j+}iv1lS=Mj{)4Nyg&SF)bJhARH0>}S#50$XmP2zzqj|Kr|c+`KO>aW@H}y)0mgibj7K z2?U=)K=8>U-yP!|8!CTb!VMV8Jc1>y7*M3&{UlV05+?yW9)T?>M06s;V^}nkE@*xP zCi7p)SREW@?~~0 zO>!t8ePhvczo%P0W}R|-Q~P2zfPv<;?jv2d^KKc6f?z=Ok@c$r=hC_5L&dil{j=|3O-b8e0O@$Qcm z3}3yle01yMB6+gwa85FW(EX{_Hbq5DT18!O7|sAG+UJxE|N7X@f@ZXY?iLaYJjAr! zwe)Z_h{}B#&DT6lbi;X`(eUZ_Oxa;UFG)k4c(zT~hM7;!f8y9xqi(qSW?0(bvbI>i zNe!PVeKu9UEO|e~i{Nrg(CHl*%Yi31Jbcd8V!WXEgVC_INi_Jx*$-iz-kM-RVtTX{ znccQv$}w7^45&8!ruOf+v^ZUC7n>_oZ%1kJH$E;7Q7_r^?F$Zpx|fq}_cw`L@pZ25 zRWwlDAU_0Ahda6han^bY^jRGiIghV4wPa=}+}l09wxm7hADcXtC}xO$-Bq9L*`!a@ zgTqW{AkX)fAirMEB0%{$cK4Uxr8~2_nKz<*@Amm|KH9xD9`#B3I!yjaz9s%!p9;l_ zr46(3yDvOSH~b55WL&U&G2*zDLp`>CQit+dAd;ON2U^JgtPp# zH$WZkr3ldJKPsd-l;uxivLn|fZMK-S4bsh3z^e8g@n*|8G^t6Xm}+?J+EzKNF~nQo z1eDX2-@j&~p<^iy6j3u&>W6Ly1wG5`ISjZN_AiW~pK4#;TnT9xYyAU&vz`5ayoW)p z2kP%_N?XU2*#c})iHEbk{%*pmzN^@LIn$+YL#e=U!CG$ElIx%|gV`ulM5x@nrQR{^ zz)*C>yy#CvJbiA#1GPl8EdpRbJn+AY^T0YHgf4V=qy#Sg(f#xoEwtHuos}c z$y>Howwh_%t732UaSa6r9jPS)_l%FbA#M)HYT0s~XCImt55J2Uq@rw|$(27}!Mqnw z2u_qsh|43i(GEBwV93o7#R|Ow{oYa`Ke-w`j{iK3y56h3L0Q24%icIWI+a{sfQa+8 zR&ND8%VkmN;1UD}9ox8-iomaDDEgzVNj*wpu;#0ol##%U0NPcgnV2kLzyWMFDZIz* znBQ;ub#orc^m!4N6RuAolL0|nSu;Eck^vw;rDjx zuHwiU0>E`V2Ow{%kJNASTm?+h{#5nfn~^lM!BA>@NRtP;ZL^1SIHjgN=tNns8HVO) zP(ml>yhex6&FbNRh>~gGt9F3c`7m)jHf@klsIh3NllEXVlh%o(f)j;&crHo99Et z!X_@F9^cHdN=cP>z@*wYZdJD;F$e>GiwBHTXPh1!si*&sgmm`8zk%n+DekeASqA_- zdw6W^aL0@+1BfrxV8uUdtbN+p0x%g}|C7wd=KTEJjkLE&?!YP%WxFmp{uG$Ej?un; zr}x%FEXU%Fo7Fm)p*k$r38bw1AhuqY=7oR;d4mcq0L@5L;bG;P<|-j#3!)*7QU?C8lAiJC0}r zs6hICv%&567*_EHI<;H?(E9B1)Tm@>`5%lt63`HUS$$DhIH0BYv1!!AhHy}5ufb({ z=&MUgvOw9^4@;nvxi&WL&jjRo;{M32dTHjI2BJ$DLe0bGLxcb9Lk>Y z!v{M@=kxkp_T#)q#63Xm$32-+UdHQxh}J@#WqdfB%9&A3D)SGi*@N6gHd=s|l{vN^ zD!Lr6w5UEhSJ3q^Uz?q-P6kys+rP)yjPZD6+3Bm9?xAulQhI^ewMoM1wGTn(v$d2y zH_eb(yXOYozEhhz4pq>;FaH9rZ(I00zZc!__s00KaeGr^8U2jL)u?$5v~MVKB6~g_ zPl?a1cwwL$N=!6_u=|=4Hdy?DxU${R?VXVTtfs$2jo*((Cm{doz;XiR*I{_W5o2{p zzazg_BUtqYduH)L_Wy^67C1`b|Fq}+MkhdN(vIX`EU{Kz%wh&KEocr>aeEgXp<)`L zdw5e8oppCL?%M)>t=ZDX?50F%@Z$!G$0F*5NaW_*OUvrGrZ7fB`%M6#%(MxpFT=C;nSWf>&Yr zhUp*`_35!id`I1{n@4yUP)Ux)5G?@G&t`r|%g^8nF0#`tIwHAcGtUDi|DG`?%^iu+ zvq`@R$*=vIam_8hc(@>Xbt{(z^d>mc{vLBv`B3~DmVV|2(ZU;}^}v|sn xqAB4 zcBlX>K}(boHR-p?h4n0&b9~zaYhmCKL4Ub=?;0aCbH$M%5X~M56b;lg_sKPz(0pup zekl-jfu$U-Lba}4n=9XL6uUIk`Q@pTQQU>m`Nu@Y$05{7e7if~)BfKp9~y=KmZri| z&r65y4%wvg+K${0#8WGx=K8*DWqMSq1Zz9!2aG zU4O62EVTp?)L+V4#I{)8D9=Iiplh`1F#1Y`K?9iEV6gyy`kyK;=aIFYwNVO`U<*>E z**lfBb*;Mp3m2`VvCh?7S*rY!lS>~4uRhJVstK!~a22mw22f~PSK=8kZ*$w^Hm*&b z$AN``P82?fC+rc8mLe%xk36+J;|uNZ|ElTFN&KpnN*{l)q4&`0e6mK9n{@W!Kd3pU z?y`+Imd3$!@7~#@Q8;vMhZZm69_7CmLo^s)uXFuH@LM2QN}Czm+udedo`cF4IUTrZ z8eUT3a+|d({HdCW#xUfj5*eh~5UA62RVLwH&4cnwZMR%< zBryE*Sfb>Q=9)=z>}r+RMWFl9RA(f& zfYv(k>WS%u?_sy`g**QZy~Nh01YlN*rV$l)Pk%BG0DTcN-$xuvY{>grm>?f|Cx*f1 z8bnRvD-uh99rHawB6DOu`S*{`tvZfu003V0B_;&$RE|xKt)`C67yQ*Qwg%<+-?U$f zxvIu#K(-6^{B7WttsjbbM1yE>Z)f;2aIlmc`7^n9U!NXLKnK&F5;sUwW;S&|`|H<# zZSwGQ3A@tIlL2z*)(rqiJ4{>LyAu)A;~M_am>Ga$p$&@TyX6h;9`& z7JJSNiJz?hV!az>6Iu+e(rySRAR#AN$}&Wbz_S4wR-k37eg|0y}Kyj*VZsEQz6NnLpOH!7Xg z)4H&jZn6CdlrE(PyE$ZZP+8gQSMndWy&!Ag;|QwA+yWkGchC#U1(=KbD#z!4ap-%a z$*0g&XcO-F-eDhF)h0s;TG`O{eEo(8^9L3y5P5AFsvKi=b%CES%|&}GkQrZ{%TT^P_c)t=S_Ew_kv^DQY{&J;mUA#iJ}R1IR|lzsGsvye zp`0(?M|pEsF=yxoZ{6W5ep{u!|%0HXBkM6BJ*G;Q@m@3~(1rYgJoD+A4-VAvK=84ja zGX6W@*m9N!VTCiggn4`BJ!(f2qw9gm;NmU@^hg zz|9d9!cN(nX|#~;0!e1mbOePc9P`D#YC=7-j5v{D=)L=?G!|g43C$sbrTf$jtYSO{ zAm?toK*l1%2H4{Zc}(-!yyJ`(eayqj3t~Jlkkr{8$zsiyYJ{#F+v_i8fwP#ef@?dn z#F}gee;=|bdaCG+@-ircA$c5;!c?TOVt+Sz+5>?4A+3MBuo0|hlNB$c;Fj<9mgtKu zF$}mW*C+GM<`gb2Ixy1cD2d767lw?og}oLRMm(SXak=p&aQpMd^8{PDdc@?4m+!op ztIzJ(ajm6*09wfr<9UsB?FpKsMXBZt1wH(^#*Gp2k#)q5BY%4ixaf#U(Mbg;k6E~E zY{=S(?aO{KJ=W6(Bvfe|)A-g>ta<8*39giNTFtI^Sak=+s|)N4|&=g z*t|wQAg|Mm2?2BLS$r`?FgNc}w7s3B{KL6z0gKk}!-?y?W43<7);}sm(t8z8p!H`@ z4h+ZjQ$mL28XO&;80zZ+OP`U>IrTzpN3Sq5) z$hcPnui7TZOlpIhNKKboDfPi|D1(F)b71t-P#)zW*j`LbG}Z}>Q=y;viKalJ@>8wVO{?li1ymK2xMHyq2_$b*EwFb`r&RpXOsYx1I7 z6h;j!v;FBrpDo|@>>X2`Fr|lru$jrBdLMLMX3n*b8pN7R(pRyePO6`xW5lj){^Z<0 zX}OYqeX36rD79fYEGu4ax;<$1zo6Hr5w(uoyKHA};&}x>So@sLx`_zW&u)+c*)p&1 za4l_YXb>xUmYq+u`H^y^#t+#xY*^Kw0nkhM-2od1vsPd>Vw;YB-Jy`)P;Ab*MZ0jN z%t0d;v>sbp6&(nxBRqI8^gF=J8y=PNcc;-r(!{;-v$2uXD`2KjH|`#jFqZarCmYRC zmJ{Eow&&1-A^-mW=0_34n@)Z3ya>2K*+avu*UEkKCSB75HU+{G_lJZ?j4u(=Cbj$+ za{2dDPR125rv`GrXmq!dUcUsKWOS2IYWgkCZ{&t(`61?D!%>Kb42wI{ zF|C9M7A`f)Zwpb))Z>Qu?V)5vW{dfH>eKVhCs6kli)eMC>>HzJJjoevKUfSRD)fXz?~ubjImngeF9ZLPW8bs?1tDj%bqx&OejD%4sF} z=`%!{e<9ZQ(4#v_&_|^5S2%oA5`|Z4v@Cpn>`~#WKMLHLX z3-7MX$s!Aqm*7>vKtBkIn!l75Bso&+xbxs2T%=Q~VIt=QxJ6#>P@!S7nQ07d=Uk>u z^6vHrni%WO;AX5efj8*G<*PdfNKIZe&N1B83>e=U=&^DfT=t{Pu} zC?PT^*lXY53;%>Y?Kc4J=7Lt?&G?=2Dg9S^{NDc+l#7`<{PgAejOO2UdJQ+oPU?ib z)@)aCtQa^q5*@${B?4CSL&pybB_y^}2`vY&ZIO^vvoc?ewTFEu%6?bay!8S`UT`-5 zf5PBD`!G~!>K`HnotSZ*XR=S8A_@NPndLJv3X3K+5$a;V)5`32@!%T?YbPvU|;EI$r3OjV=yNLt^Yc-;tM1WO)$0FI6n$%J{#d6uj#yxRtqGmf<>k^R{cg}nf!aSn z9Q=ck1xdTw?MZE&bMEJH;>uBWvkVvHfTj9Cqj{W<^J^IPMYhZDa|PnkrDH}buhgt; zR*JI3%E_!(tT|RTD*XqRsRb|IVJ3~hEk3Vc+6THS1_<(LIB>=<13)Ce$wH@e+ar_+ z<_FX?@1)yM5}Bx1n><{_D%NPxe!o z5tLRoil8C67|$Q__F7jSNtxF#=a&{^U6I_U*F202yYoB**2^TC0@5`-6gF8j*4kHc zSdikT(Es3{$s!%x+Kw)6{)hJMKtpqy-`>VpGA{P+?780>?Ic2d>Q)ADJymy3|CgpRiQY@R96qAH=^<*dwrfdT9&}e~1X;@Y;oe#M z-H0qhe2mx)qw4GWN^)8D?5}1P4CbSTCeTfjo+ix?xAeEq=G`y=Vz&7RWlzxHs=ewg zK9})cnX(0yH7Pk9@#Jd_#(UExvuD;E#*a{V@0k|CiqZ7M(YOU+x0kC6Eo=_8R zQL?3IRM*dD8OV)3SC#WE6O&YsArQ#mUGQl&m8!m`|A0F0E@p})F-yQ6PspC)og@W4Ast-K6R;$|2XLar${L<_iPuk%RyVeJ)hzov(t8~8r{EPw-wCZnh=3zTsI8>wP^i_xFhbFDh zAQl7Az@y&S!;mM>Q*#g)gV=lv%ij6O&}bI2EvE0sqw|V5>ZmeNM}xW2%d_{Mu)z99 z`1oM1?^7bSS72frLP;<5-|5(v83OOX?%MqW%@MZXbJP6=@`_C>$OrwJIRF*~GR?(0Y8aRW3%ic-GUXsvBr6fNR#)bv^$PcO{?juoX%32n&`(adW#A3QRI ztuXiTMkOea=6YQfSOcw$m ze~Jm<%y*C%6{N4OUr3mYNxHYMRIOj(-~Qrw>mGxtx@aP`AMsL@_OBZ?ah!yBWYzL~ zAb7^1HwG8BUPbG3-NuWL*2h?uP);oyz4PQ;$*0IvpZNVt zl89G-OF}U8FeA<}^Z28t+5rdfq*bfkmW-Tf#YY_%XBlTQG*pO5a1t zNI|m_OKQ2~0HkyeBk&0!5z0?;2~0E-NJr_s5CcxI)BF&5#A^vPO-H5H61vUTCAm_2 zV_!O!QEcM9L~wC?_Q~=PGD9*HO3h)u=_rO%5Kkg52epWAl0TQt6c1SCT4K;c=*`3I zAmM2*|49Byy0gJJicozxeRHopxU#WFZyY(H_MYO+hAMJ|;>9N`$Gve}lvWA=Rkgd2 zzbFsD8($@_wi~iO5Z3zsO$(rV!5&{H>ag!z_@8@ud7Q#wBGosG4vpj|ba47jI?@cF zz*4PGPY+nh;9opD$kvio zs{T9&N_**(Ir-zA6UbuYHR2I0JCB9xEu15sAg}v%$z;-uL2`p%1Va^KcjJ9slE=?k zi^bKJhl##`CFG|^*a|hGL*vLI8Dv@Cxnpu{4nEStRU6Yg(@RwSO))OGNF^Mv#Ae7U zRk;^Nho#1&K_&Vkzpo^M zt`y83m_fq6g{gxQF37tqwyqai5@>^Dl~ji-lWJN&Bi%$&m#PyaeSB*t92Hs)B%)Rcl*1hid+@A%CPHxb7SZ24F4W$?Wz`=p77y+j2(zeF<9WrWI z!}+A03yl&t1`WA;18*<8MivQ$rqvT2JGU}7{|VJhLmS*!xEnSm*jw&JU2-@}EWU^_ z)nkmMnFX4-NJeR2&`q`w$;LgF7*py&euy&WNa|Y1&*=}y(p;2d#L1+NjPfgOl#|K- zCrEU$vfT<~f+ljhtja+eM_d=SOnKj8a5b%4U~Y}9bNTT(m^&di-{T&W9K4r5)7V+W zrTWnoEfs9kzOf^u`ZH6qOx)vjEHw);$W?#lC`5<5v7vcA#<^GIO|>c zGgeZ~4nM+*<-v8K#i@1sXNV;fZK&cnD;<(rCou{C6OG5&h6KyQ-Gvo zFUB|krNxjs@hnrhw2iPXf|(hA&X_AMVkG8-*(DQqtH3yfy$I5+(GOM)8*I__cpC5z z9*St8b7W5%@uvR;nQs_5cL|=%du>2?)34EFY#R6q_VPn5mCKXWJ41)Z#sK`?czBG* z(G>UiT1Z$Wi*5yxb?7}ZOrw3nM_>{sUVaj?xVIJ=aG;;X;?n+!>y81xQ|~l07_It( zQXXg}_J^}o#Z2sd&G_7e-SEFaOgj|PxPv*`|7dAk^eJjT*`9&eT%G_a2Zkv>)amCN zWq=2u5In)BbY-~SPD6ct$kXM}I$Ft_wBOBCbVOEU%M!nOM3`i7fxC6g{;M)TXJ)f- z^>tB5dG{95nQ)6nFVCvHj_K+=H~O-c)I2Z*wapcyk4yJyMLHlpEH70Z_Duzsh-B#IHP(nXA4wpc0U zZ$YrC!LO82?2~4qKSF0T>-?3xAlQJl;Wu2_I(b;cOzhRADAXq{KlH+X+uAEBFGl2e zf3m!LK%JEyfYbH)Ho;WkFDf<^{)0bh7?##rQY3YOKN|j_L=QdGk?I!oL?~M z4_9zvyn#sk2mn`R^^NlRnfi)SF~{z8S{o?{+gNk=r!VQg`Fo)3p;(z9Bg@Dc22@9A zKHUwgU+xb=3^1`{01W1Dpt1y1uJ$lZLoN@tn$5sKsn7x-b^_jVwbAnzj=6sU3DzeV zewKH+mN(Pg`z>Kabsy~qmNQ{Mz@5xwXDZk=DuB7FV=rtO{5R5guhJby_h()2pJjI8 zoG`Sos$c(rzFzw@;#-Y(HSRqFAc^bYs&(pgo(k31Px*!Ha)+(t9*&q9AxL$V{O)3% zzDE9qIWAmR`a}(6=>7q`yp1hYx*$zyJ~0akaj$0bN2*mG0hBX@$4&sV5XTbRQ{EgA z5?`-{uVFwX(9dpG(6QsL)Fbakko;8;p7$^{XZUxv-I;3H>lg8HhYODHl?$)-Y6cSP@dbm^0H}TtbGr9t zW)a}J*}E?~f043OI8l1N?U6=tjZ?exVnBl7wVUrlfVG(+M<$%z%#e4FqhKrBeLv0u zz{aLWq^jK00kyVy3f+^(^uEbZyQc$sF-#0ltu(uV0s0Wj@rWj#zJ1r(erwZg!>(Yo zTk&o80K!a|^sleIJ=MPeswW;;Ta$zH)}44Aw>42rq>_mm0V~?=Tl!m$sF|)k38VRa z7Py`!kRr#~+UCOr_Nkf!{OF1#h z?^ow%r>yUeA@0>)o$iL7I1|^uPFQgacrcM|y;o@Jl?scFksH@q0J<9Pw6DvasU2-g zBR(9C?#}M)o8vK8VS#Z!Chzle@I8iFVce=3pqR27rc}DeP`{J){0SjV1&S*TJBu_* z?mRMs?j3xaiudJ#u;GI)BtUynd_7QUaLaOe?jd005e7{-mTc5!x#& zzZuZRxEEx%Ky6J4AQ5^ub2xgBvDUA2_lvqsW>JXilQ+_E{<-2A1z@r?QeOZpCjjr% z2vLs_QN6&$rSSEA31Rx@R|aYO`c2=I-vaWR#XIh79z#`pOdrD#2IGp)i*B2(uW$YW zb$ea|%vcUcS+f}C$|90Ny&&xS70GWg)pAd(gqXAf-7AT|t-8AVPBHa0S%Ndym}3$` z=V+d?#7DNz^|k%^eDsA@eyT0;pW~eXTmvkL@LqcCPK- za)RJQ|41&0@kGjg_$ZG_o_%+G1nzc6NV9eX`2ydPEq$gfSNv6e_fdEkR7s*9V) zgQawGIs4}y1wss<(}y0ZKTR$4X%$^+5KXN5W>;^Z!F$`9m39f6i- zals)fd9b*BRqxSNAyC-Pw+Gq1;+8%q+o4yJh4d5ayOojmJME3n(DE!hwdbx@zMvsL zYZ=Nu7j9{G>)5QHrsqitkjr|wyyVB_HHYJSe42W({Y%o>{gmEI8DkAPZEXhv3AEf; zN|?71#ld>?HY}xf0Ya{aQQifx{b;f#2CfD_W<(pXy~#=e?UqnTOzQ7vI4oQ|B3qHb z^jVsZa%DzFR-YnPR-*(+*k%5JF(*t0u>Owmlswp$-*O&M-!Ca6&`qd;)DkX82;i-Y8HsYz! zlD^q$#z|9)kN`Q~XmuUkhhHvea*of2tyt5NB@xoW5NY!m#HAMZd zy>la{FyRNZ*pB)JiX5Ck;V42ae%XnBclge-+hlVA?<V+IQ1qkWst4-vTTa%#1RoQmr5Q8I#`>Id%pY zs;AsOpe3$o?eJD3z=EQaXeB!le3CfR(ro5X^LWH1mCX8EWqapbfw`YYy&k%gvPuSd zezy}xjmE@M7pRe%TC=ww_?c|o5_hsyzv*DRbYh0CHQ*L%{Pfh;*Kdf#5B;O(J35p8 zGdQFq;T3rtvyRd zobG`#mMq~vF+ougSfA>bpFVwGJ`QHV1`ziwR*PsrL@mGZ{y$cU;*=O}(GgXV`||0F z6B{R#LKX}4Ck(xH(N;iq%n?>#S6RUB{U#$Ms=W1~iWZjtT>;fUmI>V?wsPzT3MTA# z(_>DjBjuG_UF&15wt z=l9LBZR)LMTaV}%wO(5m6lfRl~u1kpU=hbcSG{ zfk3vlw$3DU`4poe{QPh>DS5up zhDzcbF`w`VeA?)09BEAnLL5Pg{6HQrwWtgNm^H9>ctiHRv^S!iy;kPkp}jg2w)0=yhZDK&vW zLMRK$<_((n6oGNr{QP1nEm#Yc^*7?6S%86u`I&flP(ds3u{T*Vm_-a(h6X60baDef zzMi2$98tfkC&Oh6b)kufz5;3;2_EhSTWbxbWS|h#h~39;Z}RRP^DwX!Q1|8LruGkp4^L>*C8GLxwIM^7+b>N;A`hPwey3mwe#ipj=4GX zHKTMUKtrN(twx?GOXA``nxI65cY@5r{D|3VE5Vy%m2djxG`+EYGWSD@7ZnrvVvI!b zyf~zbzrBhP)E~pgXhR3Z((v3*k5@bu71a^^kN_0er4R+TQ;?v!q5*N5osj)7Kkd06 zbNBZs;_w8rMN{=*8 zpM-^l!Ao2kp*%#b*JpCqcr;ADCz#vs3VcpT>GI*H=ahk!Ym6X}xaw|42*y{5DkXOI zJ&LAtPn10ox0j2;Q0I+)izAlMM3$*iAN;{IJ`B*InjOHw)=Fx;@UgCH&SH_LuK_Dt+FIMNK|q}3Cd+Li=>q8 z?YYitr$;-7@i5vhu>x|L7t1?PUKVM>qZHA~hnK9kHp5v{!O4hQcxYm@m8oe^V|go{ z&b0f*>2EbEu})=|6j?Pjf{*lTOQSvqb3k?p_G&?uKaj zSb$iEA3G-e8)FikMEM^`pal`@6LIfIc%KZs1PfSX-cO!9S(z%${5@AHA}T8KMQdQH zAwYuGtFaNT^@uLH1Xydm@jhrF;W%g@-%m4ffLKvFJPOXvTz9SS(E+dgS6c{1R<_}1 zHm3H)W{;j?LMax#;;B+DvvuoOG9e1k;Vk3P#m)pReDbyXDZ;{ScRdV`T6A@FmLP#f zOv>Fo#gk7;{)O#S3HbvfiugxK40-;yR~L8f(}LcD3N0Bw%ZwUhwM(gg&CX-oJ6UUA z`Wk-1@)94bLo;|QJ6GtWxsE4=tY<0+2m?fQzZz$MGZajHAmZcWhFBn9^f~JuXJlCb z_Q6Pw7*RPeSFcELDLWn8Uk-XhIqWj|y1;TD4)>;wqjh*@PbZ3j_s|LZTz^V+qz&W( zfGDFw_@SCy>950={I1NVb{he6fNpgs9Cnd#254b6?!@hfIk(#h0%uv=Pp#ZYY@yMQ zQbs13$5axNt3+=P(W346}$Zj#5B! zK)aL|dB?U2x$!=U92T}X{S!G#b_Q1hAw6#fV{t^iU%@!(^I3juxYW2iP260^j7W67 z=N6^Wr-cQDxEYxyrKh72H8wMGa_TI0|gM*Y$Z?B-v z+F{jyw;z7XCzB@1V>sMfBwh4Ac_kv?F;S!;2iV7=S32bey+snpc1H_Of%K1Mi#}iE z5^|glR#4JkGu$6UUruUvfR&vqFYZmIk}FPZm8!&8KK%PA7%VF6`6Z~a5wLOUQQSN{ zhv(a{!yT|3%gvEjDK!svaz?GPeXRvZe^C)vV2whqzHLp9^BFVCI# z7bFBfd<@+!$%OB^RN5_&`J>*9&K6<=*6QA5E1n##N*f5K2Gd6n9Ua6`!gm#cJ~Y{h z-Y7LXz#Ew;)mLdr(}WCGz#}b9$;O}qg zT$q2Oi_7Gd^&|HXpe`c6JBK~>^mcE*I2!NP%cp>tFnJz+>ZeI&tgm^6T8$30V^L&s znzHrAQHIqoVM*H9Y#7TYVW~$*>K#5#VBIFrHjAU+3vO%-^W>t-2MP-(m|ZH%ke zMjHFb?7t`xt)il$Jx>mpUEBN&F|I4sd!MpnY)l<+JN{K(;2W4Rl$4YvN_FrPXjppm z_|ySWwTd)<3=D)M*VYQg{1+MUw5O(i42*;*77$uZ%JX@fJploSdkBHu!AC3l{f&KO)NONuXVu8p^DwtCR#bnKS>tiA0zG1UPwuYn3$?n!JdB zhhwU3fh-0W7O1iD+-b|5KZDQq4UFpF+cWcHGx%#|syBz6fk#|i52C7VYu<7JpYqsW z=qSxhuc)dE56L5<5O9bUhA02K9_WM-AnS#EE-I>Q6ASw|eE}ZlLJ~14oznPPdu#?H zU78f-5|i-~v?}hg9zCtj!4ece3Orj%Th3lpUEN4VmFN!LFn?wB5W9s65>SQZ;ykbg zfG^SHp%f^Q>5Hc!6Y{PC)T17voWqL;TrVCXpi?4kgW&s*;U{k815wF0<7FHY5|VEK zTfiTeW2f_d>L;WUxqVx6B7$`1`}R*3Mo6u+BHs#deb-je#=qauVKVe->FJ*x7T+GP z>8dMIN`-(xJHcIm+Y3xyktKg9ddGO4SZ{15$MCbz36ms-*p7>%yb707WMYQ5Az+CA zq5gFtud`k9XkhW_UOdL974O!V)Hq_%iUvAnnHhVo--80Z%GnA3-sx&Bv{7Y-e!%s4 z#`h5xIyO#LT%)U{+1!a&gVX94z%Kv2CQ-hdn0@q@Rrq&x|F>PUO#fvu{EtE7zdrrn z?7sgk?Y)zgb@mS{fnuLz3+kUk3!c0+M7ShPM0lt+AkIJa$RMq z-ap9e(jDWuU10j3?aSRve~0%)Qijb}F9EUdo1o>LWTW8MET4|yy;W;JD=DF3SI zzhs@@|IgR`Q+mw*f6nVa67-#aD(uYAGA-E?liza;dzTr!c5)H$(P2`)Nj3%?+kA1kca3q~)%DPn#v1F;b~+B=u931uZL<-T7b-c2 z?~LFw!rH!W+AWwt(pjrxQXEDlm0N|jFBn|VeQKFbJ%V3d;`kMQXQ8EbE&c0n&^Odx zG9y}e?8H3&(YVBmbDhaSbLi;zBD4$v2T-x>si_ex$CDU{9^S2O29YaC9}alruB9HF z0|q(2PbCNL{~62UK%#eOLFx&K1aZ049jdu9Mc3dI#VO#nh{d446tmHnpG@bnWBQ6m z9xLG962pqLjI4fC^j0Mo{%#c_+K2@82zjY420S$ixxtulG(lmVPf6703z6w!NK8+U z(g9Xok{C0G)aO6qSbq+lA@`L?6H@;M{*0OmP}`?r!nPy}d%>?$@bqPf0ob*$*abun-c#al%Vb_!GLEfS~7H3s9Q zt92?+d!rY94c{N^s+xIx=iB)G(ARo(C?MpUp`IAG=LHJ{V3u+IWVN;`yAUikW4$^_ z@&TO`dzL(3i&=DH-TPm2rT+e51dQ8y<-YyLUEKu+Jd3`!3+)2|crXZ>T|*r#tOq2I!Pts6P;s9j7Ar z&RlCKIijV%9G(_^HILh{{=FM{rA)E5ohK!#CEh*Wmw+#h53m--0yswpNKC;A?V*29Yeiw*$>z6P(_>Wz?%3U;d~10nkoz-hM9QQHp?U`S}KR% zp5iCaG6Q*php}qQk&o5Up&&g!*E9jU(D1m-FAmvF4L-*#Rb#m$8r*jJ!g<9!Jx!up54ec^_DmX>fKic4Nldz{ zZ;YlGM@iqOtCAzfj%{cA$MV(u+@N*gVR~XWKIe4YuGo^(8=;!5!%Nu&NBi!+4F}TQ zJ^oC?Wk;x;VkGnwC%*0O?4ZHTo_B&pH)W$Hf4J&hzf2Rb6}D8# zjZH5HclIdh0C_7>0d6n)wAu=pTZXu!-&eR#EX{#6T^qU3PlC7C1T3(p2WPN`fP}lH z&_pclLcEg9+EC?VekKkMsorFm>bDlKQSsOb^-+eU6fYwyE})P99Lr)e&ch4il)J0F zYxpFKp@mB96fyW)(L~ahX-A3frQGof0p>UcnMZU5F1L6d(mndQSzV{|_^W1&eK&OB zwDzj+?ba=S&v%fp5O)<#7?>~QDX2-wZ~W+I?WWZ^Se0BQI&oNj!d^0XKiV&*`WU=! z4k4X``krok=*rQAQjs()8`7dp?xAD2tnJW|>raoYbK)0cpCcu=6h z%M9cMsA2&sS>=TWU#t;Lm6fp~e2TYkW2V19K`4x7mGIu2`QgL|&`KwL2~AXQ>|0hn zvOf`CH!tX@E8w|`_f&3q8>hp{ARBR*Fx={YO{W|QWI(!?@_C2Wzg2o36hPY@8t#CT&edG41C(xp_ro`4$YQy&M!q%40Vyy{kU^6c$YNB(v-aJy;b(lhhOVx>Hq#_Bdi6)o z>s@F>q%rAgN0SzOpI9u)=DFLeO=snNQ_ZY#+Yg0zMl{@l#rlB=xVCB>9L)7eOJ4;D zyT5A`mH`qymSTzacB#=*B6Ze|(+0GMpWCc-uVBSW0cPCMWHRFt%JhzoI=Zoa(QY1Z zHx9aWp$31ss7+;x_IR2ieOq0@+F*nl9gWo@!M)G-{!nlSPcs3_bag^8OUC9~FTZ<5 zjYB-JX952&RI|xymwex%u*!VXR;Ht2VQDp7uT>avuWe6a9E9uz;{u*NX+^264Oik!ZK2(HXHO`-Ya8kXaCl0Mn65z!2(T0-2$P{hmI=E zB_@^nSgOB7P=FQ(=d_d}rdl3XqwtL(Z)Igy*A3NQ<1LD~6tZ5Jc+=8g_6tfym(Ma$ z#Au`rnY;ivWv>WG=|X>hi~ackyV~1Y6-GlN>I+ORkQz-!ek1#a%KJ{&7*HWux&=L8 zgCCHIr0GGYDIr%-EILyAXT+|5U6GAlwf#Z%VK)wh4uIWZ)DR(r% z8z@hfEe@1AUmO0LKpVi{2cRQ=5~-M<}X=4u;;*59spWsz!W?zJpptIE1V& zSp@#$W#=5IzdX7{kKhzVX7s@8{~1pF{Vz@F|4puEKE+P(PuA(x0J@z+STfNst-r&g zoa)b*IB&y#ebT}-qC$Lajl(YnGhoC4>hJm~IQ9cLf!5eoe{cYmti`>b$8-mUz@R-{myf@o~M z@x{jrcJY7!T-jRE^4BaAF|3ap!6feoZhM4AK{vutJb-(v zSyXTUKkNWMA8XcSSt=8@;Uiw9FJ=kU-ikoyRbz>+J5Bcql4?fTk)_^Kk&Wzc6@BaHUn>30z=C?|+v{n1wLu5n z(3$3q7ug&6-xs=uYy}c?yivEOs}kvlQ%ofCBzz6$+kgH9YA^CdFA5eq_H)0#TR8yw zctl9MV161u!b}HW<7J-R={9HtwNC|a#6oa{r; z>I7QIz16h)}nmxbfIc^;1Re#U!#jJbFMrOP1lvrmr`5G z8k=Xs#pEvB4o}%njq?kZ+J~;;29NMGR;Q;4>Orv7m?=M-ESnbHb z7ikMx!?yv>B=P%8fr+P-hr`J1-c~aQ=ka|ooG&|0e)57m^FbkGY+^g@NdvJ&RCQcc z-{2ejZtO=z$q%g1KH)A)Xysg^iwHP$_r`I_HK( z2ER69Qo-Bt55>D@!OLBM535V+_wXI!Yk{958^N67=jt^BNq3{KVEr4M zk3aK#b%AB<&O9mXzU+Oy8+Dcf591q1$RsuGoX|_-gc9)6JlUd{b_r)xyrtKWy30fS zFDng*2^7RM>(;{loZ!tJw?t4%ax7Zj^eR}~b~8l~#~uwNaJV)BjYPZX+vn&gF*U4# zacVI;@8ljF-EA`6eC%NN=3yY9&uXxTQ43-mfZAF04`ob9XCC~xWW|E$_!d+_UJ~#ls4O8K67@;PuLgBftk=B3)f`4=xFUvvJ;&VSYzF&aw-qrh3 zcsKKf8T7KV48wpPLYdVeL&gc)ZPft(quTOAe@k!+e*Z7JbpYGEMOwr{Rp+MlB_?o@MsT1q~f@c z4U~5@tkXvH{u~5Co+!MM)I1(7eH3_ISm}DgvKz4?(uCztt>c8R6Ym!qw*%`WX6uNE zFb>YVdc&Z_6&-`S+x5elf;1;E%rNsodeqECJ*9T+yS~!Q0OD(b;SLelpIqfTo*YpN zlM=X;Wpyv^uVz0Ds%AI*?}a1XvkVjo_RFn!v`QeR-pdA4-4m2tK6A8Zj9yZBFTL*Q z+2&HZ*&QVaC6x-lST%o5Jcd1BX+whjZx8K5{oeOGwvhSY0gD>6AQ%}qy@Upe#kzi2 z4y{B1xZHdg>3?H`AOdv$W%wCz4b0?WPk~KaeQQ`# zmT$>w!gS|3{%Q|@;GbQL3yDeVmbn2u%hN#kZc?PzF}rWhqZwRp>KIGR_(u(C8PL0@ zW_5eLrmN@nVuX@AZUbe5;vpeEU6or~c0d0-9*3Uo$3}Y8uXXv+KI1r(c6KkFT+SnZ zV>}|=+bRWAE+p4!_JBj2`pHrQ^Dy^f1G~I;9?v>R)f45~$RJ$IcPdeU{_uOmi@*n2 z%DUkd-39W3+#j1kI(H!>0risYFEKdYFjqU@Ao)$U<)$x64LQZF$t_z~g1-j)2u2@R zt;jD}J`5-_I>7C|<$PlQxx1(V9QdAR#nuX4l)ZN-iPo$m1MGwin!}g-4OjTRI4t{k zoW}|A>4DNRQsWw>G@PE-jUimv6*aSGHUKW4f)Rej`{3|#;Syzxz?rSi^q9sHJVIW? z74v=Zxy!9lcGZ55@^T~DwrN+-3E?vBp`Se}37VU%zA`1oCz?RGL6%qpDV<2X)3W_f zp9?6a0B6?OIOgkRl-Omm7#VT?_91%R47NT1lsH=VXgSB|mOy{WA7A3#-R{?qtu{lM zeALy2D!qf#q^s}(Rv_^BP94Kv+MdJ31U4YHdQZ{aq{k{)vlCow#fo^~PXJHA3nv0S z$?EaxnqJW`b-zx1++zN~cV5qSj$yxP%VT%33D@{=S`571* z)p_|^40}vvHvRRQe$Jkih5u~9F z|K;d_iur1=qr-{Fm~}xS=A5Cdk90wu`h|G&1>IPJf9S7@gU@mBVKT}ZvTL~QX{ED;YDcjtd+Bgb_};sK{UD=bz({073dZ-csUjoKXWF4P&~`|CTMfVk@n01BVCw>2*a^JT{<73sKd)v zWRs=!AlR18^Cr-H@vGt1N^RRhJ`*Ty_Plf50{o8V{Mr&EL9RFG(aW|s3Wc-0D%tO} zU*V*T_DD7Ph9NMka}#L`8Z4P>_duZgHoQX4t54>F4|q`=(QAUp0etLhyu!fCx!W$9 z^1WHLDk@7C!B$R`O!%hj0c6s5#GR}zCHDy$D4ogyvbmy+(eD2feb1Rq8?K!~*uxpp zq6~g*)Qo~n4Qj{1ok(y=EKXQc@?^|H#zO&%oD}AR24c(XWH)*AtkW1ygF@%q67sqR zpC}7)yc~3#7fr>99}eea=?IATBp$c#6#IaP6n_Z9AYYG3F)q+xHKZ zskVeq>haVjeJSmO%xm9OHUn0-)9i@^+F*uYXTT^&cZoV|*6>i5Nz(88vg$B6wd?gF z(1R3G5Jo=8X24}leE&VC>(&A`{3LaowCC5bqu<**wX{A936MxSj8{;y2nGQjUHQm~N=jQpE)N-S)t=xTfz_ z`YpI>-<4}}$x2lF<^XC<^Ki*qv`!|Ot$AS>V@=1tH*>LI0LV@-%;(}NkDILq=}Gto zhekqm#R~*nh*FMV<_6&vuNH@%9>gAnTq>x666ZTSdTkya!Pv41+lU7+j)|aQjq=w5 zsR>~JJ^?TA;dO{ch16R0r!()2soaVva5^iL1be&nbxdrnL#JbJqB6s`w0c6be`$P)Nq-NFeW6_q-{-aDOi^&$xQs zWnk<80F(!Jn@r(fItqmEtog7GWFJjU8W?+-j(vYzg6Q@0hxl!L-eLViJ7s^$P?9;5 zl=%d3O?mhz6^TtO-$ghO>6i=XE1#tQ{Ck(RbC>JrQqMh3LCQF~*ugi_}q>qqS+pAAHqvDzdS zRp(PlxNVuX{!m?+9BI!bsA211r2OBK3cz3`MuLI%K^!3roz*ZMiI<8f_21qpsjOIZtoif!_h#Or=LNu`W&+Sa zgDVpr55hrdE#5f7M;>Jh!VOrr7@D|X#ElULqZ(2fE1Ma4-I<^vRRw7_<4&G z3}xMdj?x>T!3A{FM(e50-J^HAR1ySq7?+Ue)pGMo?Sl?`HGP%KN@OBeFHV-B56)?b z^K@TC5Ka4VY$r<|+VtNZly+~EiJCn#q7fxnHdaA7=yx+tzK2V5I%u-C8zK%d?zy7QeY!ob z8_Iq-{kj5;%^OYp=B3S)iX)xL4Clkw34NDRAWrE_6b&w!)^cu6A76BzudjQ1b1#;y zr>dnhNp~w$j~k3%4DY57`t8>x{Xh$35I-~qeLBU8r8V#C3+I{P@gsW>j@wn2bhGx> z&f9GL!JgH-mIku&i^EE0o=1`)5p*Vp=DvQ!3m%ADEcBG4=FD{BwLy&I6$uOa4->GZ z5~~P)Ka?AqghQ1P+xXT$3tKY{&h-5It2yYOOfAEGBwEps(`HUN348o!)In`~CL{RV z3m4nUv00o%8gje6m%PR;CkY)UDzPaTSAX5;Tuj0HiD7`1aZue`d zP@s22FX9}IPq*ELtGix)K)sLe%(97llGTJJ@jakXcLZh6=Rv!Zo2_9krRU%+hL2HJ zY!nXzB>2jiwIaxmaW7|L5U~O8BTRZtrb-pU^h z5NF7ByFkO;LXErxY zH0!y!Ov*H7>Dl|7P#{;fsblZy!9jLZU;EK97xc!L>LPWd{#_8E%6zF+IGl&pj`ip3i=*6`S#b~@L?RWFCwp}|UtuPW2~B)1pdq3 z>?;$bUsFec?I}?zm)q{SHq1tV(upQ1b*t-O_sU zjac+&3vB_?2<;0`rP6Y9MH|~B>F;WT+L=% zT)q`b&h#F7`Ot|DuTN$9r8fdMzt9RIk91$zdf>ccz>6fhW%G4e{muY#?|kY8RhM8? zih_rS9KjNy7t6&dzv5G7@>r3uE9Upfono^5I|D@WTOw(Qy>K`CE{}ec4G0g;BfSV@ z*kDkB;xUT#Y&z1oc7e_J!~3w@JHfb*6O{5Q%6&uEf$FytsskLlX*XhRkqIGR3JK-R zv`yjdQJ%-hSJ!H+Oudl10DZi8X?VT!1+V1`3S;^9I<$~c;N#YWNGSm=>Kbl*=HvL|i_3Ga<#dum1_&>>`pvV3pbe0t*(?707=gs3oL*dyjG>lX#I|{ zsbDN=7n2ZRmfyfUbbqmgq7S`wiHOeIh+2`bcyXR;YYAN&6AkpPALTXnBgvs=M!KBV&~MIaaceLOkEw8NC&AT zbGqh)11V(8Gp;Vk>E=*27s>R|wThy>LJrIOi*c~LsJHeZ;`!1yJSPo_xBRCn`vkS1 zHW4W9IaZ9C!2;$RNMMJbVGTrUOQQ1t6gWR~ePHijL_T-*Ex@p!wIQ6yc{(T)w(`BA z)DE?Mom>3X)TBmx!H5(50B1jKvB5BUZJG9VUhfzgK)EHpzHH)o%xVCn0od$?K>{Q2 zRG;9OYB*po)cF%GiL#FgUD{ZQ(Q`Jkp>ms+iED;Rh!IQ z+7kQWQ@ns${M@~xj*vVqU2b|%o{lg5-21mE(US*B?qrcjr5mc;9#}$ciq4U9x6}#w z1w|){!QCk*jrG!)RxAbv{#sv1pA!q@Y12+@7`~MC4~o#}-p-y>_a8&OedMzjM+rQF znOER3b;p&jIkwV&mf_c9#bmd0mjBGUXBE%D{PB!vW@sTxx)JBdgNJ397^$VgosKl0 z{5}^Dkfa(BRQ=38y<GP}7R^wx8j7w5SVpMH2g&%Pv8g!VnMv>x|+;%kD+Gi1vceWZJEEj;uTxL_-# zPi6H6;!6t8&V6_=GI1mn-0KX(%BDEM{U!exvM0JK(VeTbSQ|2zm>k=s9LA+y;FZZ# zRbRwNvt??fvHDTf(YVhJr}G5oiLW1ZouBW+!aL&E3>iY7=9S=TptZO024SJ)_^ZW{(3uJ+ zLRmbTz|Ufr#`*GBPl_!^db(#_D-raR9rTqHpY3x|$b{3B_7Qype=_ubJJRQ4)+dRa zPwfU+`*e)koYdKe?M zluyE>6-GVJ0|qMH`b}`i$7sQV8B=2~+NudPjI$GpVxGOnTP3)|v%r^2P?fdYB zfIA-z^wNuuz}eX#XY*~b5w;t}7_D^pNnawJ(M`F0<9ms5#r7mhPo34R9WHJYHgxpC zj%YB7eh0)dCigp|G&EgLb02LgVrSE}&u-DGK=5v=X<8@a+;zD(rE#GgvQ7OZlO^SN6Te8^OzfZ4Hex?H?VM%;n5nr(gwlyI!ze6*bJ0)1Im4F;abljh|?Xel5{9H z$hMyv*V8C?H-sUG)~A-F>bZV4n12<~nP?(XjHZV3eU zpo@pao#5`y;x3E3JM1F2zx&@+-CfmHZPiq5&-Bc^?kD}dZ(hGHR#>KTTb~?9IljWD zH{1u1M;l=qUD5JqpjTz@TCe85VZ$Rd#E)|AZ*q?V_%ZN{n0p69VAIgH>g~DzaQC5e z=z=b==Ypi@SbToo^M2V70Y2@Tqs*LZ&AF)r@k zLE{WukCJ)W-vq52Z4GTqetH=c`Z^DFT0!a?n5>%{?xl+!G9D+>!-IOTzrt{hrfV*a z3g=@E_?;hmhh@(@Kcwe%f2A8?(;TJsNkgoWeyV3ZQzM(mS(@$|_Rx4T?OKQEe@J`e z%GYK~e(pd?nZ&LR8=$r{Vx&%kVjR={EYI!hdJ|<<@WKB?#$&oO7>oLLsnsK!MHgne z`@fto8+5@;wgGznbFy`S_@b}Fub%fttN(XfAW9o2HV3a0frWR{@Dmd^BkfO`L| z^gogTeOrp3%k}W5EJ0UPv~TeG5d2&zryeqnL2A{@h;>@m*Wv_P!0)?itj&$+CtRy8 ze;hd@s<51koA^IW1SUbuL_>XHe-Gf86^!bTPI?=S4btBk!L78F!>5m6VsR&bUONY0 zF~K&Wg{foL8&kxyW^*X#hOp2S7`b~kyx9kqDdk~LNWG?mWv~f~KmqYV4DwGJlK(9( z(v2aR-e#z8RcBD5D~Vm-_G2FM3p6SvBXpMAGjuQ5Lc&7`;Rtqh;Qfcrr8?xg{qwD$ zzx0>4{K9u+al(Op(?d?!uelup@zMFORJ^ac+Jp%bd<4ph1JHhGXSMuTb&;ioWvZrS zAw>oN08#tKRLS`50;&jtcH=8ub|WAmv4|px>6q&fKEANk)nJBy)XvX6*r^rXy#Y*) zTUl;#aO6Jt3*Suhi0ldxoB?;ef;6r?I&($B3var1qM32ixbnGz!jglokjtlrM3^?9 z)m*p02kR-O2Gw$ySQMRzI z79(R?70rJj(ls$zTwiKEdY9fRn@zkP(sSA@s6a5E{#@Z>@EDx$Ff2PznT?*T`|Q*q zRV;hP)Zr~g@_-gfJo8nWo}nK3X@Rm2S|S$L`3{K4Qn)0*v9in}Sv$jvVn!21mkUl6 zaJVIf&klzT)u7UIb^{*uoEN6qL?(r?4e{u)?9DgCFGm*b(3M{6<)r zsZt!pBTP_Fywp8X@hN?xd5)E7KBh^SY_cD12f$kvbZ&CRXI=Y{FxF5N+a~zH8wuM5 z`glwKucxOF_*q}~?Y9!Kc8N`k@P2$p+ftVocHf%T6^M@N()R6b+UFGhiGuh^#9`JA zB71+aJ}`>rAf3UTgZCYIx>(OD)hNNv5-^mMKLRzzi`F~)nbIKY*{FJ9oN_rE zIi_S9;JGSG}T^WIL`# z?@IP06)ie2R@=`NY!;dc!GVUnc5X`gYI6|O;E(2!V}W0OIYk5fSAdk1&C?}ADD^jE zlpfySY~2>#s6beLdQV+mN_CqDAjD9yXx-)6PC~Gyq9M=x6xzXYk!LO2cj4Dy8Kc0S zg2VtD#EQsK$}h*q%WA$Tbg`v+BlfrElnd$EdRb$O$zx*Ci^7QuZ+obQe~o6oC-5rb(GKIQOzo-4-h?VMk@g>_lo6-{|vQTk~&mr zp;{~lakX8Pt14uSy?I?Mr-udRJU-fN37(=SN%_K4b~;(z;dLCR3Lz=Fo4YvHpqH2# zq6a)T;r{Um()eZDr;r^KYYi?z77om5-t`D7q=yhhe-zX!+cbp{dz&XZSyJBEOYC>QCbkO-o>YbBKYy#cBpxP= z-dp=-OA3eF`PfjC-8=3~<+B_mTp}3xv&Ou380F88Mw~|Wx=DC*CWCw}jwBV$??MH( z^l93ZY=*6N3SNlsq7E^Las*ml?Q?i~GR6NDxzBqynk+p8x$GVqjeA!sEY#D4MF1H$ zkjXRqTg|RH&h}PWJ;qEm84X72iT0YdNqU#r+%Vt{@)bS_V|j#Cn7{o>iE+$WU#+ZM z$s`cz%)N8_wa;SG=09Cjx|};eMp$P$?_1QEvOpB%%QN;Xbj;J@Om6<|WuUb%TBp|H z{n_qfjU<)Pz;#8+K1lsMm)_wYDfDntV@8d}@-ENzLjUd~?eD#B3U&;`@YEN!Q_0iH zcY6cRb+&BZQknO2BOP{2iMAt!D;a5){TYzIlVD|BgC3n1Wnm9I9)+bxtgQz5hsm~u z7|dzadQ?jJU@5QPkWLUyH{J9*F6pD7(7xNH{(T#Z-Q80RRqC0=vQqAq>)ELnYSzjX z8Qd7*g&2fmVRPdGCzV-y_!-M-Ry6MKxu+VX% znyi+WGHIS>`UAdt#&{cCA{L9LHLi7Vwkqe@>!1wkGe5jF9&>{T zTif>W86*rH$gEUU=r!TnlzUxePgndh8{0Z)buoV<$s+t+kgE}y;GBYRZ+;p*J`f-0M`6{ywHRk;p8YQ`eo+}&ZMfFS6f z>$GbT&*Zt_okMLx$}D&U(((~Q6Zt(okN3PGB<1er)qJn@sy<18!!cWbjQyt(j*DjC zPieId968NheP$y$%^J`L|H>IPh9+yiS-p|FSRv3qY9Y;lbh+AVh`t`XO(`ce&qlZy~wj)Bxp0%wM+E{zE{{+#!D>T|7!cZ>(8xM{a~ zvt4i|AS6C0{tNU3tDWvwCwPp@;l;h>r$F}tKVfMY0a7|DzKOp3EAW;Y1sk0X0Ts0u z3C+pezGEtkJ0$d8j<1y8HN(FX7%u<&sUeqE>;7}i3c=d$)@(j^$9tecy;E^$7{Q!V zos2bYN9tks*&YCrs{Hjdwv;+g_3~bhhtYVxjLY8&vJ=M#Qf(38KR3}tDus@+Ycd%O zW=-xD_`~R@BaIpuORR4m$Ek@kn14~UMW-8IzW!P?olc(4D_zN+*CVX4J$k3<013Ny z4RU|>*fajNJm}V6U^kWQl0Og;Yp=~f@z7DU;~23wV`8IXu)sm#SpVz`U#zQXlG4CV7*%|DezrSX%6&(N@%b zogNxr^2Ex<IBV?KEwdBwa+=}0zS3v?O<#d(MLW0rgG>@ZVE8T2~_Qu0W_s&Dtj z^|87B6a`EAJb4pA|BD1HK*bl=HxsDm0Li$F>GfFS$#JhI^|(xzZNJW}&M>uFkqy=2&Vw?4L=Wpg5gl(prxf24C z^5}UkgR7Hrq$zUaAlI_#1b>+$ zW%mg2?O`wXw3)#J({5ex>dy5~m0vaTbzgB3y3e}ZHd?7`xI-)a_o!t$1O7=!qhCrq z|F(L~iA)#yl|G=U%C=xftA`|SpKUea>d`gJ z0Xn1WgWA>W005nEAOUP3nC&stx&U+!ZCMtO$K?v%FO_O1`;qZ!K>yC=$#^Qo&s|m7 z&jgKeK3_+#a56wvvrW!AopG0!H~m902=q;S>~Cbk9P6a#Yuyt!e2Yh)&Z~bPEEsm` zxy0z{tP>kP`id=$(lhchJ$z4g>0O-VWYX#pH|s7(<4cgP`-9||*?D-RO%PmP_RT5A zgjc+4VjPXKG6khadz5q5?}JqVih!$fC3J|X5)0SU^TZ*>H=Iy|KH<)LOPT=$f53A1HW_0c?@Bl|X?2OIxray%3K18|0V zlUr!7J-LwIGn4PC=fF=3s@sZ|XUuH)ze3fQU;+IsgmNdnnaf&1LCf{LxSh(zhn&X7 z2P>@YfX|&FH-iW${BEw?C>~H3Xg(b*!D!el4R{&1)tgO8C`Vh1nB1Yq_&YbXa4Ti; zmwIo-tmBAIXEws6g-|f(ak6=p9gr1AoT~qBGt@GeZ_1e2CX-i4kR*Y@oT243j%8241xGgIj zhPs@rLM1t*)9fOF)Vdiou^4t;x$M&i<9MfH zB}h6+I-HE0%z~~-Ig1<1k-v+rmTIRpHjqyfzvpe5^w$NqOW!U08E?p@yE*rQuiL|1 z6A1dfxosHsp)?L%Lhv<|-}0J&dfixSO2eY+2x1yH2 zA{0j=I=U=Rj4bqPjZ_N4fu!9DH^joKpd$CXR7%1Tr;UG$P9I;_J}ELl?-fp_46KI4 zrj0&-P>81d?(Fhl(RX3=wN5sxtr+Lw987x@n9pARBl)=d5Nhxe(SDO4^tWiFIdmxT z8>{XxJsjeekE-oEVDQHedWi@?Ktf{Tuby-QGK%Q!8jjCRrie6q|KYrO48hkTf z+Cj`3-(O{sHV#BUX|s?|^_KN%q+obIWudX@(l*zyH*M|LR3G4kekX z03_xGg}>OcF-y0QG2|F4DwK6}luH_{EE88!aIdc!ktE+zs#@vxnzFSd zl+|&62My%b5>l;^tsI1up{HxneCc3dI{+T0Jo`#|uU&T`LF5&S)autrg?)zdNeE-^SC znY!r(GL9?}O+A|~JJ_erJ~J~SKgkL$NCH@X{$5Qd#24xwLZ0AHZc|F^L2q6_3cIzc zo@{F(W&rvT4wkv_HlShL-G<@_qRy_WHsYNw9tK8@*DD^!WLLW@5Rv~R^d%*xK4!>} z0FLCyyr{quNg@%U@V~PuYn?(<0b-U>{^YSFFoByF6eJ{;(a}h}V#glM1Qrsf6e5Xz z9Ujd_%$y+e$eb;dTdj!y@PP}r;miy6Hi$4b_}fl)4XrRHBvnYSk8 z(gcp08fA;vub}ba&q6f#G?qYNP@~28&zNy}c6JSI?Vl^+p;7WPm`6UyMjEMv+T~1B zB^ui1vKFx5{25MI z@wZX8xwlrJXF-^m9g$9w(#n|CHZv&|PqEQ7L^2=#WjcFJn0oDlq^x6XQu^oJIK-X( z*I^!yclqRhYgr zSTkG|Qb0@f_5gKApnNf(gkQzXlIdTcR>^t7A26A4C$HcEjk$#!9QleP2@wBsdaKe% zYA_HGrfQDvs)}U<^29&f6M94~Q?qY4zN*cj2PdT6-f|xOouNCGi6jiob^) zyoEn-vt}gb#3|;AA&X|RRb;OrY_J=%{**)ri(vBSHzo~FK_XDMm760uD%hLGEUBE+ zPE@qjEvlzLQroCA$Fx&g*p~+`0+P+>OLGOPuitxt)A^zwR-1ur#H@K@KsM)-ZYoTw zhU3Wf7-13+ahE41!D83=TbFe*oK!zq4$+vFhEkdS&X8elEhzzmo}D_ZT=VXg zGM23!Bh#!5&n^c{kRK~CCTGN6+?^k(LMr(04<0IoTDZ)k`|% zBdK^gM+p~5ufFJ@9w=Bc6T(7^@>OpCkIlHiS9D`}jz51=-;k{0vc%4t)8FiAbLZtC z@mpuy=WFZyi8)OCGwl*0gATF9rkPqKH^Hu3#d4T}?jw9ztgdQyq}+dmt6u`+)N>Wv4I&!3jEd&tvwESR)5tYgpCJC3nrW+wPE)0w zzeYEW`O;#7wk?^lwp>YC${)JFDp_S(^=*VK3t<2A%wk`c&-uo*bgc4X4L;$}q)A&m zxxtQlN315J1fu(j4pW>kVGUm|8(LAg9Vt&q{D0EenB~egbJr$@1MRq2ESO}W;riS{ zAp3&y|5G$D>GTGPB1$5O!Qe~WDEW*P!k?CBrJ9tEND5{-f=EiYD}bzKc!F|wc*)w+ zZpt|FrM^V0y2rpP?l^Lx;l%Zur~Fu(fb(0ZCp{GV^wZ<6Slp@M=x*nEk^CA&(Rf+2 zRNr}u^1d>=m#heGBmHTnH5|n_o`)s9(Zg8SC|z@NIdkM&lgsDBk3CWkHBNho5WuKb z{SYOBjLX5^^j|jnH($J)M#{~8A&Dm-3L5sxVBCP^-rkFz<$XqEAcvNl`Rv1OUqv9w zea%rCJ>zPJR9WA)N0h#>DRjCbPzh9Fz%FoPI~KiOrzYs+xw;x(~QY z$!UGsFlVP0Jq$~OKfEAU=|W~NlF&Gjv&QzXgv-ao>{nZ(`xfB_FGun$my8}m7fBoT z8*C2pewD5Ve;*4n*{(0GXw3^LaI0W%=%~|jMn#|Y^e`P4$Sxu7_|=rrP-OCE#G6<$ zFwZZO6KtRosEL{(`SRpziBosxv}xL!>KmKIcLn`XbP+Su05a}8$8k>nQdh`d8O54T zLA2bhBTu4E*4G1QU$91<#U?G`v3}QulYA2%2%=-L=Dq53CgnaJ8rEP__un$`S%=m5 z^sqaKm%HU^9J>hA(9w~;aOdg}g$^oG1GBGeXyUIy(vv})W&fw~kXR@?Xj8|q{=eqY z{JWLe)`K>f_bXLLk^H7LwXFqK1TexdjqT^rRh5=tYbQjp1PgJ=iDgQ+<;RmBoe&QU z+=;Eqw&m9;IQ^ZeVF~*U2$A39pmBG`AmUVYDNjbp3+9{9??bXr<0PHA(y=^u{O`yu z!1a;-{1&vUPNKIXfZyxzeoeMBVa((fehUyR!KGF`h|h^iFI>jmUl#(bL=<7ui4o#_ zrPqN=gvBZKQU#u$WI=4ZA%N!qeq;v3slRW^mMp^QLirr!&zz;t_ePHlUy&6WEZYsz zZEW@+LeePtaqO-=j*V}v@paS4HP~>IIW5OL7nfWWLdVzdyDvTG+@@SCUw&{K{;W~J zU-s4v54a<|<+#vy+KiJ^#k9ik4n2%4@y5hfq-JG6EsSug4w;DnC6xoN{|RU<#*N)! zZv}tOHW;$P@T?1%mv1zh`4xgcpMn7(HyF6B2p3MXmsC*bi}Ca~^WTDX%@9x#gP$`PziZNbZBY$Cl4GV-bdRwntKOho zT*Q>5uANcJIgPcVIG6>gwL>}pX4GD_lZErh7^pAxPUmZM2ODWwm!K@A!W$yz$t2N6 z+;*%b96tj$+4$~2QR9OFX{I4cHLMb{1J%@auODb7% z`?~SgnXw71dusIpuALRHJfje3OQVPt{u}0(mqA9z__yrTGvVJKBJsa?=)aI?1_-%@ zxr6_HUv3ny%K0hbGT4izS>et(&*vh~nb2=`9R-mQoGlPQtM17n(R?2$zPpoe(kmR4 z@Cl^|{}%^=tjJq|`cm-ku?pUq*eRTxOWzDgwGdEz*RS%r7P}o`lBfEYE8RT+jMX=c z?#MLl_?+mj0idm2++~v}{8MVCFC4fp2!6|H2#QOTtUtyS6{IV&SPa^Z8(V9om_)d_ z__Tid_C#Oa12K7jHR=xYPP$>?MJkukkMzzQlmkvzU*Te(lm0I^BRxSgRd0XA&4hZRVbmvPVKlgSNz&fbc%VVu!Y#RQ z9ABncjpJ12LpUJ!1P=Iw1Naz^3>qJYN&KD^D-6X&N3Qf5gAwEZmiRNq593>#Vy5_z zHmUoeWzClUpl?zyS+2sYS>HAgP#mhh$fDsq3wZW)eWh{#FOytd)psZ~e0vm`=K8Dm zxmYMaQ z%|}M>?0kJg&$E=u20MQv zLIs~x`ZS=m12T5O=^$Ay*>J^VwprohyseJua(_O@Wc*pjjLM2L@?n>rD)f0duW~cY zY+(o~^X>5c#9$W^DD%+_IPSe)QcHgkwm@3r#wKtd*HT@b`?ZvkuhH)8>->PC4nEKG zs%N)I_5GoY4ztkIr&wGT!&MHtL-sKChNJuMPNn`JO8IjQGU%i8y*<*}X)RskV~I1I zZs2+xsYSBgFLfH@6U7E@)$V#&^^7q1hWR28V?dU$ApogGEpAkL(_kgU~C z`>pPZ@f z2rlO7W00McfS!-UspQK0sD%Un|2T9QJG4~IYU)Fo5y?`@aUtkHI`qXHe*M5xwP&yv|RUx(0L=(NVeuhY+mV^NAUNmKR-y(OrXuA z<@$==r2k)lIr|F-O-MnqJ3HJ;jPYyp>)cq>I9W46efp#nGqwFD<9#gpFdqZjPrK*! zLl0Vb@B4iymDE&^REDJr@Yira7pvc0A7X68FKCUJFb>+*>_iXcSMn9(U@twUWqYX?gy(kaT$w-(0<99#}8vQ5Uy@`_N5oOy_Q5UjydSJ zL5L-5F>9_5<&$xEYEINNtOb-Xah?h&K)RkF9GupFCTh(`{hrNr--b6HeT^OT*)fS3 zIjzqR#?x+%@8j6K$HJq4dpiES9l$R>HLSeLMs`sC5-9?`fY{d48aSS zkbelHR}-z@^Z;$gi2M?eq+ReG0`il-#nqa_?p-C}a2pYgk!GFAg0JJ(n;h;26!unp zdx!-egc3-s`pqiZ#2f z=GW_UrypYMZ2Yy_;$*X~=<4iqgLEJcx!9dt*OX)PoMPt{lv}vU6Og+S)c=0%i08~t z*ybR6{r*d6oiBv9$SEFv^7Z+35tTHJLmCKCV|HU};JzCrLtR}D7m$} z$T3EK328U!YCJ~;HbKxEzs-o#qoBG&UbZ@czBHbl9g(3+Nd&8>cHk#wcY=-18;o!^ zf#PJTm7v(=`|8ZtjI%o-flki@<%>mtJ3>vpq=T85q%1lK$xH=`9SA8`M8DgWkmtY9S^4eI9r(k3KN(_vpTDN{zO{ZUneHwZ38AaG!V~4(MI} zGOp`Jzwtb5U{qVe8c=&|+s?YJk-=NeRz#|zSN@4D-KUP*nP%vec(P6NmTZymU2zSB zZ9)y+NvTND`%@SGR_R>+5U&1O!15)OXtd(}!TC{NM_ZFjYi`Ln10FRi<;uwuV0B^qzRA0{afVk;|6Q{JI-~VlN=0)U2S$d5e1KVxJ6n)f2~op zP{yt(XFK~)v}icZqQ!?F+<;h`+eMfJmhtHqJA5-Ooj0u$DbltwbhlFK}a17gxZRJ)gWFQc+w z%+|;^4=glZIid%|_yvtf2XJ#sDtya@NN;A|4D&o8 zF8j}ov2CNtex0N2^A$Z1>wAbA~QmI0J@(fO|8z{^a=2wl$19@}= z`*Ur%NQKy0GXNRltv+oK=>Z47HJi0k!i>ZARl;~lhsywO(TSU43o_p4lKTc+2dySR z5Jw@zN3xSu5&g$I*5oG?aKWkLjj=O$PcBY`$;g>D&`x-0oZ~092j`E2VJ#CAG?F6# z4EI_LRNASTJiOD0NG`7ZJZtY%}>;3j1dSuVB^H3t1HV}^jZvKNEu zp>NAu21T+?!`J6?2zfA-2&*pnD+}8{Bfj7qb>sSLxptB6%#!)KWrkP#(Bi2ZBI3LJ zlS3~TzI%ePsAUmmBw>@2N0H$hpLVG|5%E~Djj4@q8I?hx1MdpFtjy~b0U?*J&Mc}_ zkwlD|17$AXfGgxUsUODmoUlyKyX4XC5;wC}tV4N31|7@f6W^5a}Q+a0oy`nPa&GkS2_C^9;EgJXoz5p zM*llWrfSLodiPzCua+oz8PUxhJwyopi%RjU8j&k;IeH|l=Dr#1LB5S#TxxQJr^}Dg z0SX4dzn^gVp#@k7R806VxF1@CSFWfYWWl;#&iA(gIbfSf5v9sB?di4YVWLdQyb2G9 zl)~e%flfxoRZx_d_O|kGQBCv()JWmAg(wrqRAPb?snt|te3fYW=7=V7!wEwk+~3l{ z_i4PA2GQI(amMl=*_|mf71P5~N?aTtXgkP!R3?ybAFA=|UGR#ltRAH`YoSv5kz+E8oRLx8JiUlTebj3Y zmS_=A$Jltg;H`Th?mOQ zI`6qlVrAAkTr#*$?p+sAH@}CQyx7>2T2}~U`t@7G@6|Or*@VQdL$Sj!aw_?5s&<>cTP}c zw(RwpVA{-gaAl>GT}asOsyblZsCMzv1DaE@cxFvyf5hkeFokqYv|zn07#qARX2s5orV@r7qUs!JOVEO6qGDwo^tsQb*|FlqN)PW(z2*mi2sQMF- zq@oFaE@lVlY_*ES5%~3$3EN(GK;|i@eBq~x`yMNY7G)CQ!+CE)r?9>b6a`w%wHE}` z+YZs~w3@P7caxyV81aR$m8EbxnXDCZ9Qu>0u2a6Jd8qNWsBu5PwjFyjtdL4^)t!?R ziu9VSu{ol2WT<+g;$hXb>;Bqmx$d^E4%4%uv&%GRRFM#m*dld$n4YPeYBy%wpcSbU zrF8d7b{@=(J+NDzsA%O+|1_+3@#-GzmENv2t1xWI)MtoW zd@S*!o21~buj-V|fGTF3hisN`BL~{AP(@RChYreHTZ3rXy*cBlL}H)xJ-aZ!(;leXf z6IiJpB*yA+g~vXWY#C;?{ycX}AnYe#G0xxxO_%y}~nrjpd=Icr99PvBXRz?KaRB=uqAN%zD6s zIIBz`sKf>bO{8f`7KePTHRe{N2%(_d3mN2{_7OT-y*BvKEJ``loQMt3S9{ySI zx~Spf{Ly`nC}{z?B{-|)w9{U$Qc=kI+&^i@1h#7OnF_pwd*0!>+Nl^g#fa*y3Up&2 z5$DDAm1=QLOsSmDK#oJp684Wnf#Xg&S_4uzdIc}}ACnGhhWJ#+3KgflbO(E&w4BZ) zP@a_~e(?UI{U_3`ObQxC!1kMb$C4q>QL#xFOIoBWBcUAm@}jQ>{?6Om9oy|~Wg-*? zHcVG?tz@D^HlsXp7n^ltG=Dy|oL|Y-SJo)@nS~=iKR>w67Cc5V)=NRY8%=Wc5P@x@ zYO|Rx-ajuue0JtivYr%o$C2#F555zFXZ8-f}XXlO{0t`Dk3mRPDl9X!~6;$BVU29sm8}Lap%*lKpWbup|B# zLUEPJ>=OmJG-Y^jxlX5Rhc#`CyU1#h^fOA%XTr^M59Z(5Uv|}z!Ju+ul7`H>qI5>V z{Py-2H5+yNQ&7lGSFODs>ZfAyW^%yqp2rBtMslOWrHw>K*}cAwNIVu^l3*I;@nnC zxk02_L(ULhwo>ESXNs_Tx9w4pI&Fv-?v23J~AF1z*5qS0~z z8Jx~rJ)wO+XxH}k=cBU(?E0CyqYIMUbef6;U<7wy1O}zpTzBTFh zw|eU7`jGIt?(-8E5>LnpKP)vp(!GKMf#5b0%g|`tRUg)fxms-T=S!|5 z$w5?~!K-&B@q<0b=`zOJ!qG@~4u5rgUXmGN=E3XW!p>E5{h^pt5Q5c&{v?CbrL9aQ z!~G8j!|sJ|jdQkO!ua}*-|O<*6L)g@s2Y?avyhj=%l)^@maeoyH=_5%e`E_Avr8{< zC5HD9^GGScB%;~jQZV9p?kP?aP%`a&?qfOz4<_Gr+S?r+v~NtgGQf+J4@|x5DySUI z?DWvjX3va4=(u}nL$SAdp9e^?c@Z~UTU$F^=^(RG%-tG4;ut;EXJmwNN|=;KzW;_r z03}RJHI4uJS7b@w1I0r_86Ko68@pF@6-Z){QW*#sQras*(PL9G*cVclN|jnJ|xVKY(_UM-~;ZD-1_OU^N1y_!aH-M~+`6U(IbvwXD<`2nL}8K6}ADce+sV zxxk6xL_UvLPxo@YU;Pp5h(<2O(5X(u0`h}uAB<(6fmGeI)~PkN4706=sR2pXPmFP{ zTqmllT(Jmb3pH;h5-XgQ;r9?c>hs0Cg9IYyPR@8ELWfV6G1yJq>(*!Pv1VJ5KjYHL zwz9B<`V_)-=e*?i$7`dP7c)NS8?FR?U z>h$n4#1)JqM*Hx6D$&i(eRbwE^Su^`4JU$hh5%EQC#L0BU9 z()%g^>}W5sJD+(t!&FwwdU6p`?FEiiREm+rf4sOGv&4~vE_ay0XAe~UOChYrL>i#T0PE^wor$Rj)_Z8 zMg3PR<{O@z%_==P--au?qcsv!wVpN=zp_#%=gayvPFI+}JjKM8Y&wDAAJ`O9|4fIf`QP$lxh9T@jnuxGfsX zqkLQ!8GaVgge?r=C%GML4!yJS(=HB{;{VNOs!r0C{;N z;jOu`)XO5cDAVlo$(#1@;Ld}y!!)4k6*+hkOwG1WO2jgt(x~ZR+fw9W{ep}we4A%6 z%fDnGGMECrzG^!W#R|gG@~qgr$WvA{XcpIZC7nu63+FCAEhHERJm5&>>zB-yegWH{ z@CD)Lp&NjY6kd|?$KUMe{isKk%iW}GT{5P~Z1}E;CiF&gP|0lTPo&j~i52xbQPg)r zKN+~R)<G{ilzN%f6nIkfkR?3U)8!iu*( zK*ce5He`Oalg(89BUtQn{A2DEU)dqlc&|z9;d?aC?X#y>3{;{YpGD)co*BRaoz;=I zC3IAWVG@kEEJn2B#ZPzfTws;-^#R?U?y7};3vg}^Fvr$L@rv(;d^W8z*dI5rI$o>{ zYuiE{(>$Z;DJP*gPTo;fCg!iZ?s{p6zOk^Ksnh#b&x3<+7%_%90MXU+#ahl(?i{zYKyf;&)j`>U;dfglP{d}_7?t&J)CdgZ|ydQG3eTOc&P|wFH14o zGqncUZCR{bZ{vy74Z*MUGz4URSDTMijO(6Jn`*_)A4-#;Ll`A*3O1{y!b~Mb^EnZB z7&=rcVZN0sn<2x!sjQd_j3t@KUNE)D^oxP<}8W;|0!DhT+Sk?f%*5 znRyj>WV+JO{SNk7?;3(4()njaoWV>1wDw!L@X-cmHRen@r>wDG=bq`ep8;2-Dqd<+ zwqxIj{QXxT?OZmbJ9G+W&&^&XxrF_HBv7Uhvh)WgyIu#YB0hW!ZN`}Xye*9sR9Yf@ ziMS<+Lt1~8@Sb#jFzs6wYBK-WbLQNjm{(l4!8EbVCr7Q1#qJhY)dqZi&>wE@^LVCa z`H?HG8JSDlhvuo-bTC$%RA?OyaT2PLTk=;sr?Po?X)z*v1yfg2=h!5&i9}+zjHI;lydh+=?!iIi zitNjZ?brp{&}JcwM~4XpafQ`|4H`DujDF%-{8&B$eM4}k{z5uAQ}s8}>>Q~@Bd})~ zUbD*|10tY|U_h{&3%$0^fv--&5+_Qt1HPk&)8mby=BlfFS*AVN>p$YIurjP>g^9_7;cRR1DZ(SZb?gc_INnh*|d7JK)8gV}=FB5Ult{#JZ*E&o4Qd&{6Wx~^TA;DG=kcnB8U zB}i~f(BKxF;O_2DaCZyt?(XjH&fxBYe$6e*VMptZ&`cqWxdw5*N7;ji8u4+M>r|GWcV4nn}kbaNg=l0rQjb8*O=M0_TSA-jBs=bFd;?`DcE4g<@8XLRL*AWYWhPbV6Dkv9T>VA)UZpfdlEe zpNX_4S-rhtq;MPxhQ&0LI??LAR~{vyq#woQNaYQgvnQ(dg zPO5XF4)z3Mz=~vf_WZrb?{}ZKRcII=I^m^Q1aKf|Yd@A-UH4TrV@3%s)H>3M-y}lL z30*m8i`L_bTc?c?b7#6*rzk6P57gz@kuFC zujeI)oA5N}YAA?5O28V9D5it3pNl`6bc6@@-HY2|Dk2K|T(^KZ#moYG4}o~Rg!ig8+Ip?)qd$TTs*9^`O}#B#n$Fs0$FjRh*TK)GeLIqTE9w!^oZCN0X;sHB`(({ z0!INqkgA3yZWc3#jDiZ@NvySdbtPUTq@@>(HYYONxLcX^D^)Dt0;cNPl?gjK@QyQq zLV~n#F#Z^3Dl51}Lc4g`;p~hxuwqg;5AG_Eq<58!1+U)DpF7?>8rwPP^4xiP*1nw< zxj3osv8GRi8FHXQEgTLG%P7W3l!2_@j%qzk9IG|G3%K2pHepMKquin0@917!Z%QQg zmX?uKrghg?$TL1Ee?9aY2`&+sOV!6B^QKV9F8k71+mwltPkWnDQnN|Ibe;6`RW9J} zz2g??fzRfOm`k70bo4K|eAYU@F_}CMZS5Y>wX`bwtoJaN>8y;GhjJ2s50iC^hF;p0 zBWp+5CJ9l40fy_e`+BDs{eT2=p_`+Ur?P+>*j;Ea#20B=)dob<}mBRc8p0bRL+(mNs9-wU`Y(-@19(}zL zmb|gdQT3LV1dNPI?A9j;_dlpjFDN`5=G+BrZ|>%9Gdi3ep zTuOgm4{<=@v6@hdea(zO4vG6+SvxNs2m3^Jw(xcT(pZ7|$_T7^1hVzcGC19seH2{9 z`RZwI;W0Y&tmv!Rf308Cmxrh`lCto~Wx(oT{#NjG)_b)Dgvgx5{j9#a6 zGCmX$2Bd_URGprfKv!N6b*`kEm?IdunWelz8w|M@&ET;!0RN{Jk}7is@&vE4wP~!k zrdJ~UvqW-K`25Hhu=}c{(cqR@(Y(8$L{|LTMzqxwHt!C%Gb}ofnq2cmxZDdh^k`Cl z6n75}%x5l5E}#(d^5SJjp5{QhV$58gbjX4W!TwEc`kmc5-ToV76Z(%=_ns&`=@e#d zdN)_NFL9sQevy^l%tPSQO7xvj^BA54X`~B08w3`8x;JAoxxv^uo&KF8Y@^|5@R?ws zukY#;B1CD}eBu+l<{(G+KsdrMIQkP%#A@PHGgzH~X*q=Wr5^UWXG zFg)Hm2Z&X2s<(2btJ)n%U(5aSMDIkkP{}&4CDlHpdDdgN|Aj$#DLS4|aV%k^HW-3@ zzL6L`)dUw732(%gzVCd(ll2a>(TqwTEHJy45O=%KwZaefT>M`9#F1*(F)0e+tRRd*+5%wE6rM2R0Z};e62{WIZ&`+Mj(Fnk6 zK|7ox$T%mld>ZM6MeScDy#i)|gJM28W;iq@B{EAY;=~#=ak7MSLM1e`@MG^C_S1?4 z#DzlfNj;J_bLVtudcS>wY3hAmGpKa+VAA?_y!u;1kazcTjK^SlPsWx)jU?+ErId}| zPGJu$U~bwT2{9HnAUe0Sdfwb&9~un*+WoHQtPwIVf7t#9HT@E7kcG_x5Q?o$-;y?< z(uDiqlzmu(Hhuosteju`DZc7(uB>wLsF5fkb!jZi#%!S7foF5bD*f`gsN{_D$S#nO z3u$msY_9LAmr+0E%BiFfd%VnTF ztRivI3QlRWdsK4Oxqqyn7dDq!khl1Li8)jL=F_n$(T3!W3Eej9*hGbAFRS6(Dg9!y zN%_@Uva*3b`{DZ5>5-Y2XLx7z)~Br-mG&0(SqC-c{%&DE@xLmH) zTK^KZX=ZmP`1G*kVKyEA3F<$dZgLbF;J$(vg`44|&P?vX#7yXnGa#SCqPw3vY&5<9 zRxW!i{h`c~n6)AT4_zT&y@AGRBkygCTX?rPr}9}BTQQe>0`ht1p{Ocq;v zhw8@(m4aIn+spc{T7p*X1K@zaNcm>23XS5S2<}QJ>z6D}wLjtSBBu=vN{F%uY+60X z+&(=}dQ@_5A~U}2aV!dNB1Xm*9X08FuYUeLDA?iq|w~5?+%#x?nBj*MXErC6l~Vj=tNTqH2TgYLHUv-8Gi4u!|=V zM#x;<#UV;GXbg6AxnZc~(S2-iUV4~OM}L}m@0$0c@o1Ip4z&BzFQ!XxajlQB9^%#r zLLuKy81b409veaLCLBIcLMTAbJ3B%@s)vkLfE47p{tzv~2&9e~#T3d3*Q4L)E zFJ+%J?j`h6Gb(ei_AKv4hBc*B(iNTn960hi0G4g*!&I;WdicLJjcj`ngVs3?U52rB7J)08U0X~)68@*yghf%6`cH+)0KT(p) z!LGMY4!{lZU8D=PJVqDXO5GY$70Cm`CA;-b6*VT_ zbJBgAR5*2&>kQ2~<4Kf)eQeR~`O2gF5nk8O9a>}nkz~>@Ng2JV3xe7r>;45O}90V z&=@v*p;b6s^1@MyJS|lvJdP;+csAl061a9`Jsd&wJ6NML2XD@4zQ+%M5AjbSp4|I( ztat(cBjv120Z+53`jgZzKC{KPCwwSpis#IgqD@ugE30_;Cd;sLY6phJc;V!toWVTT zNSr`XM~FDZ{5JtidW5urU%5TeLX+hORz%V``y0Ng9#orni*I5`CUUP@pG>lZjMf+_ zs=Is7HLzY6DSb1c8GMj28*g-3UqA5@48?LR3T5j~+xn)sCNTOqIx{FR$&KKaYQ81W z0>RF#(!%q+i2j-0|ILK2(5m3Ea(eIRMdgf%+)^LDod*8`m%InymvkwCQUOq${mukY zyWc4Kx&CRe1che5LB`l#W31t3WsMPeg4vkRu6Lw;W0!N!%pl>VUV}N)awIb}X0hy% z0BJsxuLHF%;^QA{Q%8)=2nkXHC3h9CZ3$cTl)l53I}w%ZDJ+`Ej4QIvc+r6Mgep}4 z@oI3oOhZ%{{BA7kt&`+*+RptLZp?B{ZJ@x!woi96|BHlO246_7hSP4?AgRGI5ZAx* zZiW%>)|lB;E!Z0KqUSczMvaX_wk0tooYnBj{bQ|MHp)tRC(wN^`_ufw* z{an|+r0H9SzEwl}Yt;rUJgQaQQ%tADX*JXGA|RReQ%cI)lH%Nwob$(v-nob<%@UUR*q8 z&2m99;5Npfl3Rmq_Yy*KyD)UgaiI&ixIiy6Zt7gIX_(LG`AsKt(sOb8ttBd@r82w! zRt+SvE6m9WPCJBDIw>lU&C78em6a<{u`7(J%KWCH4@awVwH(heI$moi!g9Ige2yRk zORI2@5axoB#9Zj_BgT_v(D|E*L%Il9R44^w`QE$`nFBUkBhj`xxQQN3q+Z<*^mtp3 z>MixVAH?D#E+?<8ijG=gUQ`s$YveQt=CZezPA^nYdbl=Do~>W*^g8KM$d3lqLvBi) zuEKZaTBVE2?H&3i9TM zJnhaLZj`ZDU(3$wR>2^&WK*1~1;fs8m$PC1TK5daLaMy)I~;i0iuc4G6(aj{Q!E_a zccfW*zRAGjSdzNj<`#P_MV+<0i?o47IQ2VAKAle?lzSBd4lIy5p$GNZ8owU4dD%(ht(^94R!z3MN1)_>mJhiy z7op9-A?G+Ck&okcAbg;=&>gryn}jh;zQta{sSSTYGP2e(;+<#lCxzR-pJ8eFfiA)p z<@f|ajd@Bknjp-h6*6{sw^K4nt$CrSz|D3oS-;@92)ii|cd_KpVDzSjt;GLbZc849 zGxz+@98^19B5`{)sFydcU-R8Z3;f>H$F=Z|j_h+@ks8lq0}1*#@V0-2L7`SX-71zs z{8-z{8uYQ1S+gN2W2H>D+d^wKdq?eqQ0k%4hdf+M`&a~|WNjR)_~RLvQ&-_9n<$&8 zC_3y&|K#e`9yU2Vzrbofz#Rx8P(nkJZXgiUyapWv?H{#3=C>H$-xI1RM0t!c8>#Wu zCa}0{-~QmJ$+{gMy)()tI8Emiv_5u@VN=sPb-3Iv5S@0kCE!2xFI8^qQ*sE^5mW;r zC{@C(tyO9dk5YjcZ%_51U+>IM{T2yZ48CwM`6J9cmWQF}k60WQFgPOMU0@5ib6b8g zL8g|oF_G2+8{6LLzITrd1|Ob^N6eiIAbhR;$P|$a0u|MU6q$nCoP6In+U>@=mz>Gs zP8j4@4AhTi%v^r8U!f=|vd8zxV{>5C+2G+@boWu4qTpViex1~&hz~!(R0rafGvOA; zhQmt1meKdeL=erg_1nUm?d$kt887rJOvU#uF|j~M6}?o0wvq>7nQf7ujSzw}qUtot zXq6tw`twQTmW$k>9D~ZdcC=NzPO|&PI4E;?;li~*sSW>&!Bc-GOz_WU%Sfh&#F^S$ zY9b}HdQ?r{JVn2}u-aT+Dnty*IfO6o7R5?dHQNpCIKVMF0I{*j!~kzde?#_8*juvx%0hko3D zp5RN`Hs*}}+Gsgz>lE%pz6=ZHcKcZwq?hi4WnzrcYil-Kdr_**rrz{DPaX%I|drW>3>RV)h*r zo_Sm|6L?MB!uF?p4buOOu1BJJ|Kxq0vtlcdkI z!?+pz)9_;L@0AzfhzAkanZd0$(W(#NHot%zzC+y8Nz%i#P0U{o)Y5Bm63dC#j}hD#E zY_$zB6&7Q5oVQCac!%MT&+6D0*2K%>S#{D~$6A>5D}pJmR8zBuUG3&GmIICE_1(P~ zhxIXhVW$!$b2`=K9c08|UCBFDYmbfJ98Z*0R5q0I*p&o*##iXCEF*;rC-T2gAtKWG zK~q%)eZyslU~`%6W-0P`gglI++0_nvUb1gyFFd#$ryP zb+FrXsk3cxVFfD>LBlt78A#2rI!WVMZgNS^eqaxvkBgDT!%k()r&F>ZzuZmx%+s5K zQd0%TIxuk&?19*qvAgTK=MY9XX_%oozcV(qX8sicyfvT3ht( zx01q|k5u!F&Q?JE80wHHgb>W6#WZV4%NNPjHUndFrD>ECF%KQqu=z+n>r(q}Oingj zSsgU%7ksRK%%%(DD2b(7n{F}|qSLnBm#9!;(BNK!2Yq;ZL@G5%WZnG&Ex#K0Cr$LW zT~UQq<#!MI-P>#+qx1gP5B#;j>~&WbR*-syYlLR2mL#;$<=qpOSZLJS5X`j238dtJ ziaps_cE60?+-Ags2dVnzRKzfns0iT)_2&DfKP6hh&-U)S-L4n7tpoLnqBL}heweE(Q3+v?%z7W%`^JU7*w(uPHjLvA1I-|7{7Vv zZb5?8MYl^4n~b!0VQlNL%~|YPhI_%@>z6URxd;6=*9_j<8$N^wSB#25#n71ZQwrg^ zF%e|%w{zH8q9@Yv&v*b`HH-D zxdV5mu9D{)*7-7hAy1XTP^t?IVQc>n$>?OJT=PT0v2sg zR{iwxKx5(~=M~QFD<1nSELz_z5P~50f|byAcw~+6nY9 z)uKS!iIOP9!$U{xEGLad$kO&F7xf8GPkK5=BjMtE4snl`kHr&~;cBhV%^- zpvjJmG2|fGaXUeczWRs|j#sRpjBT9gOC$i718sOw*G+?8CcCyIANai1gF_FMGLS}Z zNCRdNIl3T4LJ?GbaD4_eca5lZcJ;s}k9mjpTWpbqSGDWKu!p0`v7XwQECtjhZ!0No>4 zwCzypdD;GBs*awkYj!xN(tOR8aBVQ0crY#9)%zyoG`hzJk$*5EHjMP}`X&GbQGyf$ zrS>oDpOfRu4CZKA(lmJPoWbjd`cj*T5-D8qEQy(aLx`Ncn>bAxTQ)NoXTRUrC3fSg z9}LT>0u|``ZPw#DC*Z-2N&B)zA$Xs`0~s}@l->Ai1Erw03hT71g>ozM-Om>|RwU?w zJVy}1KAjYsK4Sl*%${9|N)T>X34e{lbP#yE(2_PIzEkBo;2Kw^GS-omvG?Lv2m6Zd z0$eYC=-OeY`h6F?KUt;jimJ2DqLP~W7`zET4DE56I= zj=_QRa+N%tIvRR|SMN|gayaCNEmmi&vH<1go(ihe3q~3+uux#9xSEwMQTM9i&o7bldVvA7!vkboE0{<+d zqdBcgxk#e?pG-P>phvEIn+;h#S+8dm`D)OAz)_$#tbo-u1R>bp#n#D@0qi2?Yukmr zd@11~F2Cb!3cDNG*uM2;m172wJ_PU*F&W6z??{=i=mx;GpvLuRX@9Hm_j3jgn!w+G zp@1QWw9e%)0Wd2{fBjS+W;b}L5pR&Sfdx&J*BJ&$dt?nfF0cWW_WkX@jj`7pfaw54 z4eDrMbYf~Xo;TGe;8Lard8(OzcCzm#)lt3-ql)}{z)g;Tc1B}*27LW@4~hl-HAG;l z%P!M~P$Idz2N_B^cWHV4e#}#jfUHNnJn$_FnpJYasQC$^|@%WS1`X< zf-!;iq(NiA7l8>%s0=3<@&8(gHR(dAEG6^9dS`Ow{~WWOL9>Zf_|7Ux_J5kE;JM`6 zTYE7biX{L05OcNG?-h0TeWUuDZ2#kh<8$pLm5abB#a}-yr;T$y)?O|c-T|O*2}Uvb z*tEb*q^-kJ-@Vbg_75GT`J0j(jCcT?Lc9QE>;N!O(i2$`LO8aUj30Ed|G#jcUt0&=-gMp9U!We7rL$#|z0XKhKh zjUm}K2y26i<_|zM&dkSj6sZtX z-OuEv1jEIBCM1F%L)THE3Usirx;#)WRp;UtYLsc{*P)c~l9>yNJujH>y+~82sB)0i zNAy06aY=}2KTeJhY;;U%Vj~8~1T+k$DDj#w#=mKv>k0Rd#fw!2&bci4=MQ&gyBM81 zVq|>x)o?mIt9ZYpf_-0A`=G)nm#XB0%fYTfW%W%ih}lhiSKMwtv5`p z7%Q)h00A*iCD`V)54cYyKlKzNf4A>dWn@ET*{NP<&IAj}6o}>0ivcrLRXGs6v3N`s z?{JA#{;v2DG7Cfvo&vCR9F;8cfzC2w8r>-q0Jy>Jo6w=ZX#u7P^_mE)8M>5sNrLcQ z712H{;7nvfo0P|M^QjPE>oK1&ezM1W-_qa_wrigUcn4% zH{4f}^^%G83zB>S{MG!NOiYvPHhUJODeV0#s;H!_Nk0H8_$-PEI*l327tF>OHQV*JWRw4KbJRc6U?}fLiu~++;oKc+ZxtKUG@) zFb%17*eR;Uq40Q%8r*Kk^E}eN7s7AX;LMrGd;5*53Zdke9-R9}H3Cmf2ce1skq~QC zfS(p9sXCr%7+65?BzgEp^h1L_``!G5i77N3tNOz~*;JgZ7jI{9?Y0%Wjc$scBOlHu z{_d|q@hVT}Mar^dPs(!rW0<*Swp#Z1Yg+2<*?xD=w-42NtxQGZRA9FCSkx~NUmwTP zeW6Pg={^HMRWpB9sH*`n0&%$GI(RKkH258%k_#w|U7`EUYcy$16xSH|F8 zYb!wLn>SE-XL_H7D5Ii}gsbMjk@GlJ1Oe`<@%&2SCsjvlS~|Z8ak}4t@Z}OVC*1=( zo;wLxkWA!gz%)+V$N^n`S8Npy_{2YPSK_t2QZEku9fserFsFyy(qW5kvreBErp(GN z*EJlOJr2#cPo_o2ppflA}jc<1pUv*`OtHrSiiBg*3a@?A1Am7yZ&S5}mtPH8ACtGV{74fDB)b;jPBfP9 zu9Vn?8?tE%6yN2EPwYaDMk*$^mHj1=_Lv!ax!UZ`L(jp{k|Fh^YlOx8bzrySYy0I4U0ifhbE zrUqvEoKD=6%aqZ2K``F(QdL_*|okm1x>dL(p8z<5PhH1hSd%MUVty$nPVhr zqDWOV{J~eC>bfxD`Xyi^a={A_>wq+zisNs?MtrxpT{rLy>DN5Y6KgKUvN)W|56mB zlM?EF?U<<+=dHZq9=+UAeVR9uL8&U&gUv69M**53}~h@F=k z?kUVRh5PqAPB+W2LNga>c?P-^uOx%T`I4;xEk0zBw;-n~@(@LLnhdNLeb(~3pDJA9 zTbq6!$%B(ex8&+-#E9O~L5VfL*cWRj` z(wzK~?S=-uu@IN|3%b2ipk&zjgo_gu@?X{NRV9`!c21VGHQAznTF2oFz)B*LSlpA` z3QILq{<3=kbJ+2)Cb|BkaS<-H?Ih-*l5}8_>x?Vc@{fMCu`dfKSLt-@L+LJ+Yj~dl zB{BQ4fjmko-uExP>2D=c>_M!LMY~IRkw|#<%M1W=%li#)*Mr1hT`nSo0 zxX6@_WYb3xEuI57vi{OP|CyYlVoE(4%IHb+8(W)M$kDHJa7o}zS+c--KD|Ki;q4k$ zJSVdxCcHnq7}OL~L52X(Qz=eYt%YC=d&-?XR-JL{Aoi4@!oQ7)0eGhWN7n_wy4Jh# z@a8$UpybQ6!aM$S^mO4JaBkc)r0wBm{j}FY5k@}R==WE4ZF9sG4#VH>KLNW=2x$x? zfMr`BmE*QL+~vRyMy4pOD;FxGsqSlZdTUqwfnwQ1kIJ^0DMDh>K?#2TDC(Wj`4p}xs zRMZOZ{?vr-4*NMCQOoC*`f&T#i%9;j;C&ESVU=!&=Cu|QoqyEY3Yt?49P3pz?AZe7 zA^ta9zoe4fHeQUG^KTZ6;&PO$I?ng_4ntqJl8OKPYhk>U0Gq|FESTUwmei|pW?MiF z7-#<&K55yiB>U|>c|>&nj&$F)_u(B7Y72aqtbmV98~Jx5HB>|BBk5SOV=op&0Veq2B>o^jVvpH(VkEpvT$9l0Rd^rAx9k% zBtV+OLooUn3~CF@ta{~#dE((6u3ui8KM@qvTlL80DGQ6EV_G}T7zTm}{xZ;CR&&1M z&6^O>efUj6CmP)1#&!N{kTiI#CyN&!BJB-x>D%&T?)c4)I-IQ~ks-%-K(Pu8+F6HT zgi?gvxCtDTOA}+z@9}%ddY%ySi~P^(6pTo4VG$<{{&sTuPXXQB@q}EO&Wt)Eyhlq4 zX9mYt8iU?a>117ZP7|8LPmgUMy;P&^)t9DnOk5gSi%jF_?z>=MdhFiBdA2j}UkrHn0=g?Yobkveoi^ZS0y|K;bE;w_zi;Q^ZkE*@VIp^e?MB#ljraW{S_<*zQlq6tW{eiTg7; zBwuVcG0}2}VkI6QB$37w-mdB>*LNK8Ckk@DU95@n+j7|`RjZG2vb%BaISpDcSaN7C zq>(mtati>g2jCj3opIQF^E+4rrg&44%QLJ=)yu1D-r<6+TM#ngBMfSGx5 zo4U8LS~X!Ynnf#H&05_A!M5E$RYju{3#{dzLZNmH4Bo8OG!icj!dDkR>v1|j$?RB^lqVL`-NK0!@B)n4zs7MG5?bi{ zBedJKT17DFKem=Qj_Vkr6)x=j7`(V8_>sUI`*w}iAf1UGj%_vd8EXhjLaUsq=&S$i zpQM(H%?Ad=w&X!yQ8_T!qG~&8@{|)Z)LcqI2?-mh$I~Er<<>qMSg5|H*Zbw_PY6*@ z#3dT;!|zit<2D%YZ&wD_R;yd8f2hjTt?GWCsGI6k`NrW7$e+)s#~%U2*3$443q_u} zywmC2j0;lk9_TNZcc-J9b{Fd`{cXos$c3jlP{w+><2N!Eo&qi$lX!Mu z(jabsMJu=N(dS}<_;#F9pf{8wpV)YRP0Eu&_YUrJ2*exYNhv%2lsOr&JmpUOWl^0c zK6ih&&1J>yjq79LLW%aGw*=Oily6OBj>Q?0GRO&fw&bS(oTDmgJYg}hFUhvI=pDJ9 z>pdXTMGvoh>bby1r?FyBT({n_Usd2K@1+>xz{)(r{&VJqJ@$($C>!illLhvWV=iw$ z)or^I&9Tn#5&YnS>0pazwSm#6+((c322&=2+F=c7&$zCH3RCU}{F;=3J@%tRQw}4? z3A;Jqtk)#qxaGv`oJx(d_nc*<9xYucrOCb9RZ+B3yZ@m%9Ath`dvy546u4FOOCnD$ z2F309xTF!s)~b0fsV6GU@-@l>>yMoouV`fx=L73GZT)w)S);qTI?V|MJ94e^A4y}E zYSJ1AUX-M?5jvyG11HLw`VF_GB?_hsByEN;T)c`j|7JA{`}H>ft_UB2~d~wZ?7MzOdGG zY&~3<;b+BiQAy#S?;K&g4tE4@YoEl{@O^$djuLWH$d&mlO!hxaypE@*$xU@q`rNkO zmS%B^m9PoV2E|a+psG=q{x)BS40Mgd8dTBc4EmYJZnW0RbRP;CraWo1WA7emzms04 z|3PQ4Og_C<9gfNQkz(~&wcR;i;Ib47hua<62Xvo|zQakAUpi?O6>bztc7?_ucNt5foyV?!o7{J*mF>z7yOiRb>aKZgbEW0lR&A+dy-j^U)<3#KeaE_j zl|VVo7!eOFGkVsVq10|`R>HkCnk$vb#4?L&zUPy3@fHbuju=gcm)FrEkq5p$Z<^Nc zA;~p@4<@tprH*(jNvC(^HX9EX5`{N-$%ixcheAMIcG9?YkO#xewA4%g^NGgr&r2-s zmYnqJ>(#*(dK5nKR%J3Nlx8jYn>FXQ0aI2994<=0Vswz>z3e^3rnxmcUAjg=Fp?|PO!?VLF3g|&41B?5@dhDpk|M6`h>W-97vb|DjJFSR#a266 zn3_D@yD|VCAV8vevT{yaR3JE@4(%hKgDai7P)T&2GLDd-%edcg&QQOd>_&wL>U0-k z{pgOOU?Kv091Av|&l{f{*ih0*YbbxFYkjs{bL{V`O6`{6`M4aGmR)_yP`6T-m*r#1 zR%nLoL7?bH(WyKJ*{y*)h%qy5x8C^dXhxc;(ZsFCCPXoiyv-UuWU+YIXIqTsE8;Txyp#V|8?FN$+>ph!{LjQeqM4_OF#AvAlNvbp#&j!A*Ilu z>K*Vqre%VjMB0S+3KQQY6SIWnJ~8lO>6&dV+N(6PPMJ19hT)hub3AeS*VqnC6nyZ| z$9uW$Pzxhmp%{K@MzKp7!EEQVT_W;<;eoot-e%xaJaa8g(4*NGxvnNX1&v-np zkwX{e?#h?lF`Oo%TF=w>`HT@bYslFRP2kmS^|`$-c@?GV1O^Q}{=tWLd#>E>hpOML z=&W_FynC+Bl~P>90|(9RX|NUJR}3sJpK^x=j|7bVtpr8yj7!PI{a^2&u0>*3kHi~1 zIUq=m-hSR_X9RtIp?vA3J+F-RU?8#P+2hLHoe+<|fUZ94o0#?`UDF;5B=dXJ zv||YG~8ZO+yUtw?3as8HT3{VaZDplN>xth2=y7xHon6bu*jZmk!@s=<(mEoqBVhfbVX>XRugcf8GOq28yFxyVr6vb0n9=^_`QZIM!(z zb(xD#RQ)d1FpJys+@aJ>+7QXgD=?91z2((j`6jz*2OSU zpXVQke&m`5MtE~YlU@G8rEegR)UVY96Xc~Bbxw^nspvSjJb3sXY4$~yAjT|&>lzvP zX{PR|At4Dk@Jc2=%{!x{ORD`K;aA-OV)zOG|F~c;J!ml}+Vnolmn?JifSUC*Jhu6( z8>wb(Om0E!y$68BAlcAPkjkr1#MSsx`>B6jtR7XHs`xM1?H}Il ze`K~sL#xqy50zItjDbc`&f6?Y;GUK&<>BZ|!`b$!n=N3Z6%AlOLQs^j%(avLmy5z1 z3o|Br=*=d~zc#knMC)Ay6Q4wAo!@ozti&U=d1U-|+)K~lYH(YCRAI_u8APd-6BNqpxisd?YP?0D^O!!gG#JNYOn#AqJ+P_)*QfhQ*EKLx@K&gAXlPNI?cz}ZJgexStq zc{fT3+sYo(dfIGv$*~pMGneQRu3g1I=3Rlzb6VUH=c@D^1>i~=^GDD#m45+5_|3`EE30W{=g%pxpAu*a*Zrn7eBDo-MWsVWn2OK_uh# zY#HIpl79H`B=Us}{w9434a5;{;GXJ-(9NnjVwc=ZQFG%{M<7YV4ZbK5iF_-np~e$_&7r4woP819HM8tMl~r& z%c*~{;du*gt3jP~R%z`nFSb!5aDQeBf10bGus?&>pgH3#9-A+Nomy6p7FJ&aC15DnTi|&Cjh1t<4Jx zkuSbAxV`iZVRjnnMy`51K7(x;6EAn2wrV{05x3dm$o%gqzFA(#fMH}Xqwa+}qz+gU_pT-4?I)AjOHeAErjDw%NWTlyR z7|>vCGkQD=WC^0(FjT6qpxfQ{uz5^R{m9CmoITKyf}H_V0!L{?G?=Kz^JJN$vH9~g z#;v;bM?B9GETqdlm;S03%)Kd~)*QrnUgA^7w_JC&XcXSqhtwg348P{m&TEIVt*#TB zw~|qVdjdO?nF{e-Q2x}&-$kx#t}<_=J#UyVSP|74m2xf6Jmd4p!h4%tjm_;@n9jTZ5$om|%0X05b)u`A_zi6g>IlwJluuE`kriN)sXm6K5uB^xPHno` zT=BvvSEQS-<7vHkV9G3`9sVeRj%Mq>{%TA`r!<8KtUfrVR54xzX36R`CPe8eFD+Vo z*(jBw*@0jgyP0%eU?|IfX-3=o{!i6`>YQ<3?04SoYcICoLxBo9Br>6R_2=&jT)!9-XIy-*3QlJ35gk6V06XE=_k(^aA*+tIM#J|?cYztQp)L#muSw`(|OwUENMdK5%(GR+x z2YArAPGZ6}&d53@L_B*XaYQQTY^Cd@y|;zX*_afn$XKVb6>AH+j;`?jCU@WSP~X%- zaGV6f9dgr6jvrf4qx-OmQ{yH_2?o|Pu(>%0A0H++C+*mi$3)tg@oO|QGo13KM!wLD z?K*sa@7rlpB#&xg0m`EqQPq{AjqQ2*o&-Y?U-G-ofpi%|$}ArL*r`F)g`>tqbk{TQ zD3tF8bUt8Le0d)u?3|_v_!%R_Yz`cQ(}$?fccous8A!v5+^!QQFt%A!ZJy1XR|s;1zU;8n>Qmhr zp+s8cTt{f*#ntqQ*^jd3FIx3uX#1!7*D#;uAeirFkD(+M*UgfgsPWp7PQ%O@s$qHJ zWC4Zq{*wvAVOtpJ==;8cYb3iykJ4B5!a=tmfWXJwcEeqRC~-ApMjlk6==pFR&&+q2 zZiAJh1ffBlYI6qv5-LvQ^=bJr;O{7x3F^A&hS%ti$MpJUdoqGnc27F9CWgCUzhX{j zb!2;Q)^$>Ug~7DeY(78bok+;U`wnT?RxqQS1B9_&t8P#CEg);9{oV4uI&sKns3*r= zR@+U)O%ZvF)LlD8RPg5&k;R>bm5(ETul`gw;EGED7+SdjQ zU>S|F>~l&TVw6$_)TB*b|OUe*9QY~DC`g? zHRi&S?U(LK^GWH(YP{}%srV}J9!%!xkn%^iq}zBQLHkk)i>nOD6D^3ZBT(Q?1y!Qk zCNl)$Q=M4|hlULav0?L59WtchhKNp_NVnm@$`b^=DNsx{@2={70v#99hfN$)J z@leES$KT&6!j3vYXVYtFMF8IXyb0v4o>UP#^=R?r-at?{-7M#s^1Zz zA0H0>@tcMnqh}2iC~2FwLMAKrg`|NdM?cb34pAkvAODJy;7 zYT|+`?IB@|H-?sMs&vXH>L&^#*)S#j9Ss6H|KVBOX-K|+ zmAInt;e*e$yy+m`a@gHY+`hW*TvZhbu4`Docz`mM^=V1qx8`cP|#wucP)Rfhru-Fn4;u4ag!BR2n&!Od1hdF*6)%sPnSY*Y?v07Gcw!*}FGdQl%w&Oni>U&qYEYk?^vp|YIkxNS_UA1|X%Bk~G_o2_J6 zwQF|WC^smGMg3A;uL|qpk8G13oksSocT9H--lVUnm-Dg8FAvqDT|TphrZaoj_}}!h zVYC!grO?%n+dy9LmxP`JFOZ)D!+gfRVrRk%y;^J~6S#HiQuGP1MC=XPNTly#!@c`| zwmp)@$1}y@=jTh7v=<-e=5SvwTMTT|Gul^HqQ6A=o=4ZzKXmxU5@&8M+wE)UFCI(S zxS#|X3T;j2zvcfDp6^v=Ze1X5@c|H|d&qbJux_+~o0WL$H+=TA{ps?N@4^6%U`sKS zY^tlHU$bqxD>Q0K{ImJz{lzA(pqy;QE`ixuQLjUaywKkUvBXwc8~s)frv0?R;?9K^ z!as9P_^TB|I1BML8R`$TwMP_PMnQRIlY+94%#ZZMfkd z!fnqtSyfJ}z0Dp@9x6Uvk2ZG9;6c@vOh^fwX2<<-_if2-!U3N?afqQtdocEA(&#N6 zQXfO#%Lt)$p2+bErP6DXk)7@XIWbkts%q{qc*bbw=1!|NU#boJxN}-3gLE*@I3R}) zf86S)2EuG>Ax>LL8CQbqG;btx{d~& zO@*f2OH~_`Ky|~CZ3;yG36i(HPmp02^zMzlyS&Ogf`_+JsR09^BXGU9`D&xr8btt0 zh`shg_pan?4nlMWsKrf`$#`D^nt^tI+tKdX+zI{ka%bdN$e*)xv=bIFGS4t+*hpTIk<%^X-Jy) z>v7aj>7Gv8v=4Fj<}za3FPk6gDoQD;@$){MsG?x9r<#xFZ%vno*ANb)O z8r$e(1-V;MJH`yL%X(e+=2+a~N>hluum10s(U}NNR9-%I$qXGA?4jI|j1ZI0fxO4+ zAB3lV^@YXuiwfkWFm>CyoOS`A0<=6ju&O2G2}67eAaQQn__LC8S99Pl=T2NGLZM*u zaj5XsHSzOF0nc$|j|EUIkfW0XKQvDXW5gJYNGv1zNCnV$_OaiSWgRZ?w>1!^PtD8s zN|oaB?>XvhTPLI?wq`)V>Wp7nt87fWz3SPOayy)hYEfKCXE$IL&uW#id*FBf=nCo{ zL2d%md;jXvza3$)jaJJ(*nanmW~-mf&C6QuX5=H8ha3Fx&L(N4it=Q_k(Qv(U!WJC zaQY>_??Qp~`0+ObY%*~wcE65!lC94e!FVBYfWdrY^wsE2!)3&L-VM64sCiO_g#dX! zWLlSL>n20j%H+IZd)Cv0tOw)YUrzm|%J)9-))-A@#*9TC+1?ZGNz{1bU#f+|ml3yI z&4^6%>b+$@HCAhBK6^f*9`r=8V|6!kr-~m#79&LJz@$9wCcl{c27WNv2?%e8=W`Qi z+>o`LpJ@F)f8HKbVSq|54IkF8`ELKx<8(V63zqaT7=3W;se?3lrQsuw5$ zeXMRfX~)Jve01^>5)iTTg9vw6SE-{)9~nk~9qT}Y%;JlD(rvAglVkO}yovBCs447( ztAw|m36~q3di+b z{lMs(RwFsr1YsL&IPi$@Ee0CzNK@Jg%K^pS7V#^Iv*K|`iKi@c35JKcUs7Om+sxSuz zHrC&nPrF}5$0GPB8vMCj4Bg)~W11+XFS=jyEJ({>6aZSjBPXxWAMycI&JtDH0E#63)~5?%#1Y^z7e~MX}3Eao(+QzeRDOvUt(%EZmjJWFu`VH*7-Aw6gu(l z+c`yh1N^lAamLsRLZGzvaT*El4<>^2Ed=zO1C>lR*@MsM1+oJdC32+_oXV`?cqUx z=1MgcsM&2q!tH~}+ubEZty0c5B{fl15yz*fpLW$EM^;sRHt!L$lY@xnyZ2BWBncI- zcXfsVNCFo**KT9dTW^H+)IuL)mVcLGK`sr233=eiq65^+u>gC<AGdc~Z08bu5gqPxztX(-5?)1Co)?@t4ae)VXHrRm^ zN1o{QX%~+|Vs829l?HDJ&cC8d`U{zB!dg&?1%`T~BVK#g)VnBAX;=Ii<@%nFHNgI> z0%?@>>Poiel22OY=Y+y`Qp|-x6+HSA(JJA|y!Sn~QJ1SHv6SME>!9^i@Y?znb=&O9 z-@gn2*wo}ddqG-)tY~|4o}!m0?FeomHI(eM;rSbeMG{R7+U!p!7}>*RA(%OYCJVQj z2FjHWVw<8IB@I00Z2>Ol4vyY2_{yvtFi)eE&GW!UR@+->=lCg@MvJIzJW+R1hv$L! zB$5sHsJ*W%VKg#M&&?4}O@sBN#t2-kMP$$}b1~$5br^@SSjyudO7Dqbxg+$Wbuo_X ztNTwCaEbg?`yjgj?b~3yr^(+ycpsVfcm1WsW8+oM=hK%y)mpk!40gh%3-0sZeBO8b{mp!p#(|dwG>)-%WT*tkl2PXb(_0SY=f}}j3 z?1Y}luSVSluU>O;Zg856#K<1ZyL)*4g>JQyC|k5QT=tcDxM#qg5+T-tfs8*9WP2HO zW+<~ZZM7E&ugz~L61-pU+=IFb5hLJaMBmFAcvy35wq@&FQL<$S4X$M7@n=(OQrt_6 zO>)J_PE&5U^A-+Y$wXUQU=?17LMc-gwbo1(b4iD`>9K|*_pbOLWg@#Vd3^?=hUf6a zYUs<4{7D7EwdEr_{h87j?M-%TsF(`Qd=jcuSt&gD-zXYmGH0?mUMVg2RJvWck7cye zCgt4q`w4u_HciSj>BWPDugLuqWvxU;8qZjVyLXr`NMRC9v-%2FXFrNJQDtuj-P=p6 zEh`$lRp5$akF?Y7ZkUesIP*-CiZ8A&rMB-HdZy#{2}QQWP+PMz*isCUz?UQAou93C z@o-n6x9cef93EWHE3LVp*A!=tlN3t_VP}+Uy-gB<4DBJpDl3z-AG>}GC0L{u+Px`$ zhKT~+(>h@1L(GKm^35D%_$n5V30>lIwbYp)BSQhnqi={#z7c&K09U;T?G z()*~n@=RVS1;k@mHn#E#jX^cBj-$V<>ke!K|;sLPsE$+}zwt6p%1)MjIedM)B_aT4RN~ zI6BJeb_Spp#6x6>5+C$9C6+ALvO(s*%lX6?sNM}Go(cyU6gw=Oqv4OjqgngK#BF6< z1O;`_1di= z@zy>hKeh&ONDf<%vh>WQJ%SJKPe)dz$-unEfnKd!v9=;QwZDGC^No;0>d^7=2Pdb% z7G#68h{dup)nsoi9RUxkZFgwgUZ=hq1Q^2eRG2pEiej=a8M=Q2{socLWwl3N-F?H8 zhO>iDiSs~Hb;}U)yD`vFt&2yOoD53I7T6vU&1rYk7jf$~ST0{pt~-N^BzM^zjduMorrqjcdPrZ z4bE1lZAnB9H2>z>{N`jwgv6;iD-jM!csn#4mD788b)1)CPxf8O-IrA~f?eBDwR@iY zovQRk8FI7!uQp+&2w9@^^Pmg4*gfU=^Il86NoSRR= z<0}?FR=GR6{6~W0-b|y|H-e)vf^_MlbBV#=Jh#^C=(f26*RuJ~#xw9y%2Y_G*?Fa1 zddb1o?6^R3%Ut=~P9Q-|O;DU`my73Kg`)4(q^GNUiRVr2*3n}AvHa-P3pw!$;RGjl zf6Vtrocf=7Mlz`0SV~{`mYSSpHLYe?*s?Z7+)*(%fN=DSfj<%hWmBTkPf}>?3Aboo zjI*9Gpf%T&qTGjeny}yI(3460yx2-d4n5fNlCioocdqQDO8FFU^XZCL0YJDnk|Khh z#ngnY9W=iHwf4y}=Vjs>Tt^dUkQL4Cdj|e#MRf3;#%`?mWV(|yzFe4|=N{h%3X!LZ1EmY2$+sb}+px{;tQz7g{tHZ%&R3r3kcls89q* z5PL*#MOt#yis#a0#u!X4ZQ|L;SxR5jou|YUn52T)5(#igvA16GF$uzTA5vM6?7}De{i=_9n6d zk>XPWP#iP3#TE6_x%UiF-KPZ_ZqS_q{ZPzt!`(^lOw zIBxwEv1WDkjozSK?co*#x-4_m<+P#9f{=bk(^C$(`~plXnJ7Q&I`In^+G#1n!asH1 zvVI|jtkY|CiXokJTRxo)kUUr z;a3NxgpI$uZZk(TrqBAG6{01q$NKJC5WEbmW<9inhh6aYi8Y|3mNi&rfbbOvj_r=- z=)?)inSqG(7^Gq8kq`uKo3#XJVXKy@F|2*F0BOo8?Td*~n|XEw-Lu&4OxjHnuXH z>*_`VZ6!>uA8#I=7sq}kc5lQ%?+KIh96gSYkn>^?eu?;OH6Wt%WuMzy-&VgC$6CJ!4y}CfLa0-R=`Vz+UW|KYp@&St{H|lUgm!qOTDOw3U)Tnfc3+G{Qx`YI zMZdy`b*yHsGE<42L5N4{+r^i`;J;$H_$MKFX^X;+AJ0~L@Y&yP7*W3|@H{=J_CU-( z(UmvEACs7>K&&l&!u$cQ4c2wh84=(iXoj)Yc5dQ-(D6Gnu7*TpT+45zp z%qtm`B|L{D9fm9+EsbbGHkFaODWBd_NeF?Q9dWZ>f^Bw7ypR8-YLPLN=!BpkyM5n!odi< zw;Ro*27|}%s0sO76Am_XJG<&4ofKj5Bnzce>Gdk{?S4hT(r=DNLUBB z$icsvSQL9y#zZfRIAkw=8%Ljz+no$bWaw1Dmu=~t5pjDU(>2;H5U;lE>*Xje`dtb_ zrA8vGd#UK_z2vUQ6UMtibN%Kp89hhd1EkCQGVhvKAi{;l7{H)Lh;8XNQ?}@%h%@jU z=Zsvv(xYUX#A2z?7Tw+hr%D=eL&6&6hv?lV)P<8nHzUCLJ;=cEQ?3eBuxf*YL^v@- zl6>O=QS4_smBhW_z}4iqu5imb>Bp`)7r%ppI^kF4PB$7DsYvf1Ed*47cgTxvebF3r z|HD{lpm|Q5vY^=U%=RJMDmQEC2LL)BPk;`Rw!++(Q`wAFaSU9Bg3ZIf3B?!*E zTFz+O7vepQq028LZ6jqs`~ZG6RY1OL=g>7p&yAGbr&m z#I(GP1}FU@+oHP6?{fIVo0=2NS0r-)1ic4vD8tI~LtKp! zrC@X>GXA#gg@%Y6AtL49$}s6t^?yF;|F9B-*j$lsh+>FVhAHxQB&kML_DyV3zR3QD zzQQw!m5T*l0IqyY&9HwmK;C@lpu|4;}H3z8I6LO}nYft_+$FD&R-TwfBtR%Kf^$2cf3SG|mlj@XdM z`5w+Q{gmUvKVR(Sh(Th_N)<1@*!7Aw>#>GSoa=kGYxm!~nZnf!zYcER`iuF;(7`Q? zN)Px&@m5=&J%Ko;Y`i3`eV&d)EQsox&U)0(YAnwNRdm?LE4&m{9WV9PUMFpumMv`~ zQ`+lB-h)g>!zLO%xw_|1QEUWxh1o><%zeyG^((W9wVdag6QAbvyoek__iFcaI$T*` zkE?#=FM$Mw?dzyvlc7^l{12(er3P6BAeu6Cx-G+~4IAE)j$ROA+U1fTdz~Ocfw}KN zOm`IAKY9f&#tI9IwL+`&-UVnrGSfpP)dyB5_rJr-SXsl}7bM5+RMAh-odLj<8JeGq z?EyHvfqxcgkPpA5&ywF=K_IW=1M|-g{LRrBXU;T~5PtK;7^_FP2BSUv(n94zKpw&gR$?tYS&XYXZHZ_AHMc%O+;+9s= zij$v;eE#-Gh;4tpr=wuSit!l<%Gb37aF{2{6B~iu^x9?CT9smPnCH7Ga||R(RrV~@ zD7~R26%nMyAWTbbHL^3Pk3@z;YCfgYr^1wY&DnNNwN@dU&+SvyfkNe+HS|e}m$?Fn z5$8s5$;YBnBV7ff(x-xTS84UX^0@DJ|8w>R6q0gP&IcmQ_luEAKW$XdWwf-tQth7a z<4I%TE0nhOo|+GW1cI)~cf&}*3<40B<0+?v(0vEwuRS}!NQ26}6=4t=`oSrE(+9`% zHt&}1PAG;t0&|F3EOSQR^XgATs=bRBsMH4fpa0Hcyr{;ecK50KHk?y7<*65!pQI?_ zx}P~0nZQi=93z9^D1AQXe39kfOxmqu{B6D)Gc4Ira!)C8R*^Y^alg*R&QFlt={x=8 z{+U^X{ucjFTX_%2@4B7V&zoyfMrd_)bf$~IaMOuc`7B&y=r6a8_Tbptk1|_gjlrSz zbh!q#KgX|z9xkq6q9@-dR;&j>k3wBNs+B(ev^0piqZSP-t*)d?wLzT>|GOWikFWXR zA4Ar#k`a6gS{>^G`Y*J2$Jt2yYw=bn9%c3{hX-P=W*x0svC)|8nlgE z9IyG;CahZ^$=;G#+^r2FIO4~VCePd3e*HkPhyHXodc=3X(xLkyCmS7k^*to_av`@# z5QmHcIUqqV+3u(aLg=~o`@)^(fB1$lff9m$x5EwA`ZF+)O8mb!5X?+9UJk6H-R9_8 zfT>8z%;38tZ`e{6fh}^kJAi}sXx=iyB@K-B&~nn{2Cv8OSmbC@)Ft*wcaTAXhW}_v z|6Koa{;Qnhf{z3L4^K{lon-@ZcDYj*P{6@N>{^hRy1Z zjisOYn1{=a)?~Tt+kd5s@&z6$pN6iYvpHi-lQPqbg-6Jt5WeG&t0k}DI1l$Jbi}!6 zTL*P?84njAmNPS*|Bi*youx7~*4)?8Mc5!dbDbXi**BK)qG{Ha-En{&fwW3~9z+1Y z%Zw!kK`S4ihjBOS%{26Sn+rXLaB`;Zl{x4){xCdnprQ$G=ph$l7 zkHqP?!AzChU|AkN(%-b(;TzFUF3{g4(qUIJ?jdc$EmCC!v!23|qB`rle`g~QbedCG zG6$+yiT-1A{nxa!?p*12#6WyX#-uldd9TAc)xD}xqs``sUzY2n39(by8q@t%IHHY> zWqqgBCg|vE+QCsN2V z=iYAge1c`MnF1KO9z7`hAvkNVPx{P$SJb>lTDTmhs6nHaX9Rd_q(b!8nKTa11mC`x z&UHA~-LJOjT%YRUyt7!@r{DSQ-6k#*gGRv@L@sDK?TFUDEG+;^FY^eNtOR(!KZ>K@ zYf1h|#)Z%B-T7dSY;kNfyzDGL^BY?H>nhgD!GTyb2+^xoQULVh^pAvFN37NNd(9a$ zB}AzP&Y6^`H6)er7mn26ObabwN4)eI^Y#heJDWiBjOkPeZqj&LEYSvEcw~a|2Qr=B z;-57w9$$E+y5XQ#YJw1*eDf3m|GP}C=(IkreDrVFID zU*|^snuLDu+)Y>b9|kE1TzPr8xb!bhs5k4KOON4cXIj&Zx+b+52oR7Cfw>l(B8*0-1wkk!KuI%3z_Td{wY+CB{$X+ zAI0}`dtvX-tEB#$Bim?lO@4xIwrIe|zT;j3-sBbe6> zo?ti`{Cfi|T`6X^AS#|WaPfhz_pvp#w}l7cOzSn&Tp-X zU&W}aO14$e#`txwKgv9QK|PGf^uUUZUR~qn*9OG`2HU}!?0rH)Azj~RSDAFl*_`t( zyb(m&vwn5>{b;K2Rxfoo8=p0=nP1_Ioj^bfAHw*YHE>iMD#Pno3_(^zCN{^BoM_p58Af0S~xYI zD)%X9>#iE$`WAV3INS&z+mWx^c(Uv&F@HJObQ)5C%Bj&dWo7|+Lcba83mr(`o~-tD z8Q*2YG%{U-gYw4IW*{{?AkddxWFjc7YXD*G-7u;fT96^bH1VQvG7u5YXBh#CL zRUb(^fo~rCF_Vj3q>e!AtjaDYZ?d{|Co;@zABtLI}497NxRhal) zda{qk_x%?}-g)|q6fMet#Vts#Wc}kGx6J1AxE;Fb;5lCRl}V_cG9%Aift(6@9Y$?d z9vJw=*-}U3((xk6#U~~MhdyO1+>?2XsOoohUVW7EDPUFeWn%^DxQrTH^{!YN$k-iH zUvs)I);sqjxJR6Nl{51ht3Hv!hwG(hP2N&XKM%nJvTwX))dqJlmP`F#o{t@2>exdd zap`__v>(H|qtlMlK*@W@{{z`P>XKyP_lgA9h)_##IXJeo8~E&5g#U_ z(n`lKeb>j}MFD5|Zp5O-paur45F4{Ot#*dp9!py$x?s7wUGCct|RJ&0IrRAY=Pc`6AT~FMJ`8;BIM5V3JbT(K=m)&rEIS^XU{ z@9bPm*1W+>GvSS0_|^;rSr*SmVTS*FOR#KhGTV<0qEj4wCJF1Kqw&Znf2C4yyMtMe zk>y(Y{_drOYp5ktt=r_;Zh9dF)xSwEH8C}> zc*t_wJ*m~IsH>a#1@y9p3HD0@R~gD$l|TE9iH-HlCyu-TM_A|UxY};`3RqucuzyX` zU7YJy6IvO83p$LXWb{62b4?3msr`@yef4WKwuS(pV_ovjNLc$Zv{%?=MIq6dSM;Wu zvJZBP+Q;oSl3$J`uf4nEpC70Q_(QwV=Uon$Rdnkvu<;H02uMkXn~ZUXr4>$hqCVw} z=1YFXZcgL7nkz>5lr1>07$vZ8=k)o{+g#s9x|EBthvy^1>?RD;>mMOD)g)9+Itr!{ z6pzSZ*zebH`S|c2M{zD^*j#B;s@3@P00-di_(Oup{RI?_wAgr83=)57+zl-WwUgH; zTv1_c&0V8J*0E*n_0Kjn;=bR$dyem62l;Ip@@7Vc>O`}}=n8kZ)b zS%kdpMKON&jUgIl=qt)Y?v!HmyG=c{H*IrGB_U)STl;<|6w4}m;EA`hb*{pz1<~ZM zP02Arac(`L*>!!Pe_u1jS&J{(wl9mWKIKl&F!V;)4|Q2g#H#-@(lB%u9zXR1aZ_Gv zP#)#REQ{dX+2^Nm;3vcCWziv{DXFmge9@K+&E?pnt?4Anmp5gizS&H3f6XtM&8n2^ zuV;3ipY1s<4WItX2BrXqrl$(>A$hrlab7}sEKjM%u}1yncYV<}!fVA|!lou|E1}2kyg6n4B~Z!eW3`cxLZ<17_BPnoy?RMC zsz27n2CjRVWx!D`iZ7U>goMJkS{9V=Co&y+FU`Zpdc}BsGrD=v%u?#}O`^Se1sg=Bon=3k)Q%jl!Tiy^hC9e$PwSh=7 z(c5E1)@^xLre*o3Yg3a_p_H<2f~WAhi*ohZO@KU>@V*+2c6;aSGf%y zH;8$QmaqkF*w?^!Zh{I4&?y|@@EE$KEyfE5=^L@`x|14__wUh3j}#UY$oZ|W@n&|@ z0a&_S*GxepZWPQ`Z#Qn!_=7GoXEVDpJQ$!B)%;#15uP83yBGUk)l0tc`zQ4NB%(;K z14Z`;>?nvnxpDd9j$IJkvWFUSMv6H<13TsF>{Y^_^l<>sYG`Dr-&(o)3)%YO zkq8cK0Vc9&*q-O?N2xSZK7(01(_~< zwHG&o#Zej~+Br#}CUt9Aa?9et2CL;)@%o+LxA#lmVsb<s#+0hMR8Y7ds*As?htsc$Bu&!C5Y8 z*>At8QLgH!Y{yz01XP3FLGb7YRua#V9!VgQ!P9V9WYVfH(C#+{bY>RFV6ZS&)S%6p z?E)eypNb}LTdS%YmMJnf%5HfbaEV(M{?uko#2ojOGC>b#v zf!wO3R)24yU&|~5t<O`X{R%Kn$`Sd^=fdLS zrPVRr%NNQ??XrEcTkM%!zc4?LYwZk~KND+5ptZJz~R?_100 z$CWs8-vuw}VQ49PoNk877)o#CYBC!(hs>DBylUFGyVFmk=zTt9+BVP13e~b!7W~DO zut%t;&igs5x0;k8dAHbo*9NJ`#^n%YJu)i^)o@X>ZAbTATD6_T)~C4lOYp=@U%k(J zmY3h9utp%*r`2x{J&JvQHiK-(ggIXh_9BjxZb;n57CERO9wWcssG!VPS&q$qzosz03Q-%YEof0=^?SXOq%Poe=Dlp=wo#(L*r+14 zigjo~!?pE4i}I)@Qf%}a0*tz zpgd4NeLCQ!Inu8Q>Bb8&8+s;JU-f)2V{F;J7R7v9tvh<@=?1b{f|U~YOilci{2lpd z&vj-xF^!j)kf1EDEKu}Nwry3IRH2^VJQ#hF1Pc>;AA}#SaNWQ9B4>GfpbDUDepP+h#?$CVCSN z+8hAmvqPI6I-hM^q8|0U8=K&lO^Qc+yPub+uQfE~!bsu#T@PfL{p{b9R|!dZ z`ST&gW&LO|eVKmtTb((Cmn~T)=sX*?`v~aFMgP+$C%|imx-4eYKWGk@^LGKBOwvAX zVyp(EUs5})Yi*1>^)J~qnE3k^{fjTca)ah|4)#z~4RD0GX57qyystSc3wlNqu|?NF zVM67;S856ZJ+4fIm?JxuRZ>%}} zXK4-BFD*V#p*s$pU&GA*5YS?3HJ#kGY6<#7n|^5eH?Hq}LNtllzh$<*n0EYNwtI_E z%3L(vWXds*vLjNM+^aD_MI?K$@kgUw?!p0*8r{70)oR>V+ONm#@$2((@)&%|-2N;t zOS>#gZ1q1_cgB{1>rp-KIoo8HD{MseoqMi8e?tewiu=E$perFTwWo)}SH8rH7ukU8 zs(k&^qtZ}WoV(sPyWOI&@17-}?iRM2UM?o)`=eitFI+??zXgXM2KpD#(%HaA7-trpmH*jvm9^c|rzLxt zKG6}5x9&>{D0emfg6xlHc{2++2Hi)q5ItK-mo zes)vvbR%EB%9!r}b7i;;|7mwstYrji7vsg@(FzNW~MR?D? zm1&}fLZZTh;Z#~P`$xucWmE0b!%aG>*mmx=aA2(%ituQ&KDyfLDaMqd$Mh;qF^!^j_XU%q}{jblf_{GvsDq zcb!UVuc(eT=oE-N?!;l-cBMCZJ=$-l$m5OTf(#~^-iLsHp4Bb2o9a$~dbpp%70k4a z!%QAk!i50}0K-SQ`EPMZeIin!G7tVb+R|kL2gRk_rEXpghyN^qv$F<^)f$V6Z-hdw zHhuML^F&Q;5C0?j-5dIQ%9<6!PT^2{mgy)JeLS$3eIVO9_Gulq3zKmp^8zLxzKe`5 zc30VJVjRjC6qInBY?vIx1xG4I-R7UC4KVvzzmI~f_WSOjPH?CKVvX71n`yEIa(d?y zaVAf6dp7U1oPv6pEh z*Lwp-WPG31)iY}#ywc?@GYgD&WF9-(DGfww(p;_q{D?v{;9izS*;)67m<_@t<^A7tS5zTmbu((Vw~vW`!^*q zM*TL^BMlb&Ir{>C|MFDf!c(rEH_WM{@-Hlj$D~r_B!7IrSd>6Ms%G+a>zKMb$2?Sz z`ejLH>wLV zp@%0DA>1p1Q#EbFskVj`8&7e3*^ViUvWY;3mfvDd{_hJlbg#D-)IgxZ_N{WH66BEX4-O|)~ z)G0`wz$6n!VAE%mbG7`LVpq`}<~{X#`X;x2RneBeQK~8vrc2m#45c{<;~UX9HqKMm z?#gzeMZx7F@mb?-W_a-VQB7q2P0UZG69s`zCXT0zp( zfqPir>*s9pzZL^RO@cM+(D}RXZAT;QLAz>C*z>YMAK#(Vh^tEv3LIn%R$8N~kNfUo zgHBWLr?Pi4hss%%S?sESQ-b|C zbAM9AJ&+sh!w^3B%~mTn-6X+6j)_EIRuW9x`7vTE(?3DIiiK`2zRv$$1!g+_hXIS1 zW(nLIyKR?8wHH4jB}J6A5Am4}gNFT@p-3W2IKKV}vhR1DI)fu0kDA*fcm}p_<%HWY zdsU?kfhJDvD|mJ1zwG{RfSvh@C}x=RzDczJibEgn*?RTB!>u ztm+y-J{ztyWn?nOXe$i-QL<*CL5)zILR(`sayOlYn<+Yz&LKzu&eGgpTK?6Z+B-OB zFJO%RR@vlq)`jFobH7$9D>Yn{9vhtL(cInertd2xv-Duxj~+Xw6Qk%IlTN(=smt%D z*$hQgw>LRtmA|fgwX~`f;!(+A$*1vv9~4jRi5dVHNUL4FtH_~d%HxMQL$&(#;JS1+ z-rs?AtYySGuoULSfe)mH*>d(dlSW()q~kT=WuaN$pRO1Ds_;01vzZQ}k4vsx}r%R-f6#Wf;lc7b?)%&ILln=6xSy5z=h zq`1sYnPG{Ya)fX!Op944Uk>|=s#aiojKkJeC~DK`8{%%>Z2y4^duGxDA$Es(4Wy1S zNPgCVLx4S^b@ayz7@S_LduDRc!gAj;HTV&PLR(%zJ@e|D%rr2XI=b5ywDqyYsmUPBt?aaZs%@CV@x z(q_qYK}Xr=K&~jDy|xR5x^atUA<+Jh{hjdF zX(ugB%um&6^duAa7G{%?t@+%TD;dPMPl>^(WU<0QumoRwYqLCMri>&VQi- zN09=1ROeYCGR>6!>IVcTp&N+p({ED&&j@Wdqx+24Cyrks%<94=$ZvV2(0ob)x&htZCpiQFTRpEnE;Zn% z$n}c!k4&KDIc*pyF4ZVQ-Fd(q?Pvu#OxNv+To0h^4WoBethynzxMGW>&CqNhZzVTjaAQ5s6_bu}4RUxD4GmPbZ#g-S{Fm zbCe}7D>~RLC{O&BG?egZRMMvVguws-QX@sa3=(};BHB9cemi~ z?h+t4!QI{6-8W8fcXxO0;;pK;>U`&Xr`_A`4}Q?vhP_zqHRl{-JiYh#SS)^pz@uSU zPJ+vPf?YN} zcJyl&50t0`vA~_Wi5{L&-nfSL2kWC}^c2}LswXl4PWgiRSDFowE_1NN;v?%vmev>w zk=QtDSy0BJRQ_k_)PafRaoEO*yBVQt8L-bD`D4E9XY<3)q4FWlBrpGz+)>Jrv zxKP}_;E>I9RLZ8FsVjJCYP@t^Oj_XG4EQF@%)YQ{oOANfNem#kY5>xMLwt?7Vw`D3 z=3yg$csPOWoAs`#&o^seh~<1UvB=OjO+@oEYrLhYN9lSV+4gi*21VZGteGU*0_I#B z2YLipzzGeyJ?Tif6{DAgRrcT&M$vet{3GhUHO=#nt^LeONREBV;nG!@mNxd?8%wGM zLtoK^sBrgC+{Q{{A;jL(N!D{62l_7#sR@%u4ho8xXi(#<7-+fohXXyg<692nM-SKa z(yNoe+g`Ad@Hl)-rfXzm&+GIqFf@4}_yjN%BNlpgx9sR(dajwPuYN$sCe~Q*HR=>0$8ImAsahTv{!F zW+Ovu-9(ewdgmMW^2%6+P}#~!WrToIgm9?(vVBswpZ;WN?Z`vjwkjNu3-i+Ua`*VK zsi)j^PAjhJwzo1bemt5h>G;Qv9uV|JwTB*MrO?7!h)CbuimTN2KiQC-AiK)7KNkpY z>ir9YlF2{2B6cK@$U0t#WXatJ3Xko-3XgvQIY%%-aoN<$756n5Y>h8dtFuUqUb^?~ zNZoA9c`eLzhFV)4ak*J1-}cMO+uH`BJRaG9m!;6x2mvrpK&YG0KM|hhCbPZC2`PZm zJoyN;N>FUp9;Zt*D^qHzOiXLWu>Gr?hyA)Zsmb0mL4s*q z6cq~~N%9(84xPP!v^gF-v&46Y(+KA%X4A_wE8|k#HU=$YY1y zEb9!}XNq)-c;(^j6uY%?kJf3fHw<56FRjE`fy7gT?KW0y?85iJ%Ok$V)aoLL5 zYnTZF#|xg4B@N@*GW>@sTVN=Eqf1=ClPeNV$Ah40NZoCK4E7yzLEe_wr-sp6U18UF z@58$I<5PBy3ZmZ56!F|Hsg3H&@LHvvGxr&pF#XoMb%_*ozj{yz7>V$?^(#eJ(}L}_ z2K07)hM7JhPwC`b6AQM_P2B&d_lnLUD=J1GzvMTcL7{_E1UhRDb! z=NiE?q;wdTNW0WcK}jm?5x!LW8SDd;$$-dc26#s%OgFid&e0k%BGT#4ba56F6DSRs z4)FK_;k!TGl|46v{Ov0qSg1OZARh}n+7jpvI%l+qF#O(b2*90WEAaT*oDsel%qoEU zh5h5B;5hvh0`((BUs$lNh%r&oB$EbRE0k7u0*3?Iw{DU>Oag&OvUvH#Sw!J_trrq# zWF7tp{-Yu$(P+Nr&YNr4zGbQiJLQ_|iw+NpY!a*APkNs?g42l|7WuqA1=?hy#2kMu zl}vR-pE1Zg>+i|@$ojO8dnEj5wmU%AN}Apb7mA3|CB94`e9L4IZ|23>bc{j~*53D^( zjX|Pu;WyR@rtrakoiO`G31Bd#gD>d-E@Lux_nijMGlz#WLhLr_TQ(s9QFxh_{VA@v zlp*{)=0^alt+lf9iEc(Ay;q%5PzG2ETJ|51u?uolbZH7?i8*!0{OOqyGD`Nbm*kFb z;dTvU_9zj(6C)u#9?9d1GhThnfzmpJ=3MMHCwFuJr**itCZcEn#WL-bH>zIqcUZ_e za#bHs6PMPkT|)xAN`b7ps-DHvWocL~i>X<6K`2#8=@KkT71{c(1I?lhPN|+1Svcam zj?TU&OF_RbyCc3delcu*_Us?78G|c58uty+j&SOOYbAlx-RtY{MY>t#ws2TBks=vnf9Iom`H?9aHbt#KawmBaNR+jdfYG& zT8ifm_aPr9F;cc(S{wF2X7JUAqwqcYI+##B^ho>&;^If{cQn`8&cw5vdw73fpLUJQ zTJf3$0gs$C&-ZcW+KfJ*M~(=aQnEYiM6nn1Qw{uA0xY z(=DBLz1k0|K{!;mA!|%0@MNU{ZQELDKFqT9SQ`Wi`J!_N*~KDP<#vS?&>UO+`bgSx z+pBy;<6s!1yU5W!Dv7JoxR0jnO>MNT*K7j8C7*LWts9+)z5actT>E^bs(dc#u;+{+ z8ZP!Sf7iOt!O#Enb>!_$hq5`A##QueBp1dPt^thRO}k`$_XAr%M50K=&xzf9jc>;b zev!ct?v0*YMvM0hbV4VyYzE~ml13DA?kJsc<2KRQCy_%l)8fvj$*KCB-x=&Q8kUf1 zXroY&^4)rMd9_~{PCZ(~a~uUE7`H68K-=I~CoeUEBqsk}bdUcd+{I!z{H&7x?4bJX z%Ps-i+V7c*H8RJzW-f(#w=U8)*SqSf^)se#ss)ptg(;Irur$9ZnoGd(sz>*M$NV0c zhbNUXRp+!Mvbhq5ypjt4wu84R5JycxO_Na~;%V3JX7!aB{WCt7ge=nDC5jzUHaJZB zJIcG=bzcbdErUTx$a2FdZM`tRC3>sbG%$9BJli!9FxR3%F6W@?SY&_y2QapAbaa`# zQXy%mUFp7P<%E&SDH{8F1-Y|hq1+mtpXdsrsx#Tc?Rz5JqOp$l#|N~uI@x_VW}L2% zYdQ!`AeL zA&QdBwlGC~#)GfiYvwlAmgXLMHI{4wsN+`rZ|VvjpQv~uy9ap+W6;xf^@SKXOL8~W z<;kp~Yr?LLA13pop%U}!?FuiT71E3z_wf``!C@ggkZVa*7F{;B*S(AKyt@}?D$K{Q z4Wz7d6Bw!8IB~$+{yqkEX|4mqtCz=J@gu2G<+rWh*seMH^8R+rTB0 zWyY6iA?y8^}QTO4aUE=K(URpX!oQyiq zHLd{S6!0@icF&$@Op=a@zvjnx8&i^tP&pY6#E@gL{XmOO#}cq^HkYFl8AN8Ckf(^;J`4(}=?yxU4X)~DKK>+0;onF5oDt3SS1&4!g#%x1zX`_nwdErZKr+j_KQATFd2u65lvB8o!2yH#!}w|ZsKs4v9^wu z!EQJkMMFsA;tAa_O{R1Azn&xiMF*YB-4yA$Y~T>>JH9hYex&=lGZtHYDtc?Lb0wz* zWpa=H*X_5Uqiy#Ucgbq`D_hl0(PVqZSNZpxJ12~JE1V@WqiIN0{oGEBqN1Oe<+ITm zPETZzOjCi+C`rPHba!j*OOxDaDXjc$>oe5<+O?+N#e|3YLy(D~z z3)HxxrVyxC!FCALfYt6J-Bx6Vq^@WOF;zE!Bx1H-v{mM4C9gPwM8P*A=Ca7E+UC=$ z+PkG0tzNO97D@Pu|EhMwa9Vm7;Lr~*bC05HuK&DO@dC~4I5$*ufD{Mk{Nd$H4{;w} zN8s*k{d*Bz2 zgWjIkCSR2Hc~Y39-0S11o9k*QdvuGlF7sSu9ySp!4W|>ndAo4xp9Uhwze+2MImTvK zfVHhU_vtY3IHlxdAaZa0j_Pt!B|9%EI1Y~h5OS$g8A;>u9FR%2CeNkrh|XnKV{wi5 zgwRlh?H^wr2(~9^G2ift92Uwn&+_qx6Qr>>t!9$lQX%0;>N$15B(Rg+)^rt{pl#<+ z8zHpkMovhC;D#-AiY43Ov4;VZn;zN3@f~ispkJgJ6Gn9~9r%s?%i6)Vlawv4Ud(t+ zfaKy|&)yOQe7P*N4j7tpMmMtdx7l*f3I8UFIIF(gOH~Dx`!CKx^=Na|wj@Tyg0RMpL>Zh_MV6tB(l~{+kP6VGJrjEk<~SawP`*IK(_v=RU*{=La(?kIB4%{j`M2af3jV z6HQCS@#Wz3&->Um-yxI9^Hw#>pQ$0l1?nC1B@c&-f>3LM3?`UcWsz3G>n@WZ+Pz60 z_`>B9C@oF!y#;}vH7?884EHu~Wd%S9>{*TIY zIUK{Onry9i3(0_%fO!Q8K4J$^f1ykDJB3m56j;P4@12Xi4B3E!b41$E(CZ_--79yr zs#jWfNI9*}jB(+|b65AT*dNfTBB+;@n!cC|J#R8O0YZpTte;QfnMfS(8JNO53Bs-u zt&^X~zTDV|K$WmN`R;9%&xIlL$JzPgc;iQ>J%{%se+esQ z?1gDY;6vP^vHfE07RAg?eQ2$PPi(rDL0x@Hmu!H1O}A7@PcL7c zrg~&U$82!xWknzJa4u1!Kmr4DotKGm3W?r`nKEtHf? zAJv+L^Ls$}91B(J@Na0U@}k;&3H7Xk6qWK=A!Z!dNc6Q43G=x+6)N-Gxdr zI;rbY(=}0*l^3+&jS;^xOQEQV!V+lHHkX=M7XthJN?=>md;7-3zMXUw84aXqHl97~ zsYE(Wa5yA*Vj2qnptSdh@HOiiw$O_Am<{G1yT%2+VNY@WCKectNnPn`+RWuglTT0W zO5j(XQd!5Q@VLns(3;yC>@U8gFq9_zX0DH#7MI}&_d9NX{kzZ)s_lDaO8a9edj)0) zS7F8sCvr=TmF_%07Y&%AYH<_7x%3IYtUX+e(To9p&Y096hod1exX()kGmRgn>Pgr> zj7I7`F@0ZoA-=dl*)xk)MHkzdl1ZbnKA(*Cexdy6>- zNT!-K2j4?q3e1dp#zvdKoGr@E*%DAxB4G1)L}kk0^KcVpU>;9vkyoF$l?ow7+5`u4@cXE!E2LBDhHvj5 zy;k~uUa0dEsIsJ8ow9`zdpw}d*YiOvuXjaR;Ek5APR>xf!xG2BnYYh1F{(E3u(qwt z*+-dP7uj7jIv$h&9d{c1<8_QFv>?OwTSiIY_Y)IL<++Qg<#FR&bEyM{8tc?poamyXle5{Yx(qT2Sx+QZ9}v8?U@SRa*N1P zJ)EY&VVh*V<+8$P6A33fyPTq`faG;*;iV-oPh7cJxHJ4Of*pW2rJ+@u-JAY@Et{PV zerG7ZbX1Csw$pFJXYlThxIUNy9y_~5c}*21$JE-L_AeN9?Ch#?6AGMl{?eYm z*Qm8CA4K%}TN|HBEA1*XvP87TX22XARF~QF`-WIz1Ao#Y1DUS_5Z*FmqXe*zZ94K| znuZ5H(r4~EV8zPj<+#nY6W5C}Osfc6rvb~?8)TCZSl7Fq3g+saFWd%6XX@6$CxP-u z{O>>?eh+FjYBh>fcXfXirKH@L8BsSL(^4QqxJ4yrqjzumX~$sF&NxT|*k8Oydx2?t z)R=YdxHXl3Zt=?=HbG`B2Lthg+0|e zeGpvt_)_BoANbvHr<5{g6$cGWG5A`WlbDD~`8*iATT*4@1$7U2XDr#j7ZMQtV`#YiI;5p8nM z4hg#N`~nqalOu+#)bL)~R$;8dBQ-^)epWuo<1a;$af`=(vFe4#$9BDch`M0^EuYze ztnu#OYaHIo1-sG^;|=*1(dMkKN&dKXTQX;ul8!>6mGJZNSG|e@zl|~@NIRFFn8XIg z%uB4W)yd>RLg$A)#Y#9;0cAe31!3__@9e94ERty~L8=r^ERMy8;+NVmjqE44CVPI5 z>Xf6)_KFublSi<#e~WX9$H8)qJPnT{CK;1>_eK zjlX-K3D0MLKhuxoAPFYysRCL-i1QzD_gOy=kw_Fr%QcQ9q+|p_1Sp64$3L59^$iqd)4Zwet;lQ8TyW1^nv# z8}!BVo>eD5lXos|fg$zA6O#%s%c}wdC;wt3$9CcP5)uZdIC<_QwodMPUXlK>D4A3r zZ@y!JCxDGwp`mYBN6m{6Pceg5aX5_Sy(V8uEC zPLALnaD~mvRlIHAdGjXjbbqOp`X*7?+oI<)rjylf^PJ^vG*5VO+PN1~UL*9ClZ9#i z8u!w{WX=^WQM-YdcV#-U8ClUgrTtH7jqgmWi`{0QgK_u6d7kq>MK$wp3!tXLIUNJz zRO*}-ek_n?jk=%_AMO~6h`t^6)|anI#Ek`#Qr3hJ{IWz2(OBN22_6X5(UmrxXHZPq zhQZYHYv&EOgD*Q+pX2s<7l3^S$xBp_n_4hj>*-97K$0#3^Fu2Ixfu=4*w077!mFfe z2afeVI~V0;2*KbAe~p#6EfrWlOqG0B)|1?gOjyvv#b)u>^+etizbu>`E&O}k5R%__ zgvdS{S)umLy3IJNka>9?yLEQmK{WVHw8R@Ld3=}XiGq~n_DB;>7}X>M$P;5_Q^n>V zr`21YNBQmx2V!;g`qscA8^X4!m8-akVFuPaK(Ujn)b+#;lV!x_LU`FJ)$>>9z{}5- zsrW-C`- zqwzE&wrX!rl2^lJW>u71W7Yw;+4P$rr!$&J4pDWG5?noJ!rnoO?2WpnR&8TKXeEpM zQvZ3zv(cBQR|=7U2^ytHL}E~IdohGKK+Nkg^E-4=gKZxL1$>4y5y{vuZ!M>`YN<>M zsB3(wH?jKBYRvv&MdD$@85O$o8X`U&_+l3qUZCNW|dg<%=qOmI$P8UExGiKbu@IE|#it9H!4@px? zZgp-UI1GwLE3ya$Y5x){75a!JIJSfj%vhn4 z>5E}qovEIWz@(}?w#DMgL&ZVb=7Iwhfg z)YNlna*AcQ15fIbC@|e*tg7EL(0T*Z3Cv^h7JQw5m{##F&Li1M}E;lYh`hf}XMVP};BAYynysGq`cBxUeTRR4Zkb9hd?U!SPhEP3_zA9QmLd05L%L|mRQOl|d!rG44j&H4S1?XeOWXmo=Ga7xXO zQWD;z^TMo=#(SA#o~Z8u^8cU00U=dmaB{3L2F_^RfDA+9&pgLNqE8NNKRL&>7dp!> zw;P~uqr)TWyeBni75VpbzkN2`ifn$QDdgnA3Mhcyn}k!gpu+~GK|p+iV;p@2St6&< z_fPkuPOuNg8HlS?N*;JBdy)P$r)_W@h~2noQ&u2pMb&5LbFp=k*}l=9A6dEF2EgVx zs^|00X@Z)2%$&=@u}`fP@L_#O-rfT`c(s&A%c*{8RJKk}T_&X;d9uNCPrqp)=gt;Q z$d5MqJr0aW?(QWr+q2Q;cGx}1N-1+r@Q5vFIq0*sP&2V7ezwrUr)@M)WN)ft9{>Eb z;`=FJbK{;XolSl=^MbR$37t<1+7b zu412y)qDYzcK7)r$+VHGMq{Optx9!sW=1EaSe&8dlQn_vAdhOB(&m<<#3=SN*W2=0cpfHHUdtlpGy4_KP%^cvpaC-xOGnf+c#5MvyA&yAM|{}`bmg5b|Ho}CYMSQ7nP-xk zlM;_Z*;jAY`spmkD~SB(E;7jEdcWwpz#=uK8Y(k3wCvl*SsNiwxbRf)^Gkcd%AmH+4}eFB)IW(eGewP?xgGd|6 zelH%5Nil1z=IPRnah;i+c(cck7ki1j&frQ%|LS0`87Z6Gn`pH;XsnytNM?xmoGQ*< zVIcer?`A>jdX;mc_8BGsP!QOCn0Fynz@ndxG39Y(Hjc63*F>kqagZRmEb}p&Oi1hd z-wE}?ne2Rj66(VnulpCos!Q?hvE#2}W%v{ll4Kn1{=X5LRnNzqRNO>3gB7P;E2f#* zB zs-*-49fYLUu9S4>T3W(U&B_EI-9+`p%9~h+A@$($G-3yr;Uxdyi>!R+nIN;$D`x}& zsClm3U#R&_;TbYKaYtvBZb*J#3b(jv*hd~1kmhU=INar_H!l~I!Dh8#u4b;xrcb)< za9cIzAn-Ku8*9Eilla+~lN0igBjMP`po{n*jI4RuTYY{2+T3rvcNvo+PGi`_8`&RL zM@SC-wQ1dmI-M}$-P4hPva`=6r4dKEQjpeWfY3`Q({A-RSK#=n^$M_CjqeSzr_<~j z1spA>3@g5$;=`L=3Q3W3BuDgyMuEWQ1Mz6U5~8^ zhPTU9h_X2WkL=zT|A$myi2Pqv_{vMNFn#@3mJ~fervU;5SY8q&ypjEzZlNFF3#7>; z^z91`HDNT3xRqGn{x>STq>t0USe@{RGNz^lzHM&#BDpRi1j$V6s6Q%%w54e$1vtX^ zKMwz=OR&m4S)7S<|HGf&z6pblGG$)RNM>klNhfaELLjd>Jzc z0Jvp-AEod;2FEPtPf4h$#aUm@KK>tl5KJcZ9(gN9v56Mpc#j}-YA2Zjn13VA6LJ3q zai;u|0k1z-Qi$jnPyN#>ull?cJq64<{0At#A2nwtyBUt^Sdg_$Yv}%mH$Z+er+Qya z(?s>>T>eM(E4$7E!0ZoAnXJs|4XT1SzfM&vCDqz0k6{MpS|b%xN&6o_+^tc zZe8wpdqPbh&gJ(fmK8GF5_?x_so5h7t?VCh{wE5$ZQr8!_Ws=+&{kA!NOulVgh6}O z#Mv%4vw~|puib?P4jnI?sjzN5O*k1BpW@44{&WkDHS$Ab zp<>x00IDtv%q_aFdM0z~`W*erxDLXIsObGqfgS^Q`}i|`Y?G_`R7u#cxMF>L@zxo% zzHqn1t0z&RUOR-JpPGES7u%G`lJ#|UMdP&~M8tXh>q?F}*Hivi58vdGlD85Fiy1NP z629B~gUZVo;lMRu>Xf*?huk`Lk*c=;GkD=J1KtDZT(KdrSJ+Ux(KBCoSjsUoVL;oS zjBMj6ld%-PI0PMV&Yu^n*2-IY@Hk^w^>p1VxaXREYz&1YZ<0*ESTt4Q;GU(?8YfDuN)DbN#ObcWEiEbyojXegc|~Um5gN5!LAN-S6vj zTZHq@YG;zVs8x}9pdCHrfyA5cO*x!;B5c}fn`cRgcs@RbkD%(7!&*;PyU&A)Zn_M` z>e-sa)O*ne*cbz*N9$4Xi31Td_XNOa41YFz^4{d&(X64L#Vew<{=cx#IHx=r9RHgu zu>x@BTlXP)+Y*+>{w99?AADpNi|=W$S)_m`j`?uW0r!5kz1_6Bqrk;{yg@yKrAB9x zJ#!{@DqKKW{MbLcG<>oz&5Xs8UG8h4QsZwIh0!!)Q+T!3Hc^WTj{^{14blS(bA;;| zjn+}rsPK>@+Y!!$!V3-?^8+}?2y_^evAQ!d&d{pORmkwQ^m?`zwv)ipUmeMl4B`Uc zt3(1*`$kepB)vJBk)XyFETw^>+gVo5wgOH^WILXZDlMQdbwwF~8J9(NQk%Bsm z@d45qAfBCx{(@I-nSgH*ftvy_gt{RUJ`FxG>>Pn}2W_sN_rF^e_KCJP9X?l%bX!wV2w`;rn2>&pNTH4>5$ii6$thfI^1#^g#9n9Wi)#nZ{%%f- z5|khq;=~%DUZ9i%`u~Kr-onxY-Dcw6eTQ{!*sUtA|;0muh4bMfG(1Z{Tlg-TkzMkFo^brC~3F+l4}! zD^0kI-3Hc)QKm+Pi&l3cJ{vs!_LJN9*>x}d8@-5+n7A1dO>TE#KU^hlxjsN5rrv?e zcgNbDA{CXPFnIW<+t!_Nq5i{566lpUpK(|C$a=RL`qGkY?eQP2m@ePlhMA}?p7nW!Ik^(01i zb(s}W&O@nOqmNntOwtT|!pgyxQ=Hdz`D!?MUT;^?Os`7K7#4D~@xvYCEEdqq{~oTJ zUeDz(Q1L)CH?JL=cZcUZQlC-7`nqW0uEtuXIl8w|r~Xl$1+48Z?W#%gzxUTtwK9q} z1gbPPjsu3YR|d3JJm!QLz6F%k%(>!2;NW48Zwmh15% zD@PN+V0VCebosl1#>(()j)R-PIXWBD*4vtFG`LfJyVpA^+quCDO;^*R(Us2cLVcWt zF(6IJ#C$m^w8oX?=X9C>PiLD$Q*e`Q9Zc>NVFH4YZ+8j&eg*qUn5|@Da$kpz$*;NQbJOi(K;}OR!e*v!@2Eho8cLVD{TA@g_Fk zm*6x4&QZ;QP1oIZ?rURaI)h(1*emijgnd_h$erh6AL6OET+rmVF*>cac>?LTunplY z2Kr`klj(ThDeP|99Z#ule+m<0kNw_N_}TX2f?`UsleU18LWz!S>6`h{k&X3v4*O{p zFfi0~MedmXll~{LUPhHpiyv5P6SIpkmmm9nM<;&0>HQ7_9vm;KuWg-t6ZOPknT`c* zDcM~f1edlVC@#d!_7+=kjT`Ne#Gq?#KmA&MqmfUTmb>+A>@PprtdiNs(w6hto?rC~ zZJq&C%)#FrF`1#0bUoJd&_G$w%|StmMO_u>(7lQU-J6XNNna-;+V(y0i*)^(xh{9I zecrjZu%MTgUB^BHTi!){lx9q|mlGfP_W5BV_@^e$Gp`8fA0{@^n1{evX1sW_33c&L ztX^TSyH`!D01(x>Sz;4$5<`NxQgw3(1`GI5CwbhSq}@Hj5f*1 zotm1>VlBS}zF>GG{|)6efuMHkA^BPGgvG1Np83F3^75d^`?QYe+P*#<6xGb@fZx>v z%XWeM1`c}+2QJD{R#$d63582I5{!l>yt~?+IbS!@uAq|?OivI+v;l@ObKyuO+%erq z>U>{QM}eq%=o@W29e#Vn6NTs)CGR46Cul6SkEU6$xAHdEa=-BgAXoVbkAka2c)h~K z-!N9oDO$5xS&5K}?V||ovlS!pyCWU2llOX9Cr}B|?LS#eX9>QmrTusgU@75J4s#om z4gIbDSkz6%#MJd`xb+_ha@y%&fei_cy(r$%?k`?Ge1k!KQ;*v{21F-nqt{yivI{An zQ#}}t+{0>Q*~NCWT<8Ev<~M+Aa8Rc=OXAKYW(~Y5Q@)lqN+A+5CCY?7f$;<~mEnGN zf3;e9G>6kB7+DYK!wE5Cd#C=jU(&GQ8=ZQ)YM0w8A=xTNaANj$T?eI`DsXH2%Tzbf zmtw0R4pr*N(K&}!%ZH3aiXDk+j5HT@SI>xAKuqj;K|toChWE;HdY$oi+L~7Xp3mDQ zGfbXF84jmk>sp%)xPT#m|01jN5);r5mHh|h3VdR*7*DXbkK869wEfy>i38~!bM^EL zIa~_+2j^l%*X$@6^ z5Q=vK&P8%yaIRCaM#NG5n|*2x+=K!fD}P8?^#wFXyElf8c}mICk-@ezadq{#brb_^ zXz|#nu3-n#&FLKfZ)WZ~)v}buub^0P(B$oJ4OqvzQIOz)@f$L|7L2imxY+&}F#QNn z>JLu}WeUE<$#~o)Gp=G@rs&FGMXDg6jwdQz`CW00e?3%YjfcbE)Z0A;4W|wd~vRw zI-vRJ0;e*4XTJYOGwOBpGb1g}J{d~y!P*j-;t0z{h~+z(sI&+Sb-WZXN$m&wDx%9P zDLTa!?<^*BUS)3(d5|?iMqX%D7kH7?I~Sw*Xf6Kv`#&kPr9uV<2wT>7KftKyU0m)p zGzYmSD!A6Edt+4R|4g7(pr2?TC*8fC?pw|y{}YeD!>_Tkb+G#51jWV9zHzi8C8ZPq z7&m(S9SGILSlBCc+o3Uz7vHl@Mf5|v%#IGx&B%H@-&_D88Ye#t zCXfl0lNZ|A`m_zMUxs3omtU;@HlbI@mH2Q%{5M=piPN8C6von{$}B;}SQaa~y_{eXaTHOE6fk zP~}Irtnw&dt6HkDO3M+g3z{da7eGou8!Xx==N&8jPIH>W{8b75X=W8O2cy8|BP=~z z((C}kmmNlcyLduSxpm-9&F7O^i#@%wwF-}!b3pX+7>E|<@>;WoTf-k2_K`{{9T@_4 z>OQ8b5GF^Ajxf_;x_%&d2@;f}qH}p4$k$i=tUGJ8N{sKhc{wrA#w?~bN?Le=hcjm7 z=n+WJgBXz$Ukz=!pP9;HI)s;foH=nC^nG;o0lEqI+6}(Z-JCL63JlQMd78h|YQh`c zQCizZ_kIimwgf}v>{ZEq=eg+dty{&;PcehlKV;G$e5V%z6WL%r)!KZ}?&oHI{Wz%3 zL}NL}N41QQx93aw0Rt`*7b$TAx&{eKRGnY{jR3JwQ@FBw>Q{NJ3ZB93qO_JYG#$h4 zo_%JlgHhsWW;fOX))%a_=XO~37qawciux;azYIf}E9<9F-wT#|qr2?}4d^8sCKs<` z*e4_}!&80G5cMJbWI)PN%h^80Xm8Fl=yP?CRI0Gd9pT-L9|xLNYt9>K3byh8)5G< zvq{2@{b56_BA$m;N$A-Ps7j;@iW&vPZdh(|(Js0xivh|^^1V|%j1UrM^Bf}`aGooZ za^{+ak_HM*udO&GJH}Rj!U#=L+IGi|rbROF?AyR*$}bDX*873mM^!R;i;Ox4e-I~< zF|m$0U6G@?wkve+QKpn+AHz1vvw1`wETJAPgc;mQ2wmLT4CTdG##3#og(fKf40jlh z4GxE#bX1-yRcM!0+rr{@Ti@b29*bm@03sy5t&!>K-N9DmX#;FPhbKS|)8^YhVvzD| z5?8tsGSygyR)R3NI!eB{WtbR-Poj6y?P6++DwsdIKG{qn`l{N^&{$hK8>A|6I#Ust z%=m3>b_N?}2p$9~>YTw|&A<}6z7~DE*N<*nZzpaG?Q)v5d_~=UoM%E?QK& zd1-i;PPXZ330J%sM&g?{Q%~f6p%GC0L8Z=<&w!1e=L{|lrI?lJ%Ryh{e#A!?3lu_2 z)+a**oq--}k(co2cz*mFtTCaE!5o?z@47V_%s)dpL=A^u(p*c(1s~%o_G?;P@AO(0 zR+FAz?*qCo7))8e1VeRxeCc8z;kYsLgDsyQZTFO`OMznkmT*tbM~AgdU6Z=X^yT45 zx!DEeN$!zP}i-R$w{6;d*@VfTjiG`zZa*Z(q(bM| z(={ExE9O-`kJ+m8;aP(Dzz5eGB;l#X?Z@V+Fl=QfS5f~_hw#G)(U#tHhx-iS>0n_H zc1x~=DHzc5fy^vjlG3TEwld1f4zy+Fkbj$t-Yx&5KV%UA?AZ&oy@v_DL{DUvHizN0 zkuA#L*C7BMQGeV(e;QdIIz{Mi^eOU?W~G?G zOque*srXq#W2hZ>fBfpvS3>;$Bf3Ulb>dRCe22l407$FYQ=WZe1GhSiqwka9Lw&{V z@uZ<iuda%`zZw^H95&=ix3y@CdZxEXQ z!JINUhTJ)AEk$cCtFDk$9HrjvSgb$mv>GKt1~p<7akJ+8nhZ$9@y+9GzV z6uyz#KsrFeD^z4zH_jp`|10lpcd1mhc^L7y1=SZ*<7r%P{OQlfDxjObAC${L`Gh)W zinoTjFno%Q6{c~8FaEfFCOeFP=HBqDUD#=D>Ov(=;)#ooZeM6Jr{6hy`&uR}0R-4Q z*)0D7bx^1ug8wFFxuV?=k{V^{)<@Q1jo+0E z)AnHCXTUeOrAub#+tWa^bJ-8{%r}O~@cmkqJy+MAI0aJ6ARwb;oW!po{3k2W!*d|G zV-QM~_jo32I%MnlOvbQG@+TEOcx9-ld=)i1p~s$3(g|pW!8&Q!@cx`VhTX>xR&xu| zq%>%Nl#c5o!q*+k9YSBj=#}q|kxam`G2t@%cxki8koYEGDy}9V&-;e4Sbv%iOSBCs zx{dr@DrxAldgnsH8VWO{BXdZFEMVFoIv#cGrzu;2bbQ~}R5q6k=77Dh&vmEbf~yAL z<-9zByGH#C!alDIMOGNt6uFjeT=xo!ug3 zvt`dPvrHz^Z*}u06fW*^wcYQzceM7tof8x>*@J%+C9KroPD+?s%BQzw$7Z*oke2ko zJQjfjWjohyfMS9VeKtF9=i=#z*Uh48zCr~h*eW!R!+)4;XH%g^fsNGZ1p7HA>X z6ku9~?m|kNwKn^Vm~q!{BUKgMn;x%E;@3pOs2O-3e7Uc&}W&Q|Y>D ziF^v=5`__7zIb_VoJBq3tz4s*3l1d?zlw>)=TBdT@pV?bFnxD?_bMAkyE~}&S|8l{ z=xDcb=ZSo(*1IUr9mZ66dL(v}GW!+As4+zS)|Pa_iyhu3&fF(4tYx zNQB27exR3odjEkT%SgqkUDv)heYROdxAM|GRRdS|X=dVZDjMbUdzP@q>ij>e1Vjme zn|VwnTK87I(iGpazP@_hobb>TnR=|j zIB**lwlgvv8*aNXjm3H!l3T|+ z)-7M+blf_-!sLcD+B~Z^xvPjSAfbj8*51+}fX2@EgqGcM+6PfLGST)J#WPw)$39!7 z;OfC1n%F%Y$g|z+P;JxIJq^d@(rB&hV1GOiQz)On>0s(2cU9g6>W%yoM=XQA-K8l?y>sBM1b5;$Lv=VVzL#JIAq2=%419TC`9|iw4y};^+ftd4-i=O zm1n7(1lV#=55z;UJiaV_BoR6+j>V2EQlE?|ulZ^+4{Z@;b5TZDDg(4Bir51W4KA>B z48N&sw(-sc)Wuo17Pto)<+{e9$tuVhyca=>yQ!gMS!rj^n&9z3uY^I`gt^Xs?OL?X zm@MdHt*Mo_?f&w+#%BSVXVgyfQuVf1i4Npy&*@ zaWnh>(Djy4ZFFJVDAGdlwm_kHi@UoOcXxMp*8rt>aR@Gj;O_2P++BmaYp^eU-uFA_ z$605s{77c)$(roUo_)(DOvW|yrz?~54cH54%&anroDsu?v=yqA1va{;uK-+5{ttJ6 z0Sb_qAW$ggI?zIX*9vIkWiVT;rXeo)>8Duy)J;Fo^W$ZsVTFTEzaOAS-BRCwFHA|i zZ7iD8JNE93H&M>#Qz2Xxt!}?eFs9}qAnU7%x)Dnc_I0d|n{xdKSx zOx;^S8)5yYXj5?jQxaatec=MeROQ2?;LFF z!PhZbrFFPUR3J(>S1`K-myn8QemB3qdy6XU9l{P8*VP!j^P^jOfCcv-dxC-31nk5^ z@tph2&mkSBqb4uaafY*dgBq`WWfl9Lf+xhjW?X37pFo&NYGs+$T=)MjlIi(sx$p$x zhX|yhH-mk-L*Oj<6KMDwYw9-0e2({cDl(;iR6ttZL3d!_#z^$778A*#Vn-7wU-pTQ zH1#$G8}UoOqbyM=V~nJEv_{qLsY9B-s!+yGsbF7B%9u49;gl980(VB! zTUf8M4y*?WMiM4}G};b9N{o31t~8BaBcEE``_2YPBrY)KV|z;0IMh93^XN!i(sI$j zxbL976zN`xpIeYGv`GH2%-e!L^~z9X74yED5~QCIIOiaDI1(h7j?1hK?vRfZ;2q7Z zDfp5tT@Ie- zu1qtA55GY)^Q!hn^neSrcBtpwvNyJ&z;?9Y!A@hnHYKEE{O~BE>yjf-`_V7VSx2M4?-23@Y%c@Xe=O+A96uJv`p;7GA&T{xMqRf zq-jAzR4E%_&1Bo_Y#$Y?mW$_iyM}&eJ4Y(3Rzuy&YhPdTSjz;o8y^<&lDlQ==$M~! zkYZzEz`f4vp=%lr#a9J09rhv3oPrc$hqnjVU&E0F!|bzbTmgSMOASow9!ACWy|?tD z=bK*)1y-wLUpdjQAJfpDYtEP!o<%O0N6eo(uICy`$NMbZR1SwL6in7o!j&MR%nm_; zdcTdSpsr_A3`KuyeROM;0PAh^tGG%Bsi{{5* za38Ct`*~FDkQ|GG??j2+zT!JEW|x3K__rwVBKylFMAk^1>EziIt+3RmBVQ=5ujuWwZ2^-Tap?k~kO-x4c6wN62|IE=T(Eqo zx(8-JPoG<#Z303>k+;}q?!PD&KsY@k1Vr}=aM#zDjhpVNHoOWFa1}bIAWj(-#r3SZ zx`TWPL@`dWbon!gom`Z$KqmY48@$N5wy|qh%HdA&j8yanDaKs!<7ZIFeaWlFn3VdP z1^&^M=f^k~z$C#?fAo?!RXm?gpA z=;i)W{2uAg$fjmIdq?|;Qqr#DLn!cse_y%ssYuB*geEg-Y7yjN2Jtl z^R>F*_y7s@5F7?XvSjbqT_heEAbqdA=txYgws{%LUqW+V&8|nB)pJoXUnKiOCuQvX ziKFT|$j=9BF<8z_IRHgndh&~TmB);Av|eI^J1%pARMI}T;a2}<^!y)M5;G6F;~1<1 zgVv9$mQl*Z0X52XkV>W*f4u=nFGS?SeQ*JVcq=5>=y-GBl~^z?G;M87K5~6Q zUo{R3S`@;l=DnnOd*N2or~B7Y?JlK5B0zFw$yan1oHpb%!-ZaNcGZ?K6dW07$F znjNtsSZ=*LBcngRdyUF>puEeHih#?jdKo-DQKLOD^n6*a&p#NI7uZ;x&7=AukX*~s zl=xZVgmaEk47Osh99wD%QTt0jUX{ynl%5vR5VaSU&eCLIdxT9ed*|84xL=|lFg`bG z?!Ajkmsi-7Yn#Y{qVfRR_PyzVfWDXAr^>d5u$X)EnE@X@7lZ54!Mj$?yCsI{(Pz)2 z{6;MoTS5FJei2UCA@^b%tE=24)def$$ zVhIKf1mcyem~9)9XZe0#ex5+PJ(2v}?tA}ij1(r?Ocf%tbtG{~-5i}kUF@zwmc)V` z87L5jg{w^4 zQikaZ4y{(^pZ;zKx=-r^PC3;(Dtr2}3j3vtj1M$-zpwHR*4z!R3!|hXI2e0<*ptSR zU8#WPR5bnARKVzmrv~tW2XooO85iD4hp7ByWpDrvhzL!{jX}L-LU38AK;%OL>Py%Z zpyMbrnC|K7T}y>YINr9F(@Eyn1J$gDA)tH)}kZSJRyHgpt-o>PHjbsW%=kHUcGONZf-lnY7=Fw#1ZgYu?@&Cpkz- z?BYn72Dk!&U`($(7g#|+o$B&yWZ=39O@ju4*e60`2}r}e=GC5$?Qj*Tu`^Z{U! zHTMejPpCH{t)Nc-o(yseTDGJCO4r3#Y@Z-m64v#}(7_u1oHnN(15xK^qC1CIRefKB zmP*Y37mev(R3jT7eF6ntJ1kWl>ih?~CZ%jvUUTh;a$-*fNuWDj`F8M4^PSzjp1XMs zW$-m+%7rXAjJTtdlc}hRe6BT;{tJ2eD3N~s0!1ru!~;?GGFPjpM)y|gmv+*8rpJ(- zSv`hmH{N?C0%37DrWwg}->%6K87y3uJOif7jT;tj1*Q0CU;WCuP-Wz0StoaJy$#|%W$9+^m1(j4Ly=5vYLaK z&eFge(6Q9i!Hyg&X%O)gn0WCDz!FY3Tz80fjo8pxQf9>E+dpzknzH=wCvA1 zBnJuJp@m$_{1pf9ffTee3`ZzwHS6io2>*;oe^-HbN6(tRZHd=g_gh1YdR;bCDT-T1 z85hv2nkK9E@ZjK)9$Ev$zKbGc(2I)68Uvq4|53L{c%6}AVeT&px5;{)TSGyb^7##J zPaC_nRt^VNWU~*AIx~mc~?UI1Y62!M6Oo^x7}2{+6piH++6wwss=N zFi=v9gC1gH(E2mlc${c8R6BFPp-^RN-9D;@=>8K7GPXK;G=$hEaUfMM(+J;+Snc46 z`*794u|4N0W}_#>_G}7Y2%`u--wynK{Xyq5`=6QEtCvsHa=Hiey_WZBWYE%Pu43lE z;l?zzXs{bEbRFa;i9a1z;6pBK+2s676hFSQeP{l^N_fz%CU8_*K~Ab%Ub~q(IUQAk8;9Rxwy1b5>?G8- zqn#8g3x~o!}Vvc(Nx|~29NJ6gMZaG zmyv~^64RNOtIEXxs^TVo)he(x2={DObGY2$|i`Rwk|L)H{-0A5ZF^?rI;!^^R%Cu+3$ z{PX+Kblw>57*C+WijL5)0$+P$!c|ypsdh>51pZ=H@t^xeC`n4Muar@YwIn$Q(N%cv zizBY))ZjQ;kC0A;eAyBCb8z^-EcxFZ^WRID_3_S{2&OrCL5+-XDz+9a|Lz&WJBX`DX?fka>F@8U1+#l$ z%n$mEGhpClIa_-nSZ+)oo46zK(4ZUfV_Vjx&?|@C{~?f}yKqPQ;Mwa%7t%3y*|bIw zu`&_bZO7HhofHEb2ENwAp}xkQNHcljem(!;tYH4DnRuciwb0;~Do?i7D~%(**6lNF z$LeVJ2*r&TSSOlzWnvGz_Nxs+XJuXv0}L&tgz5saVxexn;%-Q2e)K8<$aHBfms%(^ zI#F*#^EK(!;#!KO5=%AfNY_I}dQL6t?Q_K!4Y_ehq~SEF&@|3{#K(cCOA#-3@{uG4 z^G6*hVwiN}&lk6biABcbO0*2u|L(WL4U-=!*%*+w+BYRb2aR%=DgPrx};g}3QeLccP`OK2nCN=rU;bU%I z{DRom1Zq{P8jo7YX+NFUwVis|6Hqo72Q+||STNg>BO(DCYC;PMPkA{s)u%687Gzf+ z_Ex9U{u?ibqE%zby|Rm;!=^_6=KX(_CnLG0`%cS0_($rwbEfhbn6K|ubq4n`KP}KPULi3<- zfOk2~ZOp*+SNJer5r~`94vhDG*06Cv{Ts=wKTLb+^TD?GipFEOuDTOy&7<`l=;qo# z9!K9Noca}>3d6#7C;zXbcg!1NW!vRv5PWZVMq}IvSr^2G4*beWWHx+~{IHT}*2VIDcb{C) zKOoYrjfo;-U7@Xywl+J@V^ejVC3?eR_ObsDi-g7>gxMYIQKjODq4NDJms@B2E@VUn zfVoP3ymQ8pC<7MF#Rbw-%(Lp|>KH!8$UKCd=XSbQ>A1`>j&iml^BXyFMr-hKW8;9} zA5MEaSLLlK`Cw$iv_QDx4vTjkr?jU+(~Ec9o}1>d%d0MAl$qbS*>N(5iGW5wXE7}# zscQ=)V*SQcf+;&PN2kmvl-}CJ5uOyxKnDb(2Bo&~(dsQM-a?||nG&v#X`yJ_lg)za z$a%*LW1nWNGTv9I=LpLvMJH~mzQ516+3h#@neK?M%E17!F^y0r*Bea4V&?(kcZ3`t zEhTr^(^wuvKK@imVf8S-j*0hyF*7%<@!#;fJ6}B)AexBqvSsnHa+kJx`k6i9as@tS|wy#K*bIE|v%^swoW zBD_+|%de|eXRm0i8j4Ljv z*U@1i>7lOGAL`Bhx}1f6-q%NB;rIFy!8dDZw?cJbX)%z$s%*c}Op1X5ZCT~4o~*>{ z{n^#FlAFOtx`)T~(b2B6{8=HHHedCP&bM8OM|#85dWL0Ee>7g>pEW3BN(+z45kpj9 za~uF@)5+;8KRU1e#SwAX881z*NC?GC$YqFf+t>USv{GS~kd!9JEU9wFUWYcYqT)u} zjH4pesteW9Db^ET+0!q}iEanptUUAZ#hJX z&v+ANU2nx=eB2a~AGI>08}FQ_9j%w}wB2vXNfbK)e=xb9KK}*d6(4aly+eVdpV=lc(t9>EpXLNA);M%|i8Ew>EUioU+x1>lx9cAA zxqYQ~LcZ$TniiNREIxDiBmaPQCkph}b_X17MJN>UAJuW7;<3j6`6=Y8qHp@7f#0l( ztP3^M!2I!J69x`~*0&))_GokGdU7Dws`JE^0IL`VaYyv^effZdLQ2Ml>tKc_F^=7{ ztCH7vOYXYpC=7@Aadjr5G<4x|TM94$0ug^Zi_P_Clsa_^Wdq%#2k00ec$GAH^F$|_ zAKekLsk;*%Ec0fZ{FWkEeSu1RN#R^f_sayK5=cV-4;UOqbwZ(|qrH$)9m?^HPSNf; z)M5L?zol%;4__UOju*otuPJwOc&Nul57{q_BCZ$`e0>}FR$;P4+aBN&LPEeR9cU?a zgPd%Uv5NkqHCvKHm5>gXE0o?}Y< zBog`+_5q#Da1(~lp0d{I1^r4|B&Fp-YT%(xLJB{_7Zp$hUJT)wcP5_epHv7PhJ5zt zRmR8!Ydf-4zS@l?EKacI}s^*nj^_G6@Sq zX3ik`@@9CMVU82WM>x?y0OIF6J7JyaKcieQLZr|~{sCzQe}fE>wfkO@dSa4p;feU% z<;+-4aJ}A>`3uMT>NGO`)19e>>dQw<`*S|9$Pw_^w}DrV=rdHoQC%U_qo*C9qyMbz zjaSOOs8g5Yl+F(%?6BYOhXK+d-L)5w2W{N8kDsBru9870j*L!ISP;IudyduE}S z@!)HuB8b8Lau0(w2s|RXpD>v*p>_rBxBzJfSzaUT!}Ry<91_2PtI8adry%zKo+f zGaAH(p36zwl#SM5?P*+dLU$QjWUATO3z7^8GS)%b#j88cxyliZl{@GdtbRxeTz=GN zJ3sdDYlMag>pnVjcj>K{w-Y8j@N&YX3#pZlAVoMlu}$fJHy zr_Ml9T2}st^lsT}!m@|z>J5>!+!|OpwEc?d-Gdc<75?|bck{8yzZ!;R3?qhx!qxRA z|L%%nKS)gU*e^IYXgqD9ROmf1?WOvEdOJoyw|mM}w(Salo%R>Wdplcc4Fzbhk+3sY%-;s%TR%rO{vDGiwLgA?E(@(?Zm49-oUW|EKuq>{a_HMTP@Xf8U;><(kBh6u&3)B09yV0;qPD;pFgkoi5v);i=o2UIrM;x=(x#ub zyzs%lfO*Pc>j=B6gnWkTuhl~xHuSdX8?VO1*BA$ere%VDL-o5fs2EBHdr_iD3Z}9D z#XLeEH^=Q<%`hef?b2?i1pghe%x(GXRS#>dN?Ul_MnN&|mz% ze=aXY*Db%~;0iy1?$Sikl}&{hqlU;j$O=_T$-1~u z#3#SNRGvDmfQ*Ju{PW@->Y%7{ozmU;2IC8xbA(>^kI*~m(|f1wGZ&NkH3@$TLTzZ7 zvbeJBj?achVxx0D>1Ni-I%Yuz^9fI=Nh0k3CK5k#c(PBn(yz9jk{>>+{hNVzIX_)Z z{W=5ZeDDPA99?G4qRIF6McLFd4|Zr?xZ(!J>)(ugUT{8@i*Z=tOmq3WV$z!2tLb~x z$Gh0R8NT-+p0y%X@iXlAZ(z3{w#m_jE_sc{fpR%0=$~X2>bdzpiSI?SN9bw~%b~yC zGcKDk)2DBG!=N=`1We}dK?2jxcGv@=wL4Lk1H%#LB?yu4POtlHlSb1eiM{?%+%7eO zA-H}BPhhf2m2?OJh-1p3<~5zuOIqJg;nJ|HN#g6Imj0`R)z;FC*EnC_5!PHuKk~PSWm3*z0ofKBIXx^rbOxK14X+SS zGEYOjjQAYTUJ*A~vVg7TGlSB-Qy!tLCgoG+*RO*)pQ@;Lq}EwzUPWY|1KL?PudDno zfP4k_2dx`d3NgRiIQPgqfQa}bj)2G!SG;k1&A!A&4E*Hz3Zm1>d7Dyh0R)am1+!{C z_lYuQyskcrr*1HYCJti^`;X9WS>jy7k&;uk6WcUTUm(7~dUPEuK?}lW-s?{V_w^V5 z6*l}%2`}!C?to|dC~iQ0Ix+uBJ?t`g+hL-gp82GOtn$6U_Uq4ud~yA4?S^wEvNUQ! z7Ok1hg?8>HW-e>G%v!vsw3Qj|E_nPObC=&BtKs___|bo9)vorl&VL6)lMs63M| zKv7HJiXywHsTNdRrhqT>I`Mk063}^vPQQ2=v$hbK_^o%7=GDE?*+Y{$U#ZWrd#yem zG+=}&*H$0Re0b{r{@VHiSIAnaRx*QoFJkPvD1zR`)B`V88eu~MVHWbN!o7EbVQ!sy zI8$^P-6xlc?AQ!#-pUnt%Y(p4J#5^wB`?kkci9IvxlIqH^-M3CJx zJAW1@i%dIw2-pbnpWN3rNig+~r=Sq0eaCdEMSN8v7uoXZ6Z@BugsAobc89lp3gq-O z-112yNj4Qa`G~%QY*8}t_waH5&|J-I zI53x1VzzQk!__EaSi1fJ^GlzDIzQN`*+4I;Z9-+05yRJkQwGqv8W&CX1QeR+Ab2mae4G3`&u=uioKvrXzL*a9b2x*>_fQvj)L6Z*2nyR9&J#8i5B?BUC;oso47gLfm4jNhe zWF_B6rtG(C?2h5d7`WTU_n4TnSc>Yo*>Fuv`hqO3e8jI@2?FYnhQwriX8R30G^@de ztMzMIu0=RQ4x798%59wZg|L3EO!S}!Yrbs3`_Beus>!&)0wWz}WDaM?S6KW``w z*AvWst~D$_SSNl~p$MS^P$`oF&JZbYH)m2d93MXWyL_R9IAbtbCE*J8X2=4cjO(ddI7o+Xl#N zJDIVC9DWR`5{e2}k|7?5Lh5d(hYpU_G*lqwjl;FGhi*c+|LuTdV$iiB)U*-aXBqtD z9o^4ukZJ3z%vSYV4vJ^_l|7wuyZhBWYZO~bNvKHy+tU8Z#Tfl9jDIY2C7+QQN7Vb% zujf7Ynav`6Yl)-`5HUmvW>%p z>FhnP*3DlS@y;q>nfo_Bb`L%n#Q6%tsha#UUeM1il`@?IhmlYN7np!v-`gD328Mi8 zOc%^$RqHiT^$w1>&|&o2lOb5yM-n2;xA3;RbxFW#5Qlm64H64@b6BkHYRIUf9%W}~ zp+If)KqxwV{0dAKN9cF(1O?Vb@-3bi88~8H<$uC-uLaO`|85^Oz*^M zci06%Al~V6Jw9Sat-e90e%BA)fyto!znRD6t8wI(uXKbwn$D)z{*U*`yv>sZyqFlHX%U=-Qo^hbFwLImzINH+7cjBlurG_kDIO|yA2#g)H}hQH z{BZpCa8LPWQo5YhO-i{5igzzZB^dJDw|Q>c>zsZxP@JSKyL!~&X&~jahtGCcAIw*` z7Y#!ElZY2G0qwvysMg+=Vn07UiRjiN87_xG^!Oc3yNZ}rWuh^kI^V7i!T6r;wzQRG z{}q`P9nu9-7mD)!fwM&V?OurS1yd;*I~dm%!l(9_Y)o-r)*+X0_|ZkXMo@B^$+7>) z-sxB|%qZR|HHP3vh-<^%6bTN4F168}hE(1@G!y;+CB(LPpCY)ZQ^q-7;{Eb&pv+DF zc0A2jOWQww_FqG8(H{3ybI5S@^bZ|p)ah54&_{Aoj56B;1jjw5 zmB!PpWqOhqCPv@t-=}X&BSRx*lLHuBqx&OYnD@s#ma<@zW7(d1@2UYS z3|GT2!+%$x_kD#Y1AOe*l%8^`szkODN6M#R5p_JjH@{DU89!@}^}($PcqiUyyG|f$ zEhmSI`4&Cfw=L{w^T6AAza?_oj{B-*iD6bDTzgW9I+oIF?CObfXz%}<}o>5>hn-$FJ zyi+Unuh4WQeQSdKYkMIn7HD<+&%vZzwlB}vokOSbScj}@z?n6&7P+BCC2@#uyn>QF zIyon-y<&X22=uL(7i5B~(cV=}Z+~Y>WC!^3?C2YMp-q)bQO3h4Pit^q89~JKU3?0P z-Iy&+KYpnT1_qssGM6J4A|Yp!GgCAxd6MUo1NGJ@cm`{&CHU@HK3d-{*k;gP z{IXyed=QIs6(+v#e|#!>}OXe+nm#1dLgSuxxTrAfnZ3>p~i59#XK z%6T<8AbLG?;P5wL!VsTI7VZr2vxrK53=*7uTFF~t%|bmE{XHxb%OM5XJR# zHTA0FXw~$%`0gn<5bYpfxNEU-{yF*oY$~9k@Va?o*pN_y!sno3ADEV?X+AXtHJ$MH#G!A+jEHXS-!JN;*o1C4*Zre zp^HO3|BWev+yHO+4HYnYn?5(|w|&>VWL#u_@fhiJ<8!44if6HB^PNu@%n(mdx%@4k z)~ELEC2$%5qe7HDbW&(>A$&vSi^rllJplKJ+xnYV<1*V0SlGI7cs9PDn{S0rHD1Yf zT8v&*XQ@%yloxco%UZM+zm8v?S31MzsyK7F!2)uD#{Fl9siV_fTBu-Uxa=4oQSx)Ku09zU+*t zE!Bw)xm`I&F;vJSY8t9!3$g<36dj{V#aUVR^&FCr5}{bp|H<2fydg4_;=O=8M#>Ie z-4z@IXtH_U@g&zW*z1Gm|IO>|cvU|*9iS7fPxW0uu7e>3`d?!gYsfrsGb75>&ObDB z+AUv5$6r%5oC<4{^3}`_{i?q-iESh`9dNnP6=+@xeLUKw{CX$M@3Hk=PRu60DiKKt ze1qZ@4j;0=fS3NjE-z_441;|~8aWIj9Wj)EJ*U87(4Ps?1WOjecI|la@%8{$qgW4c zu;e>qD<9XGQaTIn@Egyqo${@I!wz)Z8O$?Q@&O%At%ti_s}qMymgEV441N=Z@W$Um z5yekS(gKl7Ltc+k3wsc>x|;7gZ*j7?ld>wi^Eq`VU#P}P$S~{kr!AcHoTTuV4+`C1 zKyJ8f@!fatp9xw6JxtWEz#Wk^3)io1nfRPS9$v)617)*sCtITS0$zB!+0 z9=nKbt>*CfKD=8^V4~ze-Es1qrW@mWVqT%~zM+YA&O<_|ga8Z2SA%wh?1pA{E*CdcMW2WBz+0C0rSh9DZ|N zl1q2WMvSjkD)Wd3A}zSQdmu`IdS=sX8b)*X@8oAA{&};>@l+ebP`CF-0o^(9%UP#J z*nh)5#V4AFzfKz1#yH0~)$=x4Lhkx3c<&yY-pvj3yP-}@M(Gs|W*NdSit{L3AM(@F z&epPWf{AAI28V?_N``Y~y1CRZ(t(x}a&ysD(ocLHtEjBa!6SnKX*%4OcRLohsMXPx zP7YCDz)oJ=e2n$8HW=~qToO{Brv`CdOlsFN`_p>qKD>A^iPUf#@7TzbQ8nH&= zP%!4#9oX_?RzptCR2Q*w$8#DxgV%M2(alJ|xBB50*|YKn*Sq=bMl%=z^xM>58XtoY zGu0-^DEL1I0r@8B+CD@@P|WJ+`{B4ANNrSQEe?&O2UF7^N|f2k?*2qR_4&-loF=6- zC7XupoV@%Eq-eCesd+i`is5%P>B8JPtE`90RLdAs#B0K z5Eq*r$pF#oPioN#?Z%-0&5LsO6RzUo?A!!YdF@NI1nb*V&hV|1W8ZK$(JrxX>bo57 zf6+%LhMU99hbs3Z<$Iszi*B!fh8Aghu7^j`O?hb#o}VgobCm97=yH8GgS z&b2>g0WL-H^4YbXA9dfk1PVGq`|1PDu=b?+F9a(Sm$%n>nGP#mKNDG6|@y8BX<&mdi#Hu7@o9R0oU$Uj|Zd&KRAku}{VDz;7PL>_%fakCX1jCtuB z9{MqWtgr2rVk5x1M9`QnvN5)kv(@YSJRETaSdi-%QJ~uyxUoCe z+vUN_m1ZIH_LfiptH0ivk~d;__JnGqU%V(1by`^g@OpZ9Zr2kZ>?GFR*X>%{L-XRor3SgRY^AagT9uXM@ zpD@f7KIHOul_%IPKk{lfR=1U9M=aEGWyv8%6`a{cM1PX?SLfN=yfa)fqx&~pKRV(M zd{?nt8Lu1s`6t#?LjHtg3C}Vl>AQ@QOj>mrCY>{0U1ca8+u=#xq7sJmB)> zKWtVa>E`3<(kxBFI=&%tU5E!vWZFC%7EfSloo8{W1!Ks^ z?MiFE{flDXW!#Ffo5HV$W>^&}q>Y?y96zH2NPpc{&Wub5L}YhJufGzWOf(u@U>&~P zItSqgBk?Y}h(%DIu1J*71nd{cTrS1%2VE%>bk5ogU)bI`EQyqO30gfJDxs>IRe2rj z=hgYbVBXrLP3P0sooh4$8s%&F83@HJ4%>O3lpG>zj#+1+dP9hr!`)wFqUC3A)I4Su zw@;1&U9Rcs^>}fJ5#BvWW-p+9>B|eAdKmeQ=1XKgaPxK4*V-0s2ww1eb|l%NyvSZZ z+ca`L5R=i~B=ymlECd@bElTH|O5ucI0oY8=~NY>kg9xBGtai82ai-nsC;QtDD>O*exrEa z_EWn$Z1Qh2nWM83-zDNKlvg}0QUv_P!C^jsw9GAp%54tFu-8>$cEAEI+~Sfs<$%Ym z00h=T1=2dmxTa^o5|4YX&UrVqc&v5N4meW^ST@v zK2By2ZWqIcc+~E8E%CWF+K=?QML?Upq%Rp>R&t*^^TU)tffpL{OF8rgviC*{L$qx# z+uUAFL}!!L`OyjeS6AXGFM`oAzbXs(hMsnh4I!6zF~pPmSIEK^vo6wl=RXiq^>&g? zmI6ot!URLhI1^@C?38aGTb%R_D!+J?$eXRun=&;tw{mGx8_n@$Jc}Ht)+ltIGu&YO z{PmQlly~LLUf-#7YGd{@H5A?~8DF4h)=Qf#$&VkFYLQT)yJKTV4KD}HjvnkC$Hr6l znH7Ke=&D&?-Gg5yf{LcN#j>DhI*TKp8v=NqTArMjABtMd#hcG7`*wCu#faEb5z|TD zG0h%p6Kt@vQN@kbTM^Qk;bs{;tscyvu)%>Tv#hKt!fUP4%RR57(pV`*%Su=Nhe{#I zk%OtaZ~EDM?6PI{s-sK?e|foK%iAP|es+i_k8vI>1cTL28UM9aUq4^Ipn|Z;*VK&u9P{gbvy3OvfE98 zVszBw2-WSIL)@wM39}%pC92)(%EgTyG6@(oxlS=HBwrAh+VdD${Qmazmoz?AWmWRc zn_H*k$%V@np%Svo@y0$+%xN~}+)xOUvW3qcs&uOIxL3}e_xE@qulf`vdT-8gQNP!X zRWW43Fv*2)#5m@!h%kS%`)O~A#jDtJB|cZ(DT-HWz`jUP%L}VhNLCMDOKx;~(jiPt>UV z3HNsJ{@q@sQUCp%n{a*6w!mGKb|2LWnflv4{&ao7^lzmJx!gfuIh_vR{``<;L88ea zD%=lKqY_4)K2=;u{t<`c_6ysU)CmD^wbQPySJ4u8OH*97OYl~1>vg6FtqSYh@l{WO zB}mnHSv`1fdCjV-+6be*a1$c8HG}6z2aQ5yKe$q`8jOth?L9Gleg|G>n;pzT=a8N! z9?xy60~@s8x;GAJoApd|9*5$y{c5noa;;}I9QSsfOZJbr6!IY`XQ&=UVxu{q-`7);;T{ZM4Gvdvaim}^Gw5Ldmv!byC4KEZu zT#11I`w33f5GAAf`cCFk8zVajd>Kqvu{pi+WvvHqkZaAI$D!u)B(Kjd8 z+QjtToaN)SCastKol|^r@uvsS+LrkcnzdhDaY&Pi2ZUGTXllk(qXmJ5u=nc>b+W-Xze-2=-c_(QyZs!3QlQJHq<#e0qiMy&B{wVi|GYwxfmwDp-cjNg~8PE73M*EB5 zDxnRE%DN;NoB?pHRg|7r)l;Zw#9`Snn9IK{lnj!+f1p&7_?F8wPcdgOZ--#wo&4@S zMadYN6#S9n*!^}%naXF(^$P*LYp|WYxNwGw+4WDzLlGDi8L4mOkb`>YtEB^c(-xG3 zY30N1h17-jX2t5oHkpyQjYE}#nKLogwULW@&aA*)@kvZ$ zDfOWg_TagI8PSaRJ4`GBS;Z<^F+|Mb2h0NKsbVuD!&nRgm97 z88nG;@7&G6s+03k53vkz|aQu?Eoa2mToPudY~0NevgE zIx^W36G-~V(83MM8LMi3JloTd;>t$kSbz7R8JCSKAuSLPE$l-asB7=!=MTpnP_m#& zl=(nK`n?AU{i;v`kva{oGy??;+`gJOMIm)jJTe{5y&Y1|b`Mv;>-mHx0GPu^uaT}CGqoFR zgWY^^kSoj;-9e8;(N|Z*!r=w6|37F9=u{BOgnn@k;$sO{`h~c)zZM-h%PN_WA@w4= zLjn*q@_qYyaNheD-2375Tt?#h`*_~L#^BYHELf^c}PZ z+QCIt%a!0a_V9r{97f=CHdiLTplkc=?`o&OgVM@izwns7~1cR(=Bj&C# zo;y9Cou~1q=VD#o=zMwr$)Uhq!{~SY&J(%qa}$Xcw2Q*+vFm5&12{97mVe=`Qkq9x z8EsmGADd=NhkPTt;{eyM;t$`i@~I;y2Y&_rLOQbQZZ_GETMr*ta3F#001CpF=w{PI z3T*cvY~9DAw9raHPHzFOj!Z)QwpL4aTnU3{lua6l$f()yJ;Vj223+cQgp!MmNOn@U zmF9N3d_G3z_G3#X9{%FAE;6*zG8HQB+h@M}00V{m|4j|qFj#eH2}X5Be$(Poez6&O z-`Ccio06HA+B7{@i?YVoIypp>4R64(8_c=ATVKku?qN4*Rl*y!>TkEAn&8_`yJN=$ zS*vURiZSWDj8QPUvE*{%!7x0s%W%4tQ14?k3PS(aW^EVyiY4Y6MtG>RZZed9UQIrd zs-!Zgmc^mAD<9fJSO)RMPG_V(;4p;)(8H)c$VYwep-7M;riYSK(j^;_zIj{6`ju#c z9uR+PU3IIoWQH)i$-2#O(>VG)P=C)r=Dt`%&ohE6SE7+S{DS-AENPd{hd1N}59j-L zw9|ysH~&t@e7z4-32?*rY5zDg)zy7uSh=cc%3j!$^Hjh+*ZXtfy=YwZzB3X%@X&)q zPyQXHZ9+rT1VRn>$I=ar6OLgE?t8xv7_|h-(G<=!<+!Et0^$p2?^aAq!k6MALX6`* z4jK(QG1&iy_M*vA_~jP`a4h8IczaW-ude!Gov*vZ<4euS2=z z=~r?z>&jLD(La_+FvGVHv)@UdT#y3?6}mTaw-z_U(#NFc{j1~WkhBS&2R?r@P*KNG zfMDD3d9UX;O9ppxy_e4C2Rv!JSl%ipwKaxg&16Lq6X%{6LWwo{-VP2Oc_PJZpsTAy z?0cip6(}s#?E8Z_3w2Y=eJT6v48@j1d{4poR=$=*8)+(D?$+$lE}d3g%Dun(_D7cL zX@2cSj=H(Z?8A>tUksj%KSv+Dz2LF^lz3Jp^KZ={|D9BOlo5T_OG#1=XUx_3za*ZG z(Eb{~Y(G)`gtKki=*%I6Ruc197m95OL@>Ba*oPr{L;i@EhvNoRP5Di%+?f#eZ0$kQ`0_+@DsY4=lTH>>v3ByzPfM>`wkb@xM zM0CF!s`c*SiFucje9G;?172aF2h9HmZvjKEoy$~w#aMJSm*xp2P7a&3v29Czr<5ai zo>naw^$hN(^-xPiXw)8fs<`Eno6U`Tav0?plUqjsZRiY}^tA6=vq;2{{SGSpM^Ot2 zBwA;nVu`6DSAyq3qi&Utt#It>Hq5N=^2%n))cj})D}?rzWQ*;HLim$&l`K>|PZ4vjD9F zLa@rwM^WvV3XiL-L>1D-ZxZ8N+X2y_PW%EYzV0Li^(t!}Jf0uPK+yt(6M~!YW1i*x zlbQFla-V%XEI^>d)ip$^$wlz=oTF`kExNzQyLz7Vn{4^`aB6k6d7{}r3^e`26$yMK zAI2y+4{_ov51xvHY_Z0(lm&y)Pq7c%>Dw2~_y7kKtr!?&I41kwCoYbO&n%P2?iO!N z@p!@;q!X!}4t7F-$aUd@RYwV$cO0hc8bDr9p&Y5VHk(rK+PB+xNXa*dVgFx73KI`! zarWfeIq|mUd8-ReQ+i+ZZSjLokNhg>sA5zuOvYXvaV+W&DIVvgkS6B&$)0Qr4a$ zVr5Rx_$A03)NEd7J1lo7r&TH8u%e}QI5^$upMBq1(dJjs+odIX%SOow4jsnuAb9y1 z6|PNyAaDF%UJdkTax+X0Azx#Y^1YsCN$RXmF>|OOg>Dl{n75?cZZv!d609~*mdse@ zV1`6nRlI}2?}Cj%9%M|9DyQ8iI-R!2`-T3vrRl8AshU*8BSTK(cduW#2_F0!r`QJM zi*C#VmWd(Y(WN2pj|O*qq$>k zMjjT0e2~&2cniTZ(nL87rMZ*NVrklp&ff-e2yM~HDBKrT0Oy%f9FvAvQUha{1G$b| zPK#FKqje!$`EujzNc}X;eY)OXG77YN78>Tk=FE^Fl>Wu{vZT4bvs{0a9nH@7+urts zliF$n=Q>N2T=55*en#2GoDmcTpLTL#s!ZSBH137P;2Zowf0cUV}k;O_43 z?jC{#cXtc!?(Xiv-Q91Kz0Z60x%b>#Uw!Mxs)Fk3HG5jmF~%b!`5H5ucu(GtF$M_a zf~qhL!5owz-IaT9T}7Mzl3A4KiEGNr!U7$0>*nnc6hChNe=H@jzx#bOd+jg>!s@g~ zlbbrzgvS$tgS_6Du`O5d-(-b+nZL;j6W`NJ4Tbw*cP!TK>zJOwo2^Z=I*ky>_dC-n zJ=75nMwh#Ee`lZ9V34P{?-qVtXbs0!AW29Tj|lT*UcJ4YxplPlQ!_SMG_ZYTMRZR6 zr2GK)bKw(qL6sh`N5(K{Aohjq+j&hO!9F=t$=&<)2Qg$H6z{#)!|*l<4Rl0b8lu!} zZHO+V+y5o4DjX7FsDec4DFc-cuNZOS{IJ`^~PmAD*0S69Sl1lI@sc~~q% zj6e0`-qL-YA_LYH=@Lv#XmCp@qdAEa!blcc-9AyR!Bqs-~iCU$fYc zCYNft+pxDC%1M2(X^4|<*m|~<&0zk)lHEH#hK??Eg+}V7AW)JlI;CYP+(l0UjtmM_ zPYzs1b8bvQ!oIz;wrrI&C7pQ@pKS@DCx}sE{{ter?Kko|jm684)0K>-!+t!5^cU5K zVvMP3Ux5taxD>uC`y9ElCu{Kf1lCS&zVzBv5{?K3fu<}eGLy<8GaUOuf;B~L^rGW& zU#I=KZEbF2Ff<3MZU2B}U!@69jx_p@$lhX$kL96858bGK<~vBH&M|&&aUxRyJ^&hK zeRr%yrnt5gSyD)(fBbe;QsD=<(w~sLW;|Cvy*Zww^b_D-qe*LT-!rug?S(zBSbyz$ z=N5N0voHX0%%d#P+)|BpES+mkT~=jJ{4zk|oUgIudf5r!X!DTFGqUT>h>pIEkVVbX z{T%P?eESARIQ_s}!d$hNxBBvg^-rJ-FT!dUU<&r{An=(2e9z*!+E5Dt45gC|Z9WHF|jp5P3j;o0;^%sL8$l#8v05NMf=-~yKlrN!3hLWp02F&fWvSm6qQ~;SrOwg3(3ZwV6Cc~3 z$P&~vR)@(nvgn|GjI?WVW8bK7j|VB_ZyuoltWzT@D@$FH8v<7NOv_Sst;-^onKfSyGRz=EC&(*|L)W|R_8NhD689(7bz+W7kxSMlA{k_50`|BC%_5t%t^x#vox>u^o?fEUC1>PQ{r}Hs`7549RnsBk`BWVFG~;8V zQ%(X~GYm)!5V?;GKZy;*=2;yzLW4rWlNf6wXrQ2}cq$52ql7*sUhhVaXJkZJy*6*l z=4yHE>wDeU4?^E8{z(R*ZG&xG>MVW&c}Fe=fW{%N$|18iP3xmwlES>bp&aYQJA#7M zL!%@~gH&#H?XKW^2U;NCJ}_rc7aUI)s{?t8;C`W`OBQMO1XUGj6`Pa_Y{wmr zlaDhFN7&EiZq!56C1vH7k`96moKuG?RbDeCSeLs(Z0M@8Gb8(hYZuT>P1nVopO?ZR zNKe;kC5I?ToqOT*Cdc4Ctq1d7A#6WHPM!o6THN4*@9UipkQ}ih+KPkznR?$pi%S%< zb0Wjr+Zhryjl8AKS@Q$~r0sw29*)J*O{D#Ap=!=A?#r@u7PsS zkNJXQshu2WxKA&xA_A%I%%)9rbQI%Wjyb7OL|aqA&6vU0SHa=5R@#~O*&wreE18+oRo}V z|GONSd)vA7M|bkzK-h+}qkaqX${{jcilhft z=uy99Q1U}oOYyqfWG#~Zxy(H2aGUj`TP$cCU-ID`8G?LxXQ`!nNb`|sz9$AcBLM>? ziyFt>cv`oK^Yfc4dfNwsZ^;YZuctq&Gysd;&O3z`n z0#NkcM%;0T`UGS={9I#&LloWxjRWE}klU5d%e2W}71@YQb-Soxh7cN2cSg==&D7pa zM4xdUC^@zk;f6(t1^rdNcNvNIoQ^n-jy0ydB`wL0te8M!Ms|*MEF>0OJ0FP)Qq#Lk z7rJ~n3_Z{N1Z@21&{n|pwwf3U*J-=hYwAq%cl%^pw#H#N8)vFc+BOPBu{%83Abj_IBD+k%{Aq9BsIzS+# z6VF6)FqV2Q#YrBoP_~jpFti0~bn%8V>!noZ*E1na>*#7j5sz&EO{%M68iu`AhtKSH zxbwCdTD~IIG+XN)qM*wlII6x-7OcI?_1?qKU2+n<0M0>gvWy{PVQM%UzvDX4dYH9g;#%21bJ6viXJ}cB$$0$2TOgiBF)0L`FX8q7nJ7N zPiExC6f}OH9QC~L@#PuX)G}1xB^{EqB>nTlo3zIZ`IcXdd=ho|OiXn9E5E2479EA9 zSyoZ2ZUlAY&gTMGMTu6aLXV3RF-UBYM-%g^!K|z}Nu<3gA)(tp9Bkv!^jTV48i!ghteS_8A@xmx zl|4Pg7nDjnK79apz?d$2W@kSg8od|Ac=SFFhvhJildC!k;67JeqbH2{4kc`zSlwjUx zY8LjyQYwkh&rLITccLto`f+&kJR7Kq9rre|4neA3(4T+N`1GU$(%)uwLP`=jY(%}R zR?*F*Z||Cdf|t!A^AO9XN0mj*8dnZzz(YvlhLN14AFu!~k#iCRb-EdPy@~Z;jLOhp z&bQc#@>oOJ^G~-y{9XR?z!J`O;5hBcY>Fl}Gg~GIl{>8e!>ucf7{sX6OzM=?M z*a@e1I4A;_yA&a&8J6GhDQWgW)YZmqO*d8oscm?|ucAqG>q_i12ff{lX%n85^;GXl zO0TYmj7-(%9h5y1W(3?){2cw_mYwX@ni>yJT^CPSV~}237oAsYvze2+Uuy-RZFl=1 z<>-8-=@)PreD7CQi5?N)I_^x`d=*E0;sz%#)3cGPJ??Y&*~OVY2-Defr@RtYLfKqO?K8A3~?e1ET!A0w*?3By2 z{=&4D&8+SVXHxo*@$gZ-aaH`vn?{nYr+Gia9fo$%eF}Ll(lVoVw^V%Cv9$cS9qcEh z-)%q9xwm(iUJUk%`$O;;6423Kk`TF@MlbH4-}an7zP8Ms`NZv-t7s*0lp#Xb$>6N@ z&SirSk3o)s);9g%9qbd03FzrJ;>Rx<10C_2R#S28NOZ})VwpW(-!YjI;+Nb7NT!zT z9%h@;r730(VDlMG1y9(!JA@Q#P|qU3;<+A(wme%A(tz;VcY6&hE<0)FAJg_O84j~w z>_B&GhEMhWblUfBD|cq_2k7AxmX)vWOt*efk56ed7dM_QYN>$Q*xwnL=gDDD+M2MG z&DUhw@UJ3G3`jw4qQCLgu6)~#W-D>wpwIwwL(KpXD9RD^kWiP_qCM?H=Y+KnGi1o; z72VL-w1X%p8DwxCEi$x96gnpj9a0kgXi6T{ zao{5&Q{iw3hqMJA+Y;iTc7F~sIVV^W^O|5NcDa8`Dj7_AZv50T)Qou_U3&OEX4>g7 zujiXAoTB5S0+OFZDguub?1$)B=~(BVEWLsUQ3J5`2wsq>YXL;!hDw%u?6D~-gSs18 z36Fi1ZfKe$)RU*S^xm8DlJD5{&|cU-mk5_u*q5jY0g4mnVX1`3V6ZNnlq8EdM9pVr z(COP#?S(g*^4d+OUpexJAGh+(iszYWfIRhuU9dd3YsC_KHaBy=Ce^AL}`Hh&l+ zApJdfFdE)Ii8Ju!cukXrlXq$YpH1;IS062}rzF|&B7f$T*1#9MwH;j7In_zWm++D8 zoHM+96=PwwRv-Uh#WPa_%sH2fxnlnhLHuC0$oKYNh^&FzgEDgwXvAUh*6R;q^e`>yX+5-nUgqG)45zdQehcL}PA=^Rnbo{ps>z!{WjFQGi zSi#c?Im&egmidEBo`k7jIT1Sxv3R50fAE7Pua^&b; zmlbZc3R+jyXoGTTJm}4c^_pDWzPJb0@-vI7*tcvQ_{EWc&?5PbGp?&%8tP8HpB}C7 zE*X&({qag7XNa~retNsvOhy6zbb50bsR>J|%pU@OaxP$!J=rzI(fpbq!<=$+R|4sj z(CuWCatVHuvt)}Q4pnJZvM&_jy&bG~^Rm7Fnq?lRCAh{ThmTx9zz}pbOF8|^n=C9= z*!_0_mmcIihh&60f|k#KjKb3+Qs&3c`p=WF!mv^WRuGZl$FD=H)aXE=x;~w}*x;kL z#};tz>M}?sDl+T%M&Fst$^8sCrmB>i>W(7MhBI@w`tF^)^o`(}CcGR=?`0tL2FBwk z*FSG@RmQhyQIw88dw1y+42|$&H77i+VY^jr$aEWe*CSdWL(|@$%@^NqiLN9RpCD_# z$*X0vqVgYRP$<*x#Z!1%ZNwQ6Ow@4X>oyvLDY#Dbr>plA67y=x(xiPU;yMJT+bnVs zIOFj<>Wp4k{>0A@t5@C;7Rm$anyw}qx*_X|ea8aswcD(a=n4JE$V5uAg;@{^C znAL41J8pa3)j`1@hlou=SZV|t^J645w6f`Ts!SpnO|D*M28eu4(W_F946o*eR4oGp zUP4#y+o$59JSa77s~otk9{DiSGjBHNUhi_!B@uz*CqU7KIRia2iH1!|=^5AG&i@wc z_A`D&+w1|7?R*ti#`tQb zEK*mq#pq%8VO4cynXRK5k5Ih8&!?2bEty}+Ml6n5wvTq<-X_#RYSK^9E<4@b0R{@( z@81HH$-lR1z#nJet4>uAY138CW%pwb$(HF*3P)Y6a&UfxJ zJw2!WgW55W39Z6JELYs4P0vV{UpxcDXXHa{%%j(#Ky(Wyy0&rW+vko>w$l!6-aF{r zons@E=TR z%p9JRJjqp>qVy=tUC2R99CCD`)$5TmEwjLvUY&q@385`2i0g{CS6@f;^j3owk+ij>-Bt zgzzaes?5Ygy-7|5d3tweSbl2~5y%>*-F|kc7XJQsW$4dL?E^v&RP#QnPT(+v>Z%#Q zW%{^f1E#(xvSCS(Hn<9LX7YsPPC1lvIY|K#CPDcmoZ?ooF&GR^OGh{Rk_I_ir(ibu zu9yBQt7%A%6;tr#w^w)^i$w~Dj;RJ14)HSWn?iM=OoZt-zb$RID}jT7M#4fJv=nOP zRWFNUy6N2)obQi~RjUG{A>j8Ld+BM{%pBf#nz(B?ZNk_His?(W`{(Z!wreI-0s$j7 zyD3c}-$q9XpVK(IO<0LN)Ssl=_l-wx5Jme*(KyUe`P#U26kMHoz6i*aoMM}{qVqqA zOVy%yFAvQifB<@I!-D^ID~n6AuE{*lD6IUytnJA zLI8{R{vh(4;tkUocR8F^FC6pBmr$&8oEJLNxY3=fExN9-IFv&DotrW@*f!|%OIe!4 z%kO#@i5N2$A%um2AnWgtT&pU5!QuO!C7mPT*TUvzTkX3q!jn^9o<<)+jv=De*KfC> zc!oK+NrWd~5?xTQmLEQ=)22D{bcUa+&kn{~hX2%n`Q0`qFmah6`f>q6 zZMy6a#WnSYH$s3kQ}@j=t-N)gX~mEN#r#Uy;oBAp1UH>`{q~K14hAIafs5|Ho0t39 z@n%Hi9UhLZ(@rC`n5S@>8N-eK{Ybd8v<9LV6f>E8s^Im~$-&jZ<4ors!udekmHU_lClrlVe$_p)KC>ps2a?}rg|=8y5LH|C%*_xEYAx1Y zAab5XqUzEl!F6hEIg!(eI%8jw(DQ`O?hU0cD`W1r>JTTQYxkUGdW|M@5e|R@Fh@B) zLpyDCskqR#kYsx8^K9ag_EEcNGA-eKj%d$#wXW!ush=8kM22tQ!f}p0YMC*VjP@unQ7*&>9k?An9zczcmARFC8+jH zDbM#+mb<>`HyHkK@-efD-V@s(ZkpFU&B<#Cz|g08wsstM6}^T(+hkv<=h`xY?~7A| zf*5_OK1d~w*5gve-vL4YI?r(4DD}pH6O(?v6+D8jk(Q8L$v_=WtM{Gf!&glj?^*5z z2^R{lw>7_Ttt*Fi#(LAN$R#Ol4_4DwEIth9%WsnnY2Fj<{AlZ%0b_qK}WygH0=Q325O>gTeSmgw1bgPn4`5 zWte(rziix9n-F*-v*4}SU~1Jd6XK4WsSgMVWt$f5Po&XW+U~J`xr{8jOBPi)=D5op+N;44dt$=9Zo$L(RTV37^Rm)=%k-f47a; zR_JaxwgiC#`Q3#xa~ouPP}GEB>o=G~$_gDqxPL~=Y0hh_jf~%XZoxGx8H1y8N`G=# zM~pi=0S@dG))kCEB4Wa^SZn(b@jh|U*?NT%Nk2||{czB)myBU*5atKf&7fxKvveAd z{MzPhn#a-ko%1x_o;IqjrC|7psCb_K;ccttRu*pDn3nHO88Bg-Fj6TIcbHo`4~oE0 zl6UTo5Y z70RW@H3X#8Aw8NJYC#Y?foVF?j?;iz@e1m9mpNIDCAZHV3GBKs42EJkrT&~wtz|u) zDroIU?8@~>jbr4wFsg$C&>4Fpc#8ryO_x^H0B&VDIxa=1hHmnl(9)QR@z@$82b!c{O#t&r zP0BhYnB#J?MjYH@czV2W1+<7cmd z7Khw!y@>o=tq5gk{s{wM%t8O7?r`I4{tEva8XVaBYo<(WFG4 z9UNA5th3F#YY=k{X1+Wz81$}m-nCec4Lm@@T?rDpXSlN{+VedpLC=J;m|@n!CLd+L)*JRdS!0Ht7_7#rW3B$ z1PNu4Vwwy(f12#hj(3Y@af#*mA$|ar;8SiT4P;}jz~pa-eQwu*!aYgE%h_0!1)?Qa zc8;p8;62bGyR(0-XYq6CkA}Z9@gYq;R}me}yj#oufH;Pf3!NxlVbn#H@aHEN2&poT zFdz5Uet?xru!LUBf3Jg!2gM`p_9>zKU`u{qZ-2p?qxK}7%~k^s<(O#Z8rN!Xz4tEa zVJpMsZ5nHmg@^C$eG2BLMhgM*Y!nfIT_uCRfWP%KVdzZyXHk`)>n(EAmtkQ{ zoZ;eQi>mW<0aoeJdaKJL`Q+FgasLM&(^*i6^R{N!OM^H6V$09KgQ>stE{K@u zki;XjBg%43$1|a+RjXssXM=N6P$j=84R>$qKHl!Ry^68G4#n@XSOW>r=}2R=O})O( zE1UnCKL0;N=p6Ifdw^N0K&{#re9%2n%!CS&(mym?!BAcg3SrdC%ZqX$Oc`Q#Opx66 zx09jK+ncp73iI&XGcjgY%Bl#S4}E}?j!NO(_iv~zwQPnN9R760$6kPWZ!c7of1FH6 zlVt+T$T;3>Cg=3RMhA%wckTReE?l$dV(w%CYvYaQ4>pYn50C%AfMcI|ukTohJEFro zkZwoDXl|vc-{72RqrTnO;=|S_TWWfIi_7XY<_(F#{7Bfq9zRjY7_8a4AmQWIxvwT` zE4?ziZ6H4>DeP|Ue)7;1a_MqIVNZ9-X77Qw`_R3x5D&1lSxO7)=kj0YUZENp(Y>jv zs;i%Cc<6wi_4AQPvrS6fX5vdlekee*tbnglGu6YVSi%-NG&cqYpyAWgyC;~k(<{Nf zIXp0P6_^Dzg6~3?rV#zxM|Hzy>m*4?i9cD z^Ixt1Q!5Yp-0Df`($K07-oMp-Ufp&wd2|F zI+6qp3e+0-(*h`M3o_qgk)e>;G@~X#^xv^0}By-=NKC7CGLV z@r}LgmF`iI=_jrP(I9@>_v&^ytn*8rMESd1dT)+OEw-!@SUeO|0R1+l_YR1LGE-j# z5pns9JUyG##YH5)Wp$mp0i;cA?oYYm+Dm2H2yoZE%_*_b*=E@SiTma5>egUtZ?N%u zj#gWVpJkYkL92Z{2*fTC z2>w`Qq--DU=uZBh%5#9*9OQpMdwpBwpni{}n1D8pA^wT#U9MY;eSJ-g@nAC_63`)w z0g(aw=i*1h-~7z+XqxM@;c+4dbb!p_UyM*r@`Z#fC_9!=e9&ZshaNu1#w3{3-}2$H zH%b+DB-~rpk1NiQq*^2-LqzAANiBddh>B)WA?bB zQy869ZT9AxifW@7WGN*!!raP}`~mC|>I3b+vcfML{ygJp-uqWY9jLW%>r>R+aNKWF zR!rQTSqZJuVvGMepm%8UANMamI?!YPQ!aXt@U9V6FVQYjF0zVR8UlfB-O%Xd1KN>{a__dye+tF)~gTVXWkE{rToz_AX!iznS7BIwMNyr+r~P zm`dh&n=D3zirx^D*7$#FW$VO4l!etv;=3rJMV%g4*i@$vE)1`1>J+oPu1TdG9fTaz zb7<#DM`;X#;T*+4RK+^54nqDJ#>b4~gD#_;YjbPMS%tqm5&kv2Z*D$wq*>kHvqAt^ zG(K(bao@~_q+PHghvS~4slr_AA6L0X)JHirxaw2_!6@<^&gm3#oSSRkB~o}}eTftd zF!i>4RTYg>v{*Z?&4Y{<5cT#$;+*W)APgihzpLKOdfRtV|Ap=Y1fHp$#P&0JR2?ck z7H2VMi$+-J%A#%~@SK8yd?{~Lo;TW{FFb%Cu2h1&c${%C&0sK?R87H)@Py0c zG8EeG&Q3|HKgG8yqhAA%;_0VZ!I0$1{D8dkD+?~86!F949_9Z{#W%Zz%7Z^J3<9b5 zRWW!%a_HHPzVu^e$>_CxH{yQfW}-57^nwiB)|Ss0RC`uhMv-QGlM>M8dFW8L$g{~f zfWs$xWB(tTyuf~`Jc|^W4+EQ4Zf9Qla|YpvEvyZRT_$qF+e|{8oM&y_w>7jXh;j(q z&_-Bf3Jh9p)~8bFXLM!NTf(tnIftJvb%O{$tmvL%_ z6Qn4b;oov)-z}L;sBS18PSBZ{IpgHy*gh;qMnZnD(DJK}cVFK|9;t4cez>fnnQ0(X zjeZOKPiEy=>R)beE%hdSO5(d0Nl9R|v;RAAC~Bztk$T z2l6 zLIbD^pL$`UK9T;?054Xd5A{h#0LJ9Mt4%g=tU3q3|F`kQ>#k%~YYcB9?6SYdSluN$ z7?lau&_gURUYpSXxW){SGOL#nu@0%=@iOTVSfJNTC!-1YTmHn}pS+KVAC>?_ABRQy z8#dAhOQ|(xfyu_}ss-6-r-8sMrl=<2-P@PCd;LNX`6rufh5YW%8|&XW&D+7rh`T2^ zg%gUOuw#d6{Xb4jlM=r2cNA_F&q^Xc&dRlwj{+`S$&XmO!gu%X3i$ah>VLq)w;TVd zME^gVD{|8VJDtngkASJ7*?;c2e=YA*p26niZyFAXulpQtez$uH#?#4Y!K<|v%|}3L z(;@9hLU%+l=v?RDZ3Hg7mBpERqcb=y4D5=X%XQ4hwi!G!p?{ z4@w1S=fCm4Y3qhIK~Kj3c3pp+NB;iiK^6DHmo?6q+|5 zJ?S0kuzFiECPVyLEx^jQ2CvM8N{^g-TgRxovHE(#u;BXQZCn3E7@@%`0eJ8%mG4FF+WC?GYot>EIfO z80+mgBNx@XQiWHpu_GL85&gPD7uk2p&Cawh(q)7^C3Kw;BTxm4lzGhQL%6>8{y+Kz zcuz#ABqSDS#Gx|_VCSGr)=)Ce!fj<3%w|GYnauNPsL{k4lq9^{6*FjwO$n+$x9Xg` zeNSf&7ttB`eql8(Y+E2%q<^ahRJ_j=D?|nw`2O9>$MoVRy=QaKv@^=SN0*5CK^ z-&#Iy)WTdYc!=ryANn^bP`_hfNtAga%Pm}F=OY?JLOWd&%ws@okG zaB{Cr_}%!h*q_cCxS5lJxa_;q1}M3E8wyU5u!n!Qa2Yu>rCv7OTNsPa24*?CEz!aR z4ld$<-CP3Hz^>cpi%AqlyrC|UNy%9;@>F42qu!FMoq9fskcMN809?htPu-i$?^ylW zWCc;sa6H<*jx}X5ap`g(Vk6S~{md!f!FU-VJ&!gT~ z#^X+>9AOzW$B9uPX_QO>%&2D6K}>oP@o9jq5BjS!JG;o|S?b zXCqqgp0~zv^AALDTpr3neufi$djBGn*c~R zf!TnSu)T$~c4S@dp6N*<bBu8eyOo1Fy1IfmZ86Hj8ZWel79n~#b^>+Qqh@h zuko)9g?D$o3>epHgmEZ=TAvzrS*_?r@zDHQIk98jZagkVxoojp_d5f)ue$kKl184v zYQbwD<68n)_C7p^gHgI2W?I3iboC|EB3ZV*Jz!UIuf9ju_#sr{YU>16sTMH+K zoAUcpe}j%&IEj~p#1>Vwj1PpLV?lKau+j|2cme0iP{YY|){6_k3{99u%e?cy&I6 zL3C;&Qr_T`_`L6@V<#)xR6G!nkpj=oQdug|*0+RZJ%I*rhCv=l< z>|y$TpWZ9!I2feMIDy?(4eAiMbAHfE0cA=? z&%_1z&3!N%l6ucc2={V*h0Ruokk&_(0)M-gl>-p$lI0E2F7X(6xv{SQ&<=yR_(*?Z zmbN;SmzzlfUN;;}Ki(vXF}s(=le|3mQyuYrPemg*;`|el;e282u8%_(;LuSz<41AA z&`6W`%8BE1QO&CGMZGUF+=JH~y{qp1hb-R4aEu`CpsuFspCp#Qe&-Fcl*{-Cl5so> zy%H};C(MDnV)<3LAy-Sq>+EO?Z8uEg&|DB_nt~t9;_#Yp@oUlSkAwazzfh)i13t#@ z#;kWWuwgLx#Pij)rXi`vu?pr$yln!@kO#1+x_pzV`HT$kKj?MBjI7`jlva=_d^5iz zl@qxyx&jUyT+I4&@9Eg!G~p?J0imcf%ou;OZ%kT=&IJkda)4xZDac}Mc@lbgOOKuk zJ#J8}UvM@9KHymL?-v2y8$?e*nb$wS-rzkUs6Tc5S5U{ukvyPGLq+qf>wG^poEh(@ z=@?Z6tv+Lc^hzCDFDbS1p-wB-nr1fc%K1VSB!)Q0wmUm8xha)D0_Z+@VXG1AI!m3Fp|<7Lk4e(ANWv0p};&*dVVy`;jtFo zlZXeN>VUY9Uc8_pRaTDbB`l4G%EThgoHJA@lv zwZOO_v|gFx(Knr$Zs~kTMj_!*Xe!seeqF1s6&3HA-3qikFZej7^oky&jg8W{uu>7d z!Epq%Un_PD?7b?+wjz>}L==tj9YqxRlao@WAIKM_!mmYaqC5!iemvj4ohMJ`I`DMM zNCpsJb`~#WH!Pl{>M(&G->6OkC1P&)DIeX{r1bE#MKfXSuw+`e`5J=Ymt8Q-OaC`A zrJ~EP9Mmt@a5mE-J^nE{J2LrIJ-o3}e}j7b9#l3+I{=mWcTk7POXc=xqAox!`38_m z=@{8-$6&S7BojP&dsO_hv?K2M>7nW6>iv@?i?ji zP_T&Y4xQmK=k}0$Jl+o(iUe4zPhoz1u6${!{kMTc376s4c9sb}6z=O214C$Vr3zTy zIgrK0xFO&4kV>`hU=KuK34%He;mh_91F^%PTsR_KuhT&aadOwcH2Qx?B5>2W#VL7( z=}7dF4Nxf-o$er-jmnL=nxR#01JP1kCgyZeYz?-lVOXS&&9orVn#DdC)!|YI6_%*( ziDzuNOE}C}h83LF(j`EuYxQCS8avltXTjVnq36}hYVcJ6h0dCk0pf^)(@~~jcNk=YpRKyM`qf5<-zR%46pH|* zmc36j*{Z_nPCSZ#U06p3RH$Wvv;=6d{4b(dq{kUXZ}vNM8&BntiX4D3I71q_g*}yd z8Gr;Aj`|mSFJ#NIyMl@d-9E#E%tFH{{)5%M97-HK9BZTDhrv}10UwVayA#;#w6far z#--`h1uqL2^<)!$Ob0$MWnc~SdbaDc2+lEByF%H8l@xM2>F3ykx2fBD$Ir>3Wc!Y8 z{{CUnh*U~7PK1LXm2Q^2DpuGKk`7veTINI)yZFqCSlHL}!(d^?8|@_w?r1~B3bQ>O zfK=!M$=35h&dG_jV=F4TPKl>|n#k;alwQFMMCQ*#vO5-0F?hNF@F2IN^fV}bHvqj| zG;%Exor~Dl1@FnF7(A}voR%+DlY7qp;RiF5Si4$C4T2rIqg>PI>RUGbDrnB92o+1$vwmpP?5lpm1CGDIE7KxNTf4+?kDy!q| z&_I3D?4Zb`d|(7{*l&8j@pj)8Oi*7y@uoQ1`s`NiwDAzVWs(b zcI~xs1(z?WAgBCqBHXS>ecN&_;Be^pk5^CVsmpt7g_-v5$CWn5dYijtwqx9KGPo=p z#Z}Px*j1YU3KkMAKH(PL-58KW?VLqBGIKX;U!J^a&aylt9(uKAN`ojtNkVz1^ zBwjLX>?seqTKH9sY>iJBg9>v3JsuSCFZf*Ta1S=X0e@$!_oWhhK_9knCFzQx*#qB% z|ETc}Se3{Y$-RMGb{d)n=L^nhMvB%pw^a#=)@`OF`G^~3IIn+x5Gehb|}L1p9e z4Q$wU`XQTu$qN+?-3Yod;s}$fNI6_wji>&{_pt?Lk1p8$UKgFVKj8 z8hg%|GtiOibuDxhP>q~>rB((Ksj-Xv0NCk&1-r;)LaR@){F-0%j^ntkcTYXSdY`)P zS$eQTTM+bS1}x{XYtuFW(>J486ObF>m=SUq=Y5aK_3@%Qo+gTMe6Ay)g#Jf<6ldWU z-0~pY+GzGA9$N>;GXS(bM!~1)xN0#o#J~;}HiWweYj}NLZ*5(#be(L>{}ROTcUBLG z*!io=YD`y)i@wFHs%Y!3rXGgem+;M_{nLo0Fe+}vnL&H@8&jQUqlxQH* zjB#2&MeaJ^j_p34f)aadQ#4C>>#CSMU5ZH4a1<@Z7p0_uM`aNiOo%Tn&AYe$CTLze z+Uhc)Bl#@%P#vwa{Jdkorj&acs3M*Z>t8LdPT9JT8{}5?{ILl!!8TPMWSenPKi~zj zOPNZoC4W3qT3}x7s>j5cTPg!K&zB3|q@JO)D9)CxyFNuXFrMAo1Ee88T6=BQLyf-u zPRioX7Sc+~uE#VwcXBik?5cEo&8vMTUG`4wO{m95i|6ZcF%{?HI+o)B&&y_PR?NuD zB{0-|Z+ZU7p%UQ!)=XhiH=2x}$}g8^r_fx9R~^dr!<6mA0S>loepyA%!1dLRNLMg& z)~&I!^F*}|dX@%O!1lq-{9>D55y9gJI|rjd^hx(Q`*S$U7o*F&NifLfqaaCY0;1A2 z_iU>ZDQCwFfx(Z_fT303{^ox&uKqYw+&~}vY*7jnmHdSMe`Z?k9P48??YMXHMbX!ZvW<$rx z($%zQI8`uhspK-XiuVU0>-#TAu`kP@D5k((TU~ip$zN0Gm@Ve? zV)GmutvHDYn*NU|;a38VN!9MKY7Mv6&zl|qTeX91BsXc5pyk=~`j`t8;OhB!3vX&Z z_mfJ5*ycSU?^obMEVGUkUGqbskZo%*>*-3)QsN+=A6do4-ev5PQAJL&;G=x)R8F?F zs4TCV&re~WrM}1yR34r~W)%rbM8+hb(OzAAh zTwkiDc|ZCT?AMHaYkz<~U{wu}p@LLLpOYnj@H;kZG>C!GTJ-2tFwnfa+3<|XFsZJz z)>=rOR^xQ{6Q?qy>I6LJ>E279{65^19P%mHR4>}I3(A2IlIZMNi3uIlR1DZpX^w)g zpZz6e)r@!StjKeTQ6dzu^er+Ij9pg-Exl|#n_Q;(N0g!e?f}Un9QkJm?C|$c{sH5X z9*Af5e6|}{%KG~Xm(N(IxSi2`m|yReBkvEcxydNt(npJUD{_9EDD63vfHxcSO%jmd zdMCMDVv*e=(JWs-bqI|QPw+S>1ni+4pIj8l&fh1{@8a!e=S%T3v5p|r#}Erd+*F#Z zXjU_rEhhV@ztB40?#V0=`dYEo$`cC!c{G#0*sb8f@`P3R zDm!=_STB21ru^!D+}0s0s$FLV6j^*lM-d(aghe3jw-7&QdzZwY z8#Q`qPyuKd>(Odc7V5uyXIna$sL!@s5ePy2SeLVJWB zfLA+3HNJ>se%oCU3!CHCnwjOL;W{gZr{K`5OVHSxClZ-I(g;|-3MRHut!8vAkO%@f z0@PFSGqlsg)m~|FVp*PTJfy9Upfr4nbltwmrjO7+`K8}mSr9$>c5cqb9gg-9XMxh( zn&8uOW1{G_KDFU6L8D=$u`nHpzmD*fh(nLF4ZT3=8{T(V8WP*k6%2`jy&1re|BIWE z;qr+WxbigpKA@9AAivSSj35(!%)d7dYEoRj>#i#_XIjOT_ILLeFjh`nVHQ9H3_oI< z7?;`o%SrPc4y2dt1M~UbedQ9E6!_t%YdfSm7+NwVmFUjNmP`BDi9#eut9e#VZ%D5w z8uw-~h}Gn+igv8P2vYq4ybO@S+{(ZjUsI;q-_gx8&%euL$ZsPEfHpdvx;>A^;PllP z-f}+=juX9XUA;5?a-!O=I!+p6v84sFVQW1O*1rgDSqh^G^{mq&%KMDBw&7%)zm#2Qr%c*_GyU9g?L4nJ1g8%#}7 z_>=pzi++qr0-NKe#u=9#*(fEyXI)_p^4nx1j+!N(SWSKbVegVQ3}r zBUca5RH>3l!_jiqJ@PsZ-&YpySRYe$m(gmTra?%jzK7B!Vqi@BLW82m_eD5)`B5kI zf{I(nS0^Rjd>UjE9lNEFZaqT{g5%y7O8#jSppNELw`8fqXtl{`cBnpI_h_%5zXEWS zKm6gZCca+qE|FeWhw=KB33OX-!EyxSvhdT1Fs&+ip*Et!m;Yc-qH=rK>Bc|d@|fi6 zxp=^`mF#S_v}N09{bWuOlg#C{iNuV+E#-qmGJ5{p_WlzF+r4Ru-s?J;@lCmsPSjD`(2g6eyNd6;G zYZI1JAT?KlEunw>y*n4>k-=3?A!`&qsw*7=S4AVM;%H!0G8FBfUyM~|KD$6ULUgV4 zow-qG9f$1A&2ap?r`FbHK|Y}&Ub6tFBW~O_ls;jyl`8{`u zW*Eh)z!`zX&AWcaDGYPK<(l2W{oIxh@1&so=+G3Zw^#a}gKFee%x8;J=3_EM)W?68 zhQEClH03Ve8O7?L#$ZDSw>M_8_+p{z8nzP!sHm~Q(t2-0XK~eqIU4<9KH<#Uikf;E z7T}fJTxjQuMge5V%|Hf{OO>P+TZ8o{1F=B}v&J)WPuTG^+CA@$+`Q}vMi#~wd(Y=7 zi4SfG2uuFJ2CxViGd#DD(j>AcSRLkd^i-dZQCA%7EgZ zY8I!*6q8 z*3fsz`l6!YzJ>CY;hlAzu}m!VT(EQwC+I1VGqdrec75?~tecOCP8E-8g|#|oNhsXWRQx9o?E~zwd+xD|yu=o>I zdZ>PkeG*X_D;@HSYFapX-4pLCRbl0OuVgdXP~!t)y>}1yx&=+6Lo+zO)I>gMT|c_~ zZ5zyGEG+1sAR*twf9Nh*H9{FzAHE=jfJ~LsK-53kCVv0^z>kXCZwA*)xUP+y!R*pC z00Dt`umVQ?SxvtB>T-a4^LB>?pMW)0R1jusec6A{ZJ;q$0sbA+W(DNI4r}oH96*db zY06*)?XzIpl(saXu(o$tV<@4yJq8nEMxJuYK%-YW?$~7Impgav%nsM!(iD)1OU4tl zLR(0XWHdRmGuD@xE88V)zOUxqgs9pXqrWl94=X z|J4vkI!~WYRxo*YA#=ye8YZ3-z|1j-L~p_@A46|jf(e^AN*yDLf{FGc&L|tkO1BvHO)WopusF zd96?kbIJ5OZ8I8+&x_``np<*2{6OelAMj@r#^A8w2`}N1DdtCPmi6{8BK{5w|KT&k zVA3&6bfa?h&0HglyMfcPKtWdoDx>2^@V2zZxMWRx+<07a;s9pXNrBc?b^Dheo~g0l zro65=G;5rzh&cyX2uJqfwLbPVV=nldR5Pj455FLa_({2Xk)+Mgz2jv|m(%nAeGQ6E zca6vC%fw^y#~5(heb^)^$Lfffe`hT6pb(oVHkWoZ{zZv?-MUxGn1?G-z(i6J+oBmt z%*r!UR%0AzS<>wuH6Rd`r4t5YHBnCN*orMs#~%2%UCuupL39aYU{L$8lJfn_L&TWg9>lNV!|&2aCq zNiL90-qp6ys?2Pz6pJl+?rVj*^i}b!q4Dg=%Iw?#Tv*!-y^aqN!gF7M**Oq-!%j}B zAvU&_0b3bw9S&QUr7BS@$rILSgb|*%k0Z)LX0(`&cb>c1(PnJ(lXtM@eK0UMAHL9O zuq`2!{p(!n>rF?Mdvy=CCQ3alel==TA4_TItoBEBQ<@nX%b)){V<{iSGYeMs(UOKs=Kj@LoC9QmSt9yn8dow(IH&VmFf= zGCD_x=yFTdy6e%vm`^_Eqt0p##QTWGxc2z#`J{wSgn50f2qInQ3JsElF|gA|7-35T zhe+aJ#|>H6|Fz}_&J|9?G>ox2-c%X_`5b4%5` z>lHsGBi5THFQRg}!H;9{f2`qFaL+iyzURty9q9TRe&ai%I%g%@AnDFxI>Pls=H5elq-$-O79VS(~WXf=#1cQ(8Yf;0!249A{H=rw7(6Zdj!i z^LjH8lwG(-mFHiZQIziv8){6iiFO-`QXk(Lhx?r;lviZ3%kQ%I$+fcd3O;y6ck${& zqiQJxMI~(Cp6~=9k9?oF6}O?N=}63~s}Y#BfMD+sdo?(0jKT>Mb!aeqW2$!z*)#drgHMd{^w(&YQtdimo(bMyRI*E``FAQtQd1OynTJQ3)&Gu za?AB)n3H44&jt1jD`kYs3Sor|D)`6egC!QxoQd*}FkeH5PYb}l$hCgT*PDPD;Q-}) ze{QvCU+MQF1JvK|(0`x$v~Sjyld=irqkJ;eU2Zi+DdCkV=0@E5rnZ=)1liWlz$~DC z78U*JT6dvz=+3OX%4c=|q{)(~E@sJq=Jf!kI&R>7dg#N*JNF%{{hmaQm>W>WdmF^P zvpr$U(o&(gTSILu#C3XYpoP4|~keOnheE^qJ@m4SWgP0?ulAN8=%VFl**XtDGQpK+1LSj)f~ zJ!)R8wfPIvQ(m&G3x_IePk$+3=#1k(3S$*VRg!Dxr=07yYjgRZggOjY0ulwt`{iCi zu4g(VC##(m8XeyPN@lZ#C0E>U_C4isn~!g`QNZD2W7=%QlWzF-wAbK}6|kExCI zO^Ytn+SdaPj36p1@-^ak`b#fOqQn|&6R`@d<26EFV`HL%s59U}jjs*;T(M9Abn*T^ zOJ;-)&p4d(WW=W)Ov2Z9+*(;(@NLGt{0q;ZZ#;2?wh%g7=y{jWBOBS{EqO8N2CDZR z!O_!rs~|Vn>q}KdM(5x#!FqNLK~A~`8}6D0|X9#P+=IIk(cw4VP!5VBaX7o zF#vQIwv0)!G2`cDXr#5V5%lLEL?at~<_pBbu^!NTT{Q~5kz|v z_=m@8bwm|c_q2)hSfOW+43oU(gNtns<{MttM040bV7re_>V_v_v+T)maH(g5OsaiW z?c=SYGr%mk!T3!+6$N&X3!C^-zqgbc{W`md!~ZFBf|#EgwGCWMR(HQh>M{av*VUP* zSXq5{7)+}0dn+NNN zisc5!%mzo%83&k9`AYdpX)8v`<`&USDs}AAkWTcJnq${Q-xY@%Jsx>8Njq`>{iURXcFt2#<8dw>457CywhpLOn)u_paO`WAWAE(kv>Rsc5Xe3w-(Lw>S1GZk=7`ygsM6Ep=Q z3Qo>OZ?YJ?OQUg3Onz1WBNU5zeJ2h@lW%){BW^x#zL$>bO1S?5{5x;EJhdf3a-)ba+gygXZ0Ry#wjRGJk~G$W53^^NTG+N~N|Y4t}Zs$LmZn@j_7 zNjGM@Bgs}qV7l>e2jKTxig-A%phcz`U5>0Xf}t-o#-hhL`SHGL_a3bxnM6=!)jJv% zaAjISE+9~u^=P~Ce&qHdf^LejcnL?alvJ5Aks_gQs$fkmS+R z461IE^T;gzo_6$Hl21fU-gWzgZj^C!9oSZ5laaFZ)o^8V*-~3|r5tngFOgREC|uV9 zOpEF+o~@mq3(3RphkB_|W)PgiQSs@qwA)xavAY}Q?BMtG{7>u*5-px8tvO>tURPE2ccTl?$hd&NKw!0$eVW2;9PMz_~ zk7Jop!-C$n(E6ty`Jc^OK|#&zViS9S-wQpKisHhD6;-vt0$rIQ{G7|0<~&FJoPWErP= zV)1Mt2SB+{QXZi{!S;x}n^w7RmH&zc_kz`jDQxFoEG#?OF55BsojpCX0;u3frde+E z3at?JwEH$!lxLDqYWy5sH2hB_@^3J8lMe`~2n`M=Si_e1uFFK5SLTXlvarypZk^3S zCCot=h3BEux|i$3?EL{jU+0vF@6lYF<1ct@nvl{?GETz9u$TqSl(?BLw-I-1nv7~TKDG}I{lY1+Q+$LLp#u5%n zY$=w1a5>j?Yth)6AH%PnaA>`8AcsTZj>AHCzENFRG}z})s^T4+O5-1X&%~LY95_7G zV#q&`!_AvoR~vg4I>RaIY-BSj@d2Y`EAEK5SgwS;-)%% zx9dt)JnnZ@Ge~l8dyjvL;1rH(Zn~lPaxu}=`X?fZ2eFew`(Z$H?S9`f9j~zP_>-8b znC@a$F-~$7`2i|HmC3CrPabTF{q=i7n#l-kq-M<+?fm{z15etAUt@q=!rm;H(HORa ztzG|YpSIdtRndR`RtnIK%j#^X;m$>=!=P)})E(K)9Y*S`S5B97%dCXg#;S!N=Dz-S zL}j?0vQBsRXGa56lek^(&ae?Q+{%ON=GqPV`=CIvsl_Q`TZW$R5WfjnKHhFkTWWTs zr3!8w(blfL+`+*2cDxckSypp+^u)`gwVL|45QyIFfmWz}yZm7`2{9s?$7H zKwKrRJfOW2QXBG=5m$^*2!E{_UJ0!0P2#m9L$V>RYxt0U2h?f;i@fRo`g-oT1btUl*1N!Wn|V#0Se8fg47zDJdydx!ET^KVy?^2Yz zYJ{Yis5NO{)OU=R)s2}7@-F6&1T`ZkkEi;J^a`I!K({K1e+qlcY6^x9(K-({^?;^- z?rcij*67$2h@ZirX3yb7^zwwoM3b-;17pLGja=;lgk0$ks#lVJcca2PS-{lx@b)%( ztu)oK`@z#W9M+@y`HP}35&HW2WxaFyp-}Z|@M^?s>pv5#v2cxKGPB`#2Ll6=j_dBP zXeTcFh)dx06N3P0ppABNPFmZUqU)@$ZkbKG_nVCdm%5oMflt!+?jz`3nu#(m9GazC z5*3x){~KPpH3!ia(foIO<#_FNqKh5P1%~|T`RVuVpPuWAfZN(k$;4k^0? zFnlbOV4qGx=-X<-<9VAqJ~=M+^1S0JILyzdfqwq5XX{a04lQ@q^G1iyN_gaFsEKO3 zc3|EU;|A(qDY}BW($3B{7f3BuAloSUe2^#PB2al4&^m4!I4-r|MeT$Aa8f7Ce}=pg z2KwXs4EjCPP3mgGc)Ef^T%sK>1+95MB1hI#}I z*;dUBs0a@fH#h~f;*YxqHg^34v8}%~h%WX%RilAxW43{KskEnbnRZMKLEyqjzJv>A z*7m;H0($B8ic~kV_SC<1l*8dSOY_!9YU>jAxnBe|4$3{8Jr_)xu zZ~poIF5=vZ56>h4wR;@z+$V29eC?c$O}Q(DS>txnL2=4ssvU@_9riJe)1S_JX6>r1 zgpAYEafaEd_iaBvE=?CZAum#Q7!Vi)rEdBeg?wi6*6Kl@rqsKA?cCRS>I9F;4qp88 zkeuOaV#~*w@aotoIEpp*1r25 zM}2E>KWG4d=^$|37cpZ9{~_;7glrCuiQ0X~^YFL2nff^vi#hn7YX~5$V&&C3x!?jO z@Het^IgkTvmTdsYIQ&XVVJYo89W7b8Yby@gp`HU})2{euTlBvku=D1KA`NtPrN>bN zu2wR94V#sjiIwcxrx+ZHx{bLUezu!(+*c>Kh7Y|5B;S8C)Kngd&GYFQYDcmJGj;%-xhHnk_#Q&41lHrJ-nbipgGf>e$zcGmdA&GxuR{{{XyPS$!&C ztUe^*jbMB@z4mq=CiE@Vg?fjULhI}+ zDXR`5&8(u2=_1(l!sEu~O6=J;apAJfiban((JZ4gdNwdND_IW6H<$&{QC9@Jttpyv zsS1wQfj9*%M&ZlSpuK*MunR3`K{Zof-h$M#r}k$znXw}!LVmLD;LOW?F)|7{3TV}0 zi8mc8*=nGXH8P5*Kfu0kmT0e?Lx;NrBxz}qwt{?2;K;mfBS%<|kTvIYx;5>NCj-=U zl$~4JZY-DU^##~1ZZd|uY#XvIWoQ3SLhh8_*P<->88?g+{q#%`zvArqR{CeZgg;+H z^sb6aYNM>~9>*t*H4POmG^_WrB5l~=XU9m?891{B5X_Lws_3W%p*$apd0=K0KP~@m zkGJ_GGXvK)^x%xFLDg!r*a`Ujng34q{q^~$v5Xo;pxqXuZTTVqAVJ$4Bzkfm!n&v$ z)8@&+tnS&HK5){sX(-eG*ppO+uKmgsUq$*yLuco4!;FSaOXRmyd7YSUL4908TMJG&?sy`Y z3-`^?e&I)_mXTDipkuLE3+?72VZ|$u$xO461-sPW9$a?$_LkMklB|`KQ&BF8PjV1F`?N0Pa#_kWV_2$r!RYmHl^>B|5Bn>>=l@@Fv+Nm?6eua_Ei>^M zC-2K@@g_)fMC%_@m2%#L+VXy?U{Wk|Wh+UPjtKV=|egDD6b7;>>0seq(&ctL7Ew;rqWWr zR^Aw@pVZiKWu zt_PJ=Fz7d6sYBZjj!)@tBOqFPXf z?md5crivU-NIrt$rRe*@=kyH%g3#D)bM?CV0-MZa$dUbvy#SEYC<&S^AhJ4WHLbj{Vv3t29GR z0-bb%1%Ks)@Oa7K|CU07RYC=Hn@L;JQzllq`RT59#esN5waa!Qu z{i5pOP|eIV(kpYi4A4w13&tm9YW>8Givn^tQ1yA>y?-3k5mi;i9iq}hO_t}hRvO`Q z`~PfVv@G15^Wu(Bd{=H4;K`s*9-uyNov3FSU5Omp)RIq9gkiv=Hxhwuqj7wCS>^HOcwf-&7Go1}Sb4}>JL;L2u>CT$W#e3R z8p6(*ifAr5%_&fKmCTXSWHV}fw%)&K#xe4;dbzLC@KgYKkUwtt$~}cE!##rZy3l!^ zkMuRaZDK-Pc;PDc7$&rTx#-tUqiKx9f7#x!g3|x&pyd-SCGR|7&p;RCY1FOYx{+(V zqiek#)&Oakp)68T9b$v1`L2R|vHgrV-sgsY4X%oun=-s2R!P!wr|%a7*%LAN;Mdbn zMx!yCnP869%8(uBTr z>0KeYrphXuV!bphww@_IUwR}JWf8_qljbZ~?o{jNCQh)EM_R45t~8>2|}YcG^H)gEVFDi`+M z<|d&IF~;|`H-0B+Rx@wkAJv)t z=uqwX$mk8t;tQgctn><~Y^R%!;mO)r=M32`tO-ayV@SbN*Il+bwd>B3CC*0r zz-8h)|5++9 zyG&5lV$|L1{Gr8(c5__0DsTbV4a(XQ?44+=;INd zP8_Js(9E4&F$Nkav7e5F_1w)W7Ut5VA4h;m%|e%N1sF?YuyszP(#bM8F@YIOBml6} zKb6A{HV*=@ny|T5ZedMnZhv79a++oCzeErbIvZZq(Z*=q$bO?ju1Wv_*MBIknNG9n zKYk}6mI`0@N?(8Gh!k@}!HNxBU*03H2-;R8Da6r=Gol#VqLmWZ`J2(J$J$%=f7!vI^Z1f z!C|-RCA_tnEXkiT`28UO;1xR7Tl8Q|w=FuId^op?c4wQJBc#_r8Q+hJ2iU)A3HsNB z9D~i0q3OFyLwsqSwdE^hFXnh;rkntwBX=l<#%p13zUcQ0)9efBBabM9-xM8w!{y_c z?7Ay8s(4eZS$9R~rnQAYl8?mCi(4g}?~LfqLwK*L451!$omU<{gHN^Zo(GI-N@=KO6{x`WH*Hvvcuq=I@+8`FGg4o!%N9 z&BXp=8dXO-H@#PF+A?^!I+P~p@EiUXqi42^TtL^J%6k*Sq-f9v{q-Z-1@3m4F452T$aavg;?wyj&e-h6D(WN&J!4Wj_{W`>h; zDe}ffTWufEc9=R9yzwLtGiE+HY_L5iosj+vD;P7m$o*?mVIQn=zs1N&CKw`BW?_sz zMTIH^Cr@P6pjM69bTQkyNoxvFzQ?D#T{b}57+>{RX;JZ!&OdmQ!1ubJ_6Sv;1AzrZ@ylN962j*ZS%f-uGC8c&zCTN`|+Y^~E(YE!#jR8CrS0aqZQnIRR zGwV`N$23yng0vny@t_8xM`C9K4j7kMW<=l4htufB_Xr9g2Z~~OKJGv#0PuqXudjZwEJQwud%b{r)R6&-y&`YsHZ1+ zMTGSg0gi7o>ur+}LKY(*cYQ@y-5F7Rv`niBYYr(aQMfP>VEOvunz=fAJfzwdg(H4@ z`|1g3E@K~2Gk(aFPub+(5qF2-SC?0g^nmJFnbgk-QuX9ZmpfLU}6;Z%iDRLc$3H42_Hm|4hBhACDTmXQL;EUaN z;O`FC^I~4wPzcolTeL_Ed69{Pl`gqti@t7hG(YOcuB03bChTsNpVNAP9yK!stVQ`6W2Y8+QlMuK*V>jM4wO*CC>hm7YK&Q^`PT!1 z8C?3%=2J$3k`s4FjG7~vakKXnm*R=;@s`_G5DzuyfjFFu-uvMHiavvjB?cXOeXS* zvKOoo$Ts?LnaHGQk{>h`QdGpz(kMWSl6xJGS@*$sOQ+j*-Zu+Y(R@-T)W~g=?`2Ri zMRteBJy&s-t_@U`VedgBR>;tJtH4UctJ2%hIa2yR?x)2;Z=q{xS=4k!&^EyA zBc91T3ZY_#$>JVk@T-c;`JNb5fzDl+aN3wCcogaKwRO8%n@*N%QaU#njN7gepZ#H| zOIX1?1k>Sr+9-l%L-wmpH#O0FNirP}4FnUb*dGYGAVn>VlDm zCQ6kT{no}CY;gRhJ5#Mc>hrOC@3bMz7$zWOU^)YXx}?+du2smYhlg`m(6 z2)$!ZR6!;Vxf06l@F`qLRhygeh?hNOZRxQ)Qo*5cc8CW0JE)=9Tt6bgy^C2@HK63!gL+tHnfEoJ}4s4XZ<_aF;W^^eA;s3_vb#I<|%j74Nrvcsa^NOdw^$K79k;+&E=8zJ#mIi<&xJeZa@D`x z$r<`yHiG;!PiiDMxT}kmkO+v=8ZD(3d6gT~EX=J50Rap99~#oQ`3W1}zuG1vGV%EnfFGN>wH8?u$kb`KI`+MWtvidZzbi&WW z#RBfvyDW5d_1d2v-yF+!_!p$d52kwTY%5K6hv{Rp$a+epHI~$tV?2c0!HIx!7jm<; zR;YNq)WMcyk*QOSVcRsf2ZYUdx{TVI4oblChP@sh%3&`XqvdomOaSQo(Pg!7uE%7; z4acq%E!f!u2VTQVw+K@u&bH&aGPh@^;!kYJ2X~ey4{n z9eM{`0fVqeZUD&T&VzZHq(qa%p8%C%Nxn*#7(~32E^=n@EA)47B<5JQo206jgmA{M zq8}W|rC;;PFCKq-*5bEcF^|WTU6OZBM?YN|U{bU^k^VtPPAeQB-%<-`8o6#GV<1%> zxtNtyWw7BRI2-JyyL*4%GApo7DRo#vWHjPIbEk_ibdw&dc1_ih@gYnawN&{PR)MDAGONgklw>pM20(qpmd!ejUes1-~*+Os4_;O zoCE1Rkm;oR-88WU_XvN8-XGFWs{z1tXe*BGy@0npVf2k2+r$jWV9qTN>%mlR@m7-0 zk}Iu|$oZ0E7N(fnXCgt;Qwq9&iMgk_JMfS%`yP~x@=GNA*25Bh^@F$Z;#L!lDz>yf zvkiU-ER9y?@6M$LmkEMt|JpWA>`vPO7G!q$OmGO;05X10Y5*QU416L`#fUNWn(`GvC9SM{@fCPE!F?08z7 zDyg?uZ+iL=!F<4{PvT92+~!ILx2D!Fi{(-|>B}NrqW}Ez$gHvigUUM9F}gpnpR&2I z9CN&!qjn?2)EHr?NQjA;Mk2ZUW1W!ve!BMR&$U3DQz)v@3xUzbU7p2)<0YNiUj_63 zS|+k+qA=TZ02{*OkmB9tL|;^5vbgezxcn8gDT3>5p6D_J%58si8BEVEP^e7TnO#$m zipwmT11(y8d<4E{&A;DUAKOB!4|c zAh;`i?aPh4Uj+u8=EiUBBGD-O`^Bn9v6-apGg5MdCDX~wuJKtYINF04Z$jffs++{C z=F(tU7Zt?>9|Or~#H%FSP~HHuSMA;XgjesMhFXmR%J%5@%{21z^pVLLzl!4=kNJ=H zm1rKHa8N2RJ2SVxnzPl`(Mq=;{TZAXCgCsnCh<Clde)<$oa;^mb7qPAL* zei2Ocko785M>}YptxcU@)xa| zE+fgd2+BOC8?Hc*Ua4G8Hq;bl+`QGg{MTRBdz7FlXyk#1K7#5~lF2!EguS~^AAc3? z1Z~uo!buFQ0e+s|`ZDo@B_#dfP!%_eGhAaC{2!7yl;%@Q@Rt?($$>0j=<%$_4!?-} zavdSrw4SQINOr81A1EN&3y!0x*RuJ~rAx)mpT^A-_nN@;MClqvTy{gV z{eug=PiKs;b$l*CM4qATUVtWlQ$fadC&=qK4xH`a`lpzO3YiDnDREsKJ8D6DFOI|h zMSqRKfn1ODvSOHr3^2#ObY-@T~f9hdyC+q#5T1!Eho8gYB_4JQg*s;ZV zGh7rIwzgGYIZ=zfU05XJZQ>$?AgN${`eibDa4zlce)(@9C>3?tIqx*E*bsJ^@I zo!Z$9VX|HosbK68RHOS3lJTztU$M{^aBCcfNYSTaVH!1>wrn49(r0r%cd$5gC1aYB z^^}S4E^7vh>>lDOZcOGUge(G^R*}qZIwjQ|Ln#8$$PthvciaI>U_*(XiJ!e+g@6&$S?0=E z7V-J<OSI0}GKZ#wXr{BHIV_C%%WL!D#Xcw#j3##W z$@(;-B-b=eS^*@}QK<;2PjoL`Kis*&p$fLu@P_x4OjkrgKOhT_d0)H>Zp$im2h zC_2&#AFMUv#5>i@nexK#SU8&@do%Xvw5b!(eIpAxDLs`-+fpGdExd-f+YKRZJ6gQG zw*WL)J$4z{@zxq4jk5w!PD?RO$yaxX{mZd~O&pnZy5K;1sXFVQB08#jo2dUiH{T4E zH*J{Of2b@wQ&q{8w&o4c65(oV{d?oBD-*1}EC~PbCj~naE!rQ(pw(B@f;VIkAgW9t zd0Oq}vBh|`yfOPE(s(+90Hh55$Y1&e%A5Hx-EAvW|6W=9&UGP&RV)cIWMgR5)0iZj z%4}g{=FY(I@QYFpFZsVgFR09J#z{&1HFqW{Vo2NU&V8OQ8;f8>Zv`kZ3_nTvhNziwP>Uax9pTZCrzmka|LJbilFXg4lnvnJ z_78#t_RW=0h`D2BMM`d%PQ_gtbaAYJmuo^{YjIL7meIE{v#*;?+Krm zthVIXSO7SK|6vn~0TVQt)jyurJTXhb_J68!S@+&*CzKenG`vCB@m}QDn4p{V!C2(< z1vHFo8|#205hk4Z(rrgNkn{~QbK=50dR+TQAqQrbzWId?J5G;rOf)-4Dg;Bkmw@=;#MjkRCp+6O+wd7dq@VcxF8ps@E4 z(`NjBIk9t;@%J<6`u73#S9N^-uKmYJXkkX;(f3MGEc|*8_cSNXP1!>-y4M{(guE2N z4c%0j?QFd9P#Nbj^%)jajFR(`L+@_ikEWgo4kwdZO3JktF;O+krj2nVaFckq{Gufj z(o}Bnc8)vYFRB9FFt+PJeoGyQLUn&~U()Bm7|EWs+rS-<4f1O&gg*#TG7X+ysG$pZ z#>gWOJ?2j;v_)bG5QMM&I^7znz|C*+oRxevXr&%6bobudneMZH?2>y+iuYw9fk?^~!^$iu<6d0+ z!2PDT`~YGZy+2YbXS=kPmSImwp?<_nS<*8-ZHR(< z5@&}9l+y4uCqO|1qxz*X>*OIzFo(aVpMK8vbHgRu%7RGTw|nM`>!`-u*THP1Kq(7s z>W$kITu-_-(W~)b)B?fjxy5n>9jRO6`6}wx264O4naTg85gLGMG6vJO=E+hw<`4AR z%5MRjALpiOC~4*p2&i$71Vo71fdlm&K2X}EemqoRN>F5a??vop@EebfPOIlI zTA%g{vv%LU`67%Wm8Xxi+E3JLL7QqcB%N4O9FZNJG>#|jQ1J?$doRq<}rNt5V!-`Yrxth!@W`x|5j4&3$=N0kvLfP4gm$Ce8hju4L6qLaqZlib^pDz8y z{x7faolp_D?|=#dJ{UlJ`e+N*PBBEjtTHK|d;11_=W9w&Jim!r0wJoHsAvKjDCRK( zQM4m!7%JkIb;?Xlwr3C~NInynN{>5k(sA&)p3g)jxBLRj|6_n}f&&59xAX~Y)JLmO z_#N`Ts@E3Q(vToyEh(1cj%68r5UWui*I$hgnC<96jJ7ZT)@As|R5+8&NjJEy5ItB}#DT+sNiOMK@%}y;yTXbV<-(d)TN9MZRgL;w24NJ(Wmd>30`p&qFD{2; zEU|rj$V!K^;H3KK!%!eV59UOf#d(5fO3ED!HWVX<{3J5&tF-BkDc4OxdW!OPbRKT8E{&xVwZta3V5?JvIO~T57&D1M4;r zsQ>Vm<`9(rs}Lfc^zUqk)p8AvjGQclE%ol_c_iG;y`UOQNA@@%Vf67uk;)2Z$?;sU z1>C#V*(Jau^5%YiMXb5wbwA{-6*@sv@nxf~J3$F{$7=+@4e@?8BHj-_B_==4t}Z5o zmG-p~b>VRj+(Lb#7}F@^@N#5td1+8oD^te5>NUOf?!n|bWCf#3=j$Hw=eIkQf)M`6 zWuyd530oj>!{v6>RkF^I+Hwu0yFV|B)^r|%Q&kpEuO7l^VoCUI=ket!MSyB1Z4=y;oR|xCXM~BU?JQnD( z+mqoHUx8Yr7O&8jojDFnG7iPb=xJ5ZxJ;IQc}flSWN?N6XQzPGenW_I}16L~>`fAd~<8548hKd68335BCJHZ=r#7u#+4%|)zW|;;_U&2yZ^s{O z>G}*&0NBXf=ncD?8jM?JFi0VTb=p1Il%!^+9TIhpb|IDiTTE?t|HKH~`%ohfa!b6` zxJ{8Vi}Nn9r*@khfc6kXjlLUk0dqVh#`}ZB{T6TnmOkejQu;CxwaqVCeT;uB6HUkgVn|6%oC=;Tu&Y5e!;!!M%dz{@s}^j7 zp|`hJ3jbcPtk&fWx-(B9@C~%VXG)O4gUa;J31pfvWwj~qUaR81Qp_A z?9!hs5~>_-a(_yc!Sg#}JLzBXbB`mZI4Y?<4g6bsgR(>0-3#BQ?fwJm6Kx#pt6_cP z?O{kp3(z)dn3XiVN=glN_W#x4P!7I)$Xs|~E|}2Wi`Cldz<9$C_|vK{`Scri$c*iGYtoFa@n`Ut*PVL&Q2DnK0m4WrGn;X zV4xfKA-=?#(v${}BzL}FJOnePMY~mgHGE{R*a76MjO@ZpX>+-|+3l;};^WDgI8C4` zYiqqS_}Rh3AFEH*Zh6&>a^}Dj*g?hG{^dy8=prN2m)K&jT6Ycb{I$pX|4drlDGK5j zo~-;0SWIcqQ$SvS4OwseQYd33)GWo|6u?`c$iuXxgVpFmd920;XoTob`?A!Jr2sWx z$JDBU{4p@$P8JqN5&uezd3VTQed%}wQ%e(N*zsG^A>i4sSld~NgUM<$BRs5<$9cN~ zdIj;+MsK8?-cwcD1}`StQd+6d<0Q4v?MRQG1XtmlU+|g2J_B>N64poTJZK)#_kzUJ zfaQ2d(0BXR&L91v)*}*2Y+wE#-rh1O&UIVYhLAuKAP`7`ySuxE;O_2Da0}2_6Wpx{ zF2UVhJHegc4gnf>ryKh^bMAH4JZG<6``=f^531-YDBfp0?-=7A_ce+GEk_HMEN_m( zUGqo<506)vkU{S2qVbu|Ss5d(D<@oj$s6929@x)wKDTN4;P_jP?qB~l{y&hc|7L?Z z|CZx5SB_U-ya{w!Y;YL7MUa|wy*iX1(hRX3`y(Ksx_~yM2VbD^S7x&R#@vmTi;N`X z&&2qTV%)IPe-z{RWR|?2fv97xrhlP9wiQ|+RP#5Z>^ z2=FE2j4N%!gI#0vtn=+e<6?@MkuSxOY6Alqs{IKUU#QKk=6>r23>OQuQlYy56UV>l-zaq9DKbfm`{v#zPl@Rt}g<{uGO4VL2K-C_uO z6N#n%2j%T;H=dw1>8b-0A27Ebop~w*1*1pAnpE+HbNL;!XsKDNpJ)Vk`)bVQ9ck55 z(E551+G!35`bDvDz`lL>Wo}{U)b%-;>Xw?=&7dReOj}BGKIft2h%9$yODJU{F5^(ICYi zc+Mzu(;>2m)B|F9y-gqN%WwGDK~HAoQlsmCxm&*2Q3Rs|F3zS|j;5J38-7RADAS4j zF#iU9`xu=F3JxPff{)HDGorxqY|5y-cRCZ&7aAu1%UV*1Q?4-lp9Q|H1a@_be#a0v%pc*>Qf;PNn=y4voHuz(EE#(jwwNhrSDKi+UM|AF**|)T_};^hAt*3Xq)o zrlEIJOKeOvQV%~x+yD8S_A&Q9dUes~YROdDpI@mBo;UgY2nZgA(k*3~fC_&Zyr6WT z@|vOFxEOUhJQ}<@nqa&m!&{bOr^l3*`JJ(fmEBp;k)e<5x#{`9cxKT$0^8X03>`lA zD3$h-r4ETPu-2XGNBuHO7WaK@8uJpDXBWmTU4ICs8Nu`U zD0sh0k>4*|rzcm*bfV4M*aQZsJtkJfWmjbI^07Tj#TsNSbL)Jv7Dg3@#k++%6(-i_ zRHAuv;`S3bGaB*2h1z@hBn48lX!~LxLJ1o4XQ_wDu|Ib;^!O_DA89UhPA6{ibzfj} zQB~(R+1)V+z6Yg~g6CR>vk?0WFBmCp~|MyoHi5#UQ zL0n-2)&C$Ca(eR|(*v_H^l7#2L5&3c*7oNBNH8XRx^5gu3q$?L7{cqc0qC+~j1 ze0%lnv2pJxLcx#yj3(YI*6WXU*ONp^gzJ{?;}NFbfW6{1Z4M3Itz{n^r|&_)b+8-) zOHlLsN(_Vh_8fr}21l|d2=90QNR8r-2$R_J0D929nG|1^0bH59{7By)0VO8xB+)fw~FOk1_oCodU6m;L3z+s)9FY8p*jFu1i0|2 zRCCW1I;G6#J%NtkZO5dsTV*9ZOcU#h$Nx)LHXDw9Iw`m@k})Z%>_l`r$1F1x#DGXLnS%dQ8i zx81|yWbT_n2Zn|yuZqm_%|awvpyE$Gu8IFRW`EjoDR96j(f-QU>MWO#1H}B{;ebX< zYu0|NdxG}!*2}bciQvxN82tmxrAF!;Tahq24#TaA7S|L$04B42esN|&%O3ex^G4tL zuxO2FhY#QoDv}Bs4IM)|qSgeyMsz0(YqF~kwbu5?6+)@BRd)eCiQ)(dPnR;{qUsD) zBC;47Ic}h}pwbtr2|2If7z#R_d(yuYMz*(yG9P*5CS&`_(JK%A8_|tYH4s3+j(>Tb>W>wZ;f5AOHzgQ*#50N=dp?W%gNb8 zUS)v^k7YN*R3WJdj6_fZg0*r9>0PNz>{b~xk;JZpiW8-qs5fTcG`tog`(IL{+P|g7 z9g&Uio?;qsK@sr}YhvPW)`YlC0SDi?9)5GIg(T#8h!7hH53v!F%}0G@XYUvj@~pN_ ze)cW~vHC~hBysE)-k5{Z4vChdWY|im&+QeN#Ao}S8o#qpI{}%79o9X<6eV}r1*-22 z#inCgQ(JSPHj5$)NPw8XR(<@{oeH9NJGgjh;4AhwId=?>u_Oo`I{y$Yx2YI)z3;Dg zhr+6oQ`;?x>g9|0AK(%tJaB&_Jx9_ETrvD9a8h%a-jQD>h{xrQiBI~JRGVD{AGH#H ztJ*y#%mp7AIi&N{YK_$QTta|Kg(uDV4sK{1@@Fe(`Ym+IhqNiLBzB}}(qA?a*Yoj3 zbu{?w;pEFpzx&Seth@J!3cQlgE8+x^O`hl${h8J}W=|gl>h1YkHE;RSb@7 z^FWV5`ftLgX$B+qZ{`rw>T34AaH|rgop1I7k)yr|bA8-iknOtSlwUjMwIzYN3Ewr)2 zb}++d>vN5*m89SG3yY6f@YK>UTSmG-s2o3LT5mY(*H^_jze!56_`$PjL3PE|Tf9(% z%UwkbD<&q}Z@63o9Ur%R;T+FAQQgOu+Tpnnuk`^g)dZVGy^$aA9WfI+P^=A?&_v2! z)_jD&@D0T-5P2K|#!GKTn6{&ZC%+fEu&X1u-xEmvSifw{nQBOWb|Br)>Eu z!4v`YFv>nvlc7RPGhy0xSwP#c^S@vc;coA|)>l=64L!1^25I;*GbB3~pP^Q?XOvSa zZl!guVSyd4<9xF=xrDQ9oCvYuGr4P|rRK+D`22vRX>!-)j_9LU6hvR48f)qzs;Up) z`w4M2Pj|oBgZ~_b%_dO(yVMLM4dUD8G&5->$%XKGS#-ac;_d$@M}QCJca=9>+gq$Z zSQktf&q_ufOLpJ9cy%)KwLRTKx>@;d+-9zoY7>c2W_+fOLk2&tMzOs8^u|3`O-tl2 z3c$hFymfMc*@xuYa#Pw)`O>i%;;(^tAKv*tegE2wlWn;PJA%A>){1K0a7ISy!}}8v zzGE(QVpuTyLBv;1R*}t|kfwoBIPQDa2mR=$5QTmRggQF+vZbubPnutPI87?xcLK6)Go5H3N{t>j%go5qX^o8=H< z(O&h67-I%{>%?6h@c3lYS_s+o>ZU!rzARvbZVG#=YU-R5J2@*ME3Ms_6lIDem z!r>QTq{-VE3yl#KMOVDSu-7cY#4+A_-z(DZmqM@(4M#fHn3F0Y4F@TAw;RPu8+S}a zEgJao205uqr0oqlgtB8Q$cRpan2NFK&USo;Q_K$0IfL z{1&0eUD1+gAbr;14M#Wo{G$fexJE7KLs?T^Zc(-b+Z;nem3H26jz{f4;!9(Yh!$)- ze0(2XzCj~3_NS3Zo>W=l-JHlHDhUPgB^MTTIAX@45r;h?01qoOXx92N2rOc-n+u&O znx*tqqx-QDc%&6tf(?FPN}>~o0pmoDZ^mlR1k4E1&?_Hr@6dbT|GKXRSNZz0#yh(q zT5XIm8Sytzd7{xa*l!Y25-G$t`b)_JTn@Pm`~r8@5?J|tsa^(S`E>b zKOL=ydK5W$df)WS^`jWAbv}PXD1E2roG~(xJX-~9uJ`--)lIFS`mF3knjxP!)=bNX zu#6a9{u&_>TT zjmtJ<-RmpdkZsANYb?N-(#GQkc~YHeM~7eZ7{bF}ABz`l23K2Ry*(IUN;Wh98jt11 zA&LGUs^{LzjjC;UJG~#ibYT?7!I%Fb!b~_0!DapazPyZX8VbrR* zyzJjfkuL@YjV;G)=XC6C9iYAOI+1;uR9*Ah+QA~X?6gTeXU{lt-^;?8quw3=QU}15 zN?+md7eirl$M&C`1|qJ1aT>mj+7@-O>{HpfGnwdhpYS&nO`4jZL$!v0O=)GO1gmh# zPYCa(=~ca)XH}OyhRZWRMg-GIMVaxj&cC<_3!usUXzjO4}Cs|(tEVd~$bry14ohz0JdkX|{ zxerv-MVL(GBrfD^>}yD4eGFcp{?azhp^r18=hVc%Yvhhsh(fRHwk}F+H|NxvuPyg0O6v@6sIXW|9m&+4xU(&#?swirvrKceS}0u1z9>E>%%3!!xUOtB zD4qBFR(edRIl7}&W-|VECOePVC-Z3WwAaniov5wLJ6-;ISQi&!A=d>JoOGe*6%LXV zJD3TETcuPy&uZaEa(h3?v-9Rd(^OsaX%D(3{3$Wz4g=Ht(nZ0**@Yo2WUjsq9ydA!HH(b7Tzj`i!Dsl@_&=;AF*ycA=D0n53|TI9cuDY=SRPiH8>H(ysg?Au z06DvAbD*ySw-p@}I?S1~KX^)vd~H{;;Shf({Y*C#B8{|sT8V>fKsES%vDlPg_A|8d z(wjXZd`fC^C9bD3p+$uu?p$asyN-;uepx)sIuxyIYk8JB^WFP>nQwQWO+Kr>`>4fD z_u6=%f*!ssD#Sirt@^-`kXuXCqvAY2ZHrd!toW*k6Ov-L65XRII0(2uBSgXAZiTnBHlWwz_I_WgBIN2ue*u z;HkqWCW-3HC8om#9hj~_zm(1^!lJs3qS`A{OT0kCprBB8PQk=e|1pcTo6w8(;F~<1 zyOU%*Q{6+No?oZRAt4yl)jvaENqiyaT?y9i_w9kr59bFvC-e6Uc70mh_HO8cd^c*d zm1%PM?7xb?tv$&a-GE1JwjZz83ZZTA1sa$z`Lw|$SIdj*OOv5r*#n_8R-@ZTe02ba zeD>>pd@cjsSP{!Rw9OViAmql+sj-e!9q@29#XrlOstDZD3u?W{9-}GC)j88GW`~;+ z(c$``GSKCw2DWw}qqY44O~&q?HjP3f`eWib((q%)H<-8KzKK)xFb=cRI%Hh@sVLyc zFy!my34B4I2Q&xhCXk$bbVdCVSjce@{wS!KwRVB%%RVQ!RIX(A3Q@>mSa+}ohcrbHDX>h+s!Mscv3g=N=vRN z(j>^ES*Y{n5okQI^!7pb+P#gn(iB^{bt9AqJ}EXgFPNC=`q_#6`EmbscDQXA3QrVa z|HWX$f*{_nR~628%P*#N%Z0>J8z2T*ZtCqtl6}~?zhmUAXb!*S+|9&+r5c7#fCW`| zE0*s$sHtTg<>y?UnDP*TIB-3hauW)n0ioz!pO88LIaSxnlXc1%S>}WlOnb*7R#Ax& z0}Ddk0y^HEK=Y(A_$rJDO;l%7e@NwrihXKVh&cEr!s2sJC2*Z)m)xN@Ut`0)snBRN zZv0v3G37`NrYzs>>!@~9mno*x?11VmoL++V>k0az{6VD#{D?a`x$`ir>GLS&et4fy zfIaj96{C#45Kxs#@nP2yhgJU`luCss2gRjOD zP6}+ph}W1i)P~P~bbs_yCA)F*d%nq z_q<{7@-vFE`2KZp6b8QR@9#wbGHMmCxZ+F|%PdUhd<@7d1R*Ap*wc9*^-h_eP%G7tHYP_E<1L8H;^w~H z|IA=|ckIZrw~fF*wdZiTD=Qk+(`EA6EP?mJp92FrguGZ_^BXI&d;Bc7kVU=LIgBb6 z^~Lypg9Yl9;KxWOhVfRpOZ%BO`OyjWWj#5$|(=B zf>o6;Kqn1A6mP$%?W0%6F6hRyMTe>D6c`3;;is8;^K}<$vXbn~3&PYL5`MZ;BN<_} z9k{la-qSvSO&lN{dda;mZJqF4rCYiWH|R0SYESZW_I! zJ-R?ZP4=3rVNZd*|K;kVok41rlmNHi(9)*t%?pG(vMDHJsTO>c1(f)mGaKJ36asu0 z*RI=mXj*nOJ6tip?`iCQ z^0d#|mY=<+tTi3DVARkA@p-;snJU{4Ml?)RFq#Pxc<83^JZ?x84+&*|Yyh&Fe5BE5 z)J;J5M0;r=-7G@v-*JLKUL?it=RXK|_G2PcbRCq!wPTT$jwuW5WrYwUeM+4fH_+v2 zKo{#8o}=rBe*&94^+A_%Q_T0B(B{e>3GomnKu8cd2n#wXPyX$SoaXIuIdTMV-8JBo zr0{6Ib1c^R4U##6_8rH%#1zi}p)c;`w7jJdm%s3q*VeQ72^!9T9YgQb9qMJJ%-Vc= z`{mVhk3=x2-~VNncS(S~qx#l$sec|uV3!yk#8#`dGGc4#?v3&N{1`7nJ*$*mK)gfFqK3@L&>8%K` z9~N80(c@;M#axza>q523RkGJ`rDSwI7@2c^M)GsgF%R#EQDg4ft8n?svSqwtS70Qe zl7NQ=ZyXc}W-MugpJ@w84LXk_*oB9wHY8O4FKHV-i$GA&mx(Ln9(|OSC*7c`4wcDtN&7;Hm0a_J{4JZANrmoho5i8-x}?jbQ($VtMgP>A zP9#iIbI&F70pQ}tStlo%1Dcyy4c)7w*j;!N0jU(z^#*2n%Msg*tl`KoTef1}3;WZ{5%oAiT7 zfb@>px%)U_aU?Eb5ipRbWdw%70Luy*U#ofGWZT&gy&Mg5HqmvQ-4VCE6D>)VI zs{WI=);n;hu2%s_psqz0kP>tq5Ywd8s!m*hpWtDr;MVm z+WMm{$(4x=OGZW{^_{*Iz}l$v?ysmV7m_K}jb0pyAfy+)<@{VNcsgb@QF9AN+fI8RY}2gc0+05o~7hr_i3`| zQ2T-zbw`|C*atVBWz=%dN>3Z?=_BLfaDP#UcS6^oyhyy6%45aN67>T2BgD2&;Ssso zY=9Av%A#QE)5%C)kGV zJcF3q{gi10rkVP15NiK)hPH41HuQ2p{N8|L3-(D4o}$1N@nz2TX^9D@7MeTkAGJ6d zV}Y7K%9B%syoFBc3+vRyyf5tT=Z~@4YnsrQQ>o6rWFG!7Zn>JFBe;ob6}5JDlnl6w z{PVj(1Ft3<*Xs5yK;~i!jao03_?)mD$5KbT<_wt)v;a^6;dDh3Kz8bTMw$RZ6jsME zyVWRl^R&e;r0tZ0zG1OAV`j(j?BHhf6ODD&Q> z$=ZSBszHt~=UmV>3ooPObE|cfbFMx6y$dy#?SK{{U*hcE$vGd{N;bKA<$GXB;gSMV z$N1t6ULi&`12i^laW&h@5EQfhSQjlc5dCB{^l9!>q}&hB?FWHc7z$Sq_1^!rMGvmK z|J`xMs9oc=@u?hGj*OG`SEdFG^FUk_YdiYXSQT@DID3*mQnPbYu8!{*GcE*;D0dOK zm^+Xe_RH2sbo(%fB?j6;2M}4Cx?b$s6ciL!zg#&(NAqQjkWtG^IsGQcS)oG%8{lE+ z^gB}#mZ3dbzmWN*{45mQH%FaqO7F67tUvv6UB>kByuN-JTuxPjsq8`SvrpM8FkM`A zY&rW|B?!$gtqQay|2q33`=?!a$r#kl^(P0{*{U_dB4pr+GoIa$|vXWQBkp1Xc4Al{!biHH{TqLEDj%O@@S;kKA(#x^~iMq zi2|qx!#D9Ci67ySVB`m=^N@SxT=A4L8B`I^pWFdVZX85z(JGIAKxjNvjQ8vRaC`}n zT@95D$r|0*MV4?3zkVz?c+b=^TW)4j{f(3IEOyvakNB~9yv_`g6HzxWAhl1#@p**Z zZxm5?{w=%A<0$o2uNeBWg2}~nllJZL-N@rGe#{4Hu)We|V}*EF(==)kEugL*y-<>E zon?tvO} z_(}74IJY?Xy->ENq9Wxj>s_YRu47dp{E@@cfAc*UXjfNz!u1m+i+WJs?MI^QYz^&UJ>ZYBu^w#K^2z^{+^jk56E!hZlhi4#I zZ2-6DBG0cR#|GJ#80ZX$_qvxOcZ8RIW|4p?w>Mkl_uWC8OCEI>A0q@~8=Z}q$hrLf z6nr$V{Lk(mwq6R3Fw7@`ENiRf9~%-LFm^?N)JoU20lAFVmD&T#7(DTAMAI=yDa78boaIi&Mz$l1V70}P`%J06hMgUt$~2@v)X!sFZFMog`~EQ z_9qYUUeS49DT0^&Y4Msk7x< zO3<%X)oHhC3C~7hkll)ikyzztEqKq-<50kdo~g}5tRUifUy?~BO{C0?00X_3=Cfik67}`1S%4S9MF_8Fg_Bzy&3Gf6YiFtJ^Pk4(@hF6GVtM+QkbzKL!+}R%i>0Tj&JK2kA!gkj2(D&5;SXI`sH*zWEZUY&l@`4iYEK<7_9%owZeU?x%QWT!OoZLl+s&hBO2l@z| z3k5&DdhNEK6Ai<>2^85=ZplM-RZLaj@wrzzR68DD1Ox_w^eU$2at7-{2{CdWtko5Uh6`TW3jmm~n8gH0oPA zJ4e#Uy!+OcghkByULUBE*t@EP^ER(o_l!${sj}q2$yf8tDyAql{XVtn$2uM~?G(>; zx+#C7agz7PE+n@8jp2_N$2^@HbM87OjI$~nITB5X*V`t<>0`kbS0Y;x@8n=t>Nx0@ zo*IZEZ zmg<}~$0p9dcwKGPdjIrw?`Wo=n|yfZnW%O7zh}^cLa=?qrh+F$W{4l#iS}a>aYnaB z#_TN)G(W#!Xf()uS$&;77?KSyws&5jeR&?iT*K!Te2z{%%kFIhzX`@b4g+W@V>D|H zWDYBBj2Rs109SH664LQ94J)Ryr!@`NAz#V(oZOrsi@tRFZ{ZmXLeWnr$pw6pL=>Lh zGafxhPl*Mtf08SBqSp>NGSIWp_d&tLVgw78=bLz=Mnc3r(194BoG=CriOkHC&Yt`d z6?vcsPuxI+CtzM&N=fogDHj}CFELJNsCQh*v4@f(03>r2(I2)ntD@*vwuA@SDz|0k zuGxL%Po!NNGT@xv`ASu+*}PBX%j(>5#?Q>tLQk)Ib6&j5T6f(|y_r_s?! z*$b?%3;5??lk41`CENLd-K>WeRSxIa-~l5xDE87fWLQjdC&S!(@V*Fc254@qIZ9kM z8()k(rQPgCkBh$tnetu@-T#c_~Dya~e*>$NiNhMYuo(+4d$C&gF|o!`j{-Td#XYUgIc{ zAP38d1v=iYudrWuCh7i+h^E^;e#TVK>vo+Byk#Xf()~CxJ~5&hFvodDmz9REUwFUS3}u=DRcSdex0`~VLc zlah)fh6G8sOdTv|YH88NehhM}6GNXA)o}USsQ?O0Fb9;x1L3^NOi7RJBphV)$ zr7toV(p$wQY)$(W&Y0d4mToMHr?^@C)p$ee!Er9LW;i)@%j{m?8-ueg(*J8r7qHvX z6o!T`Yy6A5rAXR-kGEBSUx$F*{2keNd+#!p9KcEjEj@=A7`r%le;UV0i0giND~D+? z3dvazF8^)0wDQc|VQqBmijC0+%EgTvO5b=QALMdd6(FvO3qFWb5ro1%t?JY!dYD2W z5B{WH{3Yv^F;Vckeqexn^OlUepAj-x@_y}-nZg&Tj+mhL7H6l%<-y^aIzNtVO}*|q zdt|Nb=s^v8N!wd~@r_L^so`m&1GyDtd6mSt9lY5oFTg`N+?{Q^$Jbf4y!nVFC8Bld zU;e3&3To-YAOL%g@jVvF{Jt~2ph{i%oe$n4OP$*_CGyq2c-a9B^vjNyFf`d&`?__2 zA(f+581hcNv-%awNQLLvNE$8g;M@g{EXH(7yD)S$Un?zu?d=lSTv#!@F;mbhU@>!| zYGz{{N9--nKEQisneszrgym5wMk-NOSCtWy*>$8!6!G~7fheKzL;f5>$WL_#p(@Y5 z<8yHTvd3Jll+GEM)#mSzhK{n@+i{8;@g&2OI+q8sG1`OaFb5WsJX?mZ zX|x(t6^2R@J41f3lZRqZonKk>*e#A$fsH=x2UYZ0KwL+VNh7O_fN62zRczk9)b59h z5c$G;5p-LUEI`W!@hRBvR!|Hc^5qzFwSYUa1m)4F(>rtM_h7B zhdJE|qt4GoPYVT6YyCfdn5PhT>T)Y@iyFAsrsGa7g+VTBzvNFZ7VaKGHhohx&_O6` zLHLvt(qUvCt3+B~hktBndbb8?@ha@>Dfcj&|J-h(MPWEUY(V>E$IoPMTbv!y8^t~| zq>8EGD@WgQ#%zQN)mC>k8#C{L{-}d&^6l5Woh_!F`DCzE+GRcNDSMu`G%B;Kf;ke> zX*DPtaa`W~TI*b)I9?N1pksQVh7qvJ%58uq2Q?5|prkA4VrrFW}iK*S(p=r|w|gcrG78FBz$M3`

oS`%tQ%NTMT4w$;w{N45=v#8WTPt}=8MySXLO9IL3g^M`hjT^R_^-Q_T*?Z(ZV65WVB zO*ozSn;`m@9JQAndD-cs?R<5#$)O!t19MATIYz!T7dJ8y#O4024QGV5p^**udOL+R zby-bXqp4;mldqD;3@^`ke7$zDUi2B}wWe!OH^<@^%+x;KDT*g0 z-d&6^sOp_=zXKXmv%&l7Ry##e;O6jeC(P!vMtQhGJO0a(#h+)8`dE^@svP`EIK08S zeM_!}6z4TkP+dQ1rIpeERIpCAtdJo%jP#2y8tM+X_i_+BwL_^>(|#tR^h9>nr%i+5 z7z^*bJM~Qr^+m#)l##Wafg{6@pKDG}UlQ^~J5S-z{b7p!z3~1x8*ao#6#yk`7L==k z58<;ud~ZphPa#9&b2f>N#YB-m!sUG!F;OkPOOE`~9mJ-4c_q8~RvL_eQF;e32$5oQ zZXTqVqi0$SS*3xS6Fk|#i6d{ROT=oaxgrnAKX+E0*+Oz2WDVD99yj;4LZvE{9R|^b z?PL43*?`=REa8?jyk?Usk-oq5ErlFl#GhZboI>k#c!8~)cJ?oX!&jMB#H+7&zB!7R zuV=KB-?~#jR3N%QzxZl&KWI`$D&+#H8(w~t%cg~f4307NFcDL)yrggKxz@1#>be_c zV3--G-DVCiN1o!Y>h9%2CL7LDh7W1CqB#U$?IP(-$?c3Q&tc9&&O4Bv&ELR8Mysvo zbgxR}Ze?UY&0gNjcSenZ=Y6mg~>*58XH0`tcG|z z#io$hFvH0S~?r)wnat7=SFX)DZ3ck-=1bxp28oOacFHxGMq!T`gC7cRGF#s7z;X&iumGd2mVE?X@8bLAGI$UAXMQL} z;Z4GceEJ=$?@rN(swNU?{OF*C_m6C-ukWbiX|^=Ti#Zi=rk~olFTh=;`vX;Mh-P<) zt?}EoVbr4*W&D&?ooQpuO)0+Zc&d0BB?@BwbqBx!HoP^SS^F)0%Y6yJn%yn}+9@2T z-sao8S9d#Bu8@oyxOc+#u0kx{!-G-@bVgeqrF}WnCMIhr;S1F^+B!{rP164eygn8Z zF0^&LzolW9>r`W6^Ak8l#5 z8HK;&LM5_&d){{S-hp^RGIj0#ol@BGb7S%goFg%N_)b^7ScD3;-lW_ zbem?WP?d634E9X(yNGCu*~)pzgeE$%tAjRl8am;m`7;jbm^b1WkfSG+)NU0{o5tCa z15J^-r0)BZnF%vOr0@q9#8+p|BStWMB*Dbi#q0MBU(wDhcP9j39ETJItGpb3Z)B#t z7_N@G2;Urq9u-CZF7B^5iDU!L98mvJHj0&qE@Jj$k`;t+OUc{#VPP}JOGqw&XTLTGhBH*AhGSUjgSGnQd}~0 zz;W&RZ3&p@kPN%=TQf-3Pr)Py4DeF|ush{I4Hq!2FRV z*u>pMNj0p2MHr9v9kvq(QlaBhz>TvXHZ+DM!OD_jxNFfD>=_nYh9 zwWU1HpN9&!DcW8o6ppXS@Jrw4-&YDBp*fK^Z2wFvq?X?gN?(j$yo^C^56lBRRFGM4 z&?kyd;(vK0?qFDLP4pul5GL#jU96G_pjacwu{`R^E}k?HRKmcd_IUI z1sNcewBKAYj}_nT&+!BdJleBW${4RR7npSaqKgXq*zmLq2D7BbP6|*440L|AsY+DG zdb}n(h{6A9xS}Xd3eWGx4t0J;fygu~zhPMbUU|b+JM~lU^{*iR^(wYk#&d1*I>kPgybT3L>d%pgog|DByN@9)^0Jzt zg|Tj``0Z#EXzzs{iedCKpXb{_`6^A=?pY0*_mB4vy`XZvTQK8D?qS2kX!gL4 zS$On$zY|;Fd<#^XUKs%Jx=$yhnqzq~a&Pf)ONr01fl6$9OU{)HCteaE>Km>SLj+ zy2`l=ja4f;FzwX~8eoVJiJ1C#kV{9_DR}4#*2+%OXw=`>%!eq!clbE&Q`l|~C}v7k z*cVe;^)(A4BF-NZZA+r7qB^VF(ibKXGuP58Ag*`v+H0u9Z(GY6>q-RSRuQ}mi!fQf zFuCQ@+DCWn)CEEJBmynWGFu(4IHI5A%_Lputs)pt;J)7&PbU_lF$7&y%#N<);@IVU?+4>HBvC z{2`cl??ugxfXkQ0@A2T2u zq&udlP`tw0`)OI?P=9%8d764QSdGC-UuZGTx73|uVY=rY$II#P&_a08Nk3hK$%scP zf#$}fKi;R1ie%anl?*=>F#d%qKaYdrsS`AX!?^il{&+laaVkdvwa}+lDP6;sm?{13 zY;SZ0$_1-Y6|{WI45eWw)lc~TYRb*s@||^^{p^8K9c~x$_PuLl!tY<-`i+QX;+#=0 zlI*@~w|z2z#u{%}i?9o4_Af@gUR^tRMIH63ul<1tYi`b`kte&T2y00gln))Q`O-tD zd~-lYRJI0fPjao%ATxdrlH#lTqDc<6xv&{Y{`n;Ip~=Yd6zXno*sXPZhro-L_QO0| zXvDx~GxJiYIL``In0PIapyr16sWwB-1WW0RMMK?c|Y5sDeJ zmKtT@opc6ksbe$cgfU=fC-+@U`sEqF9Zx*Jh8$UjPN08GWEYa9hZ-V;jlUF;LUyKH=o+K?&jQ#zqUoP$q7kYSN+*u3?1oh8)Ml zwOHb_z2{=q7y@OzC0dt$(Z9qM^wVDNW9rHCAD#)^QrO18ucK8llw)3x>e*+MzJvn&!o* zi;U@LUf1tipNouGUC_jNZx?glRboFJYstT2bE4jQAvx06dVSkdbAR!&oo}%8v_@S^ zUB!i{O|!85&CAFiWzyaTYD4ZF(Q|FBiH=({_b`2`D}H4SsGUxs``nF|^eX>zk!jmL za3HP|5H(68BN8TLG)rH0L2Ku=R!N5DuvEWcvG10}NMAX%On1b8V|m-mvrq?kAp*Rq z;cnBtzE;@#)j`rf*=W|DQZ`!IP(;c)HTSOrbyF4L`8?tq%zvnvw9zI`L1uQOY< zr>|0*GkcKFJHM$nE#X;IWtmeGeW-OAI>BJoI9q9&rej#BROmOx9%fVjl98K(D}0T< z_ywKp^BC^=c=n{F7K7U^iSv!k_*|sXjOB~~i;)O(hKSsH?_v``OX67`7Vi6vEYhzN zLmKbG&H-NZHAgXVw>OT1qM_bBu2E=by$gsDsERdv=@$21UFD4B7X@>56G2U1etzdr z^Px*8TBE7Cs!oeVy@@66&hum5b%20so&>Xg8h%5O+#6G?iutavX!-$|Tww=v<3#e+ zcWPFVJ|65~d@@C9Ng8bnVcmez-s=TGC)xaluX1CH@Nutz0k0;G2K5{K8eTXUC z-B8#?40;L^UT4PK_4mA0xqT7YIk~K;2NsclSlX>$ykDZLZU?rBM&Uk^K8`8wj!3-A zSQ_S$QyXAM@|EKF;?KhA+#`yyy+NKhqj>h9mwn!Lb=_g$WNlhVYAcWpwE5z5gv@q6 zxO&^*CT_)?i%Wa!OO#!;oQHO?fj<%_A}Ajhik)&XxCjn*)T9HzB^^i$9EdS8QIAyL z3Z@Z;*_t0cMC6~m$wQ5{*bw7yd^92l8>+5V-B?qHcEhN$&+p!IRm?7OoNig|S?qP% zF89~ofb-(3q3&J|{hn?Cj@^Y_EUyI&>el)zd?^{{F@-;&57B(A z_)Ts^-TVP`wgfUHjyaH*na5T=k==W%ucN$^Frhej#z*eW*x6Dy$=jH7{n!~oXRh#z zUH!wG4x+Cja$=3)9C`^lwJr64Qj0knToH0iBuRJh4JYVP&9ZmNnwJ9ygXV`ythbaB zTFQVvUuH_es?ZJbt0+ySXL&QW#a~s_H0jg*STb?zQ+JRqy?{-mvyjkJC-ith6UWW@>%W1c+Yez4_s#d;jO0UVz3vHY6KA!MKIm@TKBOwz;6fT#X&`d)xV0kcEFtjEI z-i&j)Vi8pem#NXQTi2$*K2J{ph@Y=X4-z0xoBVLK$R!#;afa2{Tzb zkZFelzY70E+;zF05mSEm`<*}%Lx1~zJ|0Twb|M~`4o+9tt}t#4u)947z$mbx&=e8k z#C<|`QI7h!Vk`Wy7BlxuT2_YYgn!q!3m1o^pK;i>7AuH?WmzC-CA^%E6z4o8a{Mab~p z;oZE=fEZa%hiPxbyUFv9C)oLD-Nl{Ag1;j=Pbt$yitXmLoV`<6--oT!kn@iQ8mQQJrx9f+8FBg|7s}YGhuOL`X%k{h$MUGqD0|pGV@BJ zhO3|cP%h90gS-2ngS*UY@}2M8bMAfbzWdgC zf52i5J>AvSUAy-FNtKsMX=4j(IOSKFmEl=RCj{vVIv3Vpt$cxx;SD_lkBx9ShLuxf zgrzwj$3wgmS5wS44pQ86pvJvNv6fzE%z48#n7=R)n(OP3*Ff!k$D8tGLyW(*WueP^ zBgvMGn4cCJedcvpD5X{cMZecO?PsTWt+`;>h7lk>mZ*zPN|YK5K+-2j&g9Wv?2HI| zIgd2Hp?02GWb_2pOH#g8%g8Wf=xw*VN@`$f*bfT(5u++SgXf)_FKxNZUrdbJ)nHNc z5QSPhDj^Mu1i5{6Jbo7-l`t4KgTA44G#2Ek`#imOSh@~Ee#QTEldZGzPC`q!`Rs>D z-8Y={Cbpl`O7z^5Gw2<68;=8;vId)6yNj10EfC$VJ>=ZmD;x1f*Sre}G#d$XwcnRC zFVV5fme017SDWKjlwidlI&eHweEcYB9Gf|1(B#;$GdDk~_HG879DNB7 zRXV1uJQYN?exK(zZ>K20I`mA|63?GKIq_mreY1%pndiW&+Z~P{<~~lO%D3{K3d9p< zW8hBayBf4H>Cf{?nm`@I96q;cq$aRAl zH=w??k-A4+Vw;TjI0nn4?7RnfX=Ic`W;;hht0gL#Z%b0G5!@*%Q|6`;o^MLq*gWa; zwsM<}iiAShIr-%!&o+8NyIW`9;q%D0X6zGjilpmaTwUshRV)r;3WzaP z-A_o^XPIn~Jl)sc(HNY?|2tUry9jFGs^c0 zGxL0_li>Yy2^68F(S4=;!&;084g)=Ms70aBj9YDmU`~k3kw2aY+Q`;=8jrZ9BP1`; zi~)8Igzr_@cm`57ybeZ8Iyz_qoHc9Fqk4r~9*_P>K@j6axXtNNJfxIcr)mKnz?323@=gDGHf0oven#CpWCh78?>BQJj zr!3R7a@Y5LGJafG$(-L2h9MbiOlaUUJ$}@Lfdoe3uL?z*5tEp{F1RKthVGA0pHOT` z+H^^KPNBSqOd*gPbh|trd=|H_f-$_tDlUj$h=p3bI~3jSXTx2e079<M z`y%-+@oLN6ZMa_nf>WPvxsUa(33=HklP4h*Snx>S0TF927v}_{l@+j|+Q6pqDapM< z+1`~+Je75fkQ@48wCzSo91f+T3HKwizr5HgLy_0mn&`x|jjVt_{o(Rcy$b)nZ6_tY z^v-2islkE01#XL+ZN`FhR~i+iD9jZyU~W4wc!^><%1r)Z?a`$@$+zo!DuE+2L&Zmt zFcho=*3BZs25&0l8oH{-o!AIQ{7B;-?8Gs4Kcc%v9__A;t&F|KgDXnK@`wrC9hsqb zf&zJBJiSG`ZCOeAS!nTp{9Z)TPf*_?ETm{*^3FJyPN*6kaeP^l!aHx>Eql#v6n!6> z%f*P08RmS((WSa6UBa9Q?d=S;Trpkedl6sgu1n)@cVxVy{x2F+pHz(LVh?&)8A6*fQwN&cv{#@dH%yh_s7 zafF}1?fFQL61$ZO*m6Pf7rmXcCpZ8LD*WYMjb9$lnMhRDH*l)$PH0;o2A0-)lFQ$x&N zO8V6q@}$O#i zWM)d6mVRu0Z5Tdk#?F;G@Q7TnDNO;YP2N6DM8k4bhEKAUO$I-H#zzM$rtq3K2CFaj zRsuV6IJ(`L5RMsz<;EkXG(<8bN;s~L8AQsHSP~B;x=pLB$ky_bk&)S08y7T*Tx7l;qIw*To+jq2%kI3F7POYxG?9&-Zr_+vFBHOoJ*6<9qr9OLX z0Xz_3K;$n@N)=t|?a^9P;)%>Zjq~2J8xoLcaNMt+0sakp!AEq#$yF2Vurl69q7=!g zpWZhP6e)as#->^g)1_s})V2PC9Tq&Q--4L4UzDwXQ z>{y01A>Uw`ZRNW`=r`xbMLWRSB-WG93SD@~)X8a^W!rAJ>zC&n6MRFO{hpaGT~+jc zF;5@#y^xQg_=R8(El`pe)}6zOvtaxr2HpAI9%~-9(RE+rV@9AyBk*xB-cChBXPG!C zp;au8fUu(N1wI~ zt^|BaY-)N#Hn-3XA?0GF;Ld`Wz7teGrVwxw{5YhEYf|+bn<(<;aGzix%0sBaVXW}y zsn_e*CuP5e`WV!rgom0%;+F%j%KGEs7uwUoqV4~{@_Q#@5nGDKBy--NbuU5 zO2B?^$qv&uD}Sk^Z!ol!FVc~{s&`aU>CgNS#Cxr09(MZTnaDW31?~ubV&bjacK5?i z3b2v!#V9Z!Oml=$#o_doNPghD$v<-k>39!T_Zf)BFPN? zcxR<{*2uWnq|(Dgy8rq8z5v z0ZzS3&D5V$+ntz3HaM+qasOlL{CgHScRhD6A8Dr8dh(ITON#<)r&M4M0Yw|>s3BO$Fr*jrf@~d)E7l)Fn;cn_Qdx|D&lvMc5 zo_rCayuUW#eQ`1!Vpk1{hc~!%2d-)t07rmC{$_LP7_l2xyg_=BUWt>M*fT ztV!BR9z9FlR)_ShTqYlw0UE@m4Iq1N>N?hAv@ef&R+&yi&5 zHD&ilt()4$a73hSAr{{y+;sgjrE~z7!hGM6(Z+wC3425)y$&Qu?D|7<}O=O1Vdq-46f5ywZxz;64;FsA=RB=SzW zy}Mbsp_J+sghcfIN`7|F^gGTSHB58=%Jrydw%8TAvD^Lq#E7oZ*=)A)HOpHcIVu=B zfP9F&Gkm~5oHVq*8MT5o95CL3&w1;Y_x2FDFzL9}t{A#ElqcEK@>-2i?76FtTH8vA z^`r$Ssr~qn{g1uPm|FI^2Nt-pz=-M19@Z`8(c|qL+@7S{>P;+Li1NanP_1`pgPGJM zwo9KTJwfCJ`x9Zv;zeAR^P#i9#Jek8BvrvKHYxkkav%pFgjZMvVY`G1e7; zIAVGUUr&{*wdYp0w>^d`PSJ9^k^AMbgMVVNcY>gF%?=Gpz4rxEcJIy}4arhaJ`EDS z8*_HXE~+z{OVc9pLmhObeyl`h?U-v${<|&5UDYouMRK9jJS_k0P08~qTuY0l9IA3; z@gefLPlG;E{Nr#cgj)02o;XNyb!fZ}dYI*GRM()?QarNwoYR)K)=k80=S;nhM`8;# zN-S=gYFq`f_?c2)U8j*D1A^ULe`9%XA@ zN_>AZzKH6DOJr}v$pTGR>toPCqwMz=Zr%zBW#O{v@R2Xp_}NoN)W@J$aai$%avhWA zYeIB2Ihw(G80{@j=d4Qbt!Yg@R1e>!^o}*I2~N(VxLxN5JuC4>R2)Lc9I<=k9;Y=Lac91O({B_atx(KDDVd9{rp4=VSzuEle%g;?_ z8p&k3!dE_Wh|8-sl+CHWJe3DDwU(+m5T@>@cs2QDt;S_S5Zxfhj=oj-Vy(EI!6dE9 zA_b3`6_HE-UV9#}cgNX+i)Gh~+jo)#Mix}t>7=s1xz37y-=%%LlW*b+JB%E&@P?SzEMTJ(49|O*9);^i^sw2dFKUp*Thyy! z^-Sg@A7!7lJtbZM>WSH6(;C~0&B&vEOHu7=p&@9=5Ww6_w6JB4C z(oOp&h+!!cQlu%n#{f2%Pde}}4@VZ4>?M0BLx5NsbZyWF9nXA+qw<`iA5EtRK?|dn zSsSKiml$8}Ep^q*f2~15@bqeLQ9)U7B5D1zH(V=L%Xn33iJIG zfd^t8NhSuxoV)VW`cM@%O<@;ZC{P`YCBctKu^19xCNXYp(FofWs| z^)}5$%?Fn=O5(OtlBz8s?F>h`5o2I@`0X=z6*&sap|y5PCBNxN)jOD;XOT}yL*K!K&kP$x$(ce zvAg>j<(uv~VmRd;g_+pRlj=eyT8DPaGZ@#@$NwcH0$FHJ7&b}ClY%Nc5MJAa#+fp~ zpcGu&cIw1pJDUAi2|y9^=0yE@3FYUHQB;yItOL%m++7|~Q{d3N<4+mRh$Fk%Kr@*A zJcq5TzgNXi!IR*lp28XiA=`uwx8Gq+EPbQSa? z;-x^>^w9`ES?QkEC-Ad&Kk6V7a>!Z~5*j*2#`kURZrgD)kEM|>$+@*kwN$!4sAC!sQxT48+)*nU?xyE^M+AMi{Ie~$r)hGwP&({$X$9i#zs5M|T zHgP6Oa+f*;LYVz>M8bgU&P@Wf!*Ki;`x z4cU$DZCHm|f#r?&V7%rKJSw4^0w7j#N3uobBknv?=;PX})`jkAaW4)eGcK^%a8k4C zvu+PHk!$T^%u>=HlQDRnDBvvHZ`z2Hl zkS7G4QWUX3h;sRVff+#7I8!C9$jM=0?Mg-tGIB>d#z@AMIXW0f zkRW1_JPWare91RBk1#u;r;Wt4c(nGnMJP)8EB=knWS_hdQ=7 z|J*U*5xOH&X}L5Y%J7k~dEPKeUwWVzkNdv=o?l=f6iLfNxa)O|<_IDi=fO&tK_Z*U z1L%MbtFp_3vs1g=Mj;}j*1myH`;1oBeQ6SrKhLIXXm=aQbYLafo=F%VVSHQWR9(FS zR*w_Cq$FN(NPsiIQ|XN(bNDav(Zfhbi1Ml$xE&F#_lJ7o#Yc)Z(7#4%R%R~h-Yt-e z8JE4_CzrMZc06X>c{Y1?z%&TE7y&_FRLcB{c6e3u>TbO^I)%UB)>%4@`xB6hZ3(eF z^2Vo$dGc)T>S|=LOaL_^@_S%ZD&!o4FK=vK_ElCcjs`s-P@STVNZP9TznmwLn$Jl@OiiEPg)vs!I$R5}lS zf%oh7o`3!ks*w2f5E;JWL;X#kuXAdbc_2|F3Xw7rMPw}{Z`ZRYzTH$6X z&4)R1L)b{dH$b2}S`-EE>3VfD#RxDTn}cwb@`eLSm16NkkjHn@8WUc%QGP;ED>55Nmf07QA@wHxsOQvy)2UUzSw zLzbEtfoGZ~w^Q(H)S$G!L9Pi=u?Qka+=&0KHLr#pL{v;^&)*rbA^Axi9HT%3;pTo< zm&AqL8tT)2vzFKp;ZkM90!@{y~T_7_`2bUKtVHup_IF?lgwD~)=Rxh?d$^rF;N08q$16SQ# z`rvb;J-bZPNT#A&uK~kBp~)W`bR{iLu^-T62q`A9*)xb|!qG_=YS-E0OEKOA6}W~BJdKWB0-IqljWRjzpvR7RT_FJ{VVTiH&cEY z|9x2{(Sd7LyO%L7ERl3t|HL^NfWr8<#=5_!$G>KQ9xVM>nNH11+bN^)Qlfy1dSL~| zzdZ~2PN(m3ZlvFPgvz2nfRC&(CpmgcGH|EQyg4X?SER{TSnBl6OM^*tDpleJWM_QB z3&HE*CF5_hW^w_=jECFGH!n|xQk;^UBMa6K?`|VrYAyC36OisQ`cONUw)oujKfGY( zgvI@BRq(3Msl5TNA0~FW7@P`GwjZo}_KKW%l3BzKZ+nvu# z<#TJ1(jSR$UwIt})a^}QMTfSS!$`LpaIKZ+fA?_h1G4l~lDd{v2c zy!fIQnfxXyE#PY>hZeuLyHL(soI-iXy^;rZLFqjvj$hhFV#Jku3yUyFh&v-2YTbV) zY(mayv#n`MPG&P4Go(;f)(juj&_`5pi_wz-hFQ(d$zz2BFL^Sul=>DWn#!+FRjS{8 z|Du5)BcMlOlZkS6f>EZE^#*5urbnhoaQqF|M{Hge*V8?Nf3QG7u19W<%nf zRAY-!7X(7>G3<_S=;`f!b8_RMzYT|27&Xp&*yg)R`?!s^vIfe46noE_Z6ByDcJhmK73W(`NS@}BGS{fvNnYEpQq9&V0L3wOG2ig^g+{UpUC zdf+;w?d$GeFw9RfvSB5UrG zytsC+JPT9jBQvV!9aYZWM-1Ho{ zqr3NaJd-JpR+;-tmk#Fm5;N0dmbjN(cM>?M3Xwt_q^UCAuD8mjzsXx=en-OLUY=RRH9{iD^N<|f;U`ynXTrz-20@CDU%PBn|Xnk(gmOQ+ntAjY5sNQ zUdai&Aom`V`<)#^6Q%{Q^g z@1=S_E@f*B_c_M30OB50gepY1L>C(mQ`(fKM|i=IeWyx;GdVYxlfSQ>avf?Cq7l(9 z2`^-$;k#lR)r%NG52Tp9tlet(BC#q$8dAdbj@b@9!O-fL4%Nw$i0tH;M0V?-qoq7nyI$MOoX_y*beAxCbNc4-Jc>a z=pQRmncH?h$|C&VklNa!Gp6ED^54;GYc7mTX$57?UDS(;b0xsQ7+zs-6uQr=rA3qp zFDCD6W>h`qNgm|L>CcEdH8r7|E2JgeZS+E+IK23sViz(7WItR&o!gkpQ=n$I_J(1z zk205xon3(@eQcV2Om}NZUkB+k=YTnp30rR$k)La&?6bDZDZ;I4et)ff2O^DcmgLY< zk2G(HRNqWdf;SU@(YssXdhkMy$U z4RsLwm{BY5*&2`bEBw3;_B$&$#14sfW)Z)rba$z<8gWz{@Y{fRziujFDj&|2vY8m3 z)yYdcwBRS#9t^gA%HEz}OD8X?i*dU|3L};*%rLA+pQdloo!eQOC{Vd;+V98s>Plq2<;L$A@7gq=i4nS-KiR$_9-oe3$0p24fsmA|XUNF;Ws z-}y0n1hING<}9CN<<$!Isg9>c!*(y1!bym;*ZCFy-)gT#tZXeAZX7#Fr6A3b-)vSMBWQw>;Uz@a8+Q%_`rig zZp3vL{n%ZK?P-(HXLrYCXv>mU6FMo=vgX4E+S4(mxPe76p&4EVSHqn9goBCUI!T z_X)M8J-Sl11TXHj{gE%dIK*9fg4-_5my@DCeW+ALdwh(tB~CWQEkSe}mD`?ysHzRg zFI)p(mBR@+I;v4OW|5|e@7r+E%JX>yzoN2h2R=HTtl}>k2nY%C-|_CkNIO~wrPnE>vNHy6(joT)?)J?BpGl>VV0mhMNKz94 zp%LVb!-?T!GD5AHr${MG)~&-`qxFf9`)q?LE-~cQb{Do$bIdPD*mgI3DB* z#Zyk1N<_>(0$rYJ&mGxhewsP{7E&g!h13Y)UaNXpf!WsqVd2$}P&Nh?!{xJ-=ri#%|E0<4Nm7|555^3+?ld?uutvXPpM?A>;x&J(6t&b%Nc1OZ8&b$0kyFgfX{xyx-W z!@o-R)s!6k@xn*`jj)d$LX%CzDN!39%;ur{gLEcI^c(s+*ysUEa*o=pi-fi)$u&j9k`bTBaSM24N(NNnY{Uzd;)M{m{G$Z_EFx)Pw- zG9;L!+i-D5w>6( zX&<{2#7A#8^i>9BtZ?l+4efD$ob(gD^31!hZSQ9Dz}o@+T6eF~LFuo3#H;nYJIpXc z#>9$DPaNY@9B2tBFMu@WvkfcvR#_BiO+Kzf$#t3dcQ>2P4Q@GYUEm=@~JH`c??Y*;qZ8J1$Tu0V8GuozoP&P4RK@{@E-Wh=7-VzpXeV|A!_ zz-k_#{1y#t50^K;<_|8R89&H1L!;It`%Hw&-pTdei2wIgy5}I3WJVh=B{@!K;MMh- z%+=kaOMZdaX_&O)K~djqYl@d8;_ith)9){=H!?}+#{+ULq&ZhOn=c36b!L8gf;v?M zm5CX-KZ`fnT>Esb1+~uZqoSwZoQ4TnF4hMhwLY3 z;;D2)R{zl@Uhkp9f(}QnsOdygHxjT|ts- z;OBYIJdgfEqIalb;{YIFs`0igc#WghdFdTs<&=Ws&$?lm|v zz67d#qu!HXIw*aMqv~K8ledpv`s;b55UHu*ZF1K1rCw<8QY^ZN5*W@|>cwR|1>$P& zZt6{g&|h9PMApnqRMuI24usut16w|?bv_rsQO+rCLeZQ0h5n#z0~{)`4P0JfT3x}S zEtSXZ+!l@yWd5${EkWn6Vm8saKdM51<&%29cb)+ zgTvhv_}&yHl_ISB>&I*R_{zJ_eNjJo%^h|{6Yb1hXxXU^P|^?b4dD}ReDJj#gRJwD z(ppc=4UWAfPreqbaQG>1D>q-wQ!5HQhB*YB={`&18jxM)$jV>wAXpL{?JC zPJTFHgI<}a-GiE&+1Q)}FP;NL^y=c*sn)H3XZ>HXy!Fp#{=-sBXpcMcDlwCl0>u^A z{Yc5$?X>=zS{b9Zo5}F(1(%(PC)Lx6wMSXDykxW7I)ce%SXIzvU!Ply2+=@MQlcI{ z8JqBf269$sUfGoFW(lcnEba!bPUV&BvvN2u(x9cR^&~i$$N!0!8*y`T_iy6mmgMgqcy?NL5Rb|T?cjyohn-;n{t2`fyH%1?*%eS zeMW{ey*d_(1hkkv6@X`H>7ZTx1n0|(e+V? z^hqZ*B2&W2U<~4qcoZMB9II+eh2;wA=1f7rV)sLb_m}CjU!`OT&XSa* zW=uI1pAzKg--VRrg@C8rs;3N!sC@gGoNp9V4$6mNEe>&vyP*LYrAsF}lVwbRf&a5_ zi0s96omAl^`E*v{UD?ia-O$zBxpL2E(E8Gc?!zEG?i2uQKmQH>J`}-(ZKvTa?AWFl^giBNK`$b=R!L zY65gY7w^c)RN7Y~rzXrP4#eA|eD~8Ujjxz2#IebAy5ku4@~^ZV>eAX>9Nob|R-RG2 znRp7k&CzFSj+0A`r*&$gOI}nkkq4>AA3l0sG^FRs0BN`7~jkB{t>|4+3N{Sa* z;)A(SNtG!ZJD|JUu?_QuwA=wJRooCyrFS#?HByoj}s(pI_)#!VZ`GI zt9&?qX91uyisxu>b^qnN4ef0ULOm0XqebjclJ{#AKD2n_<5-KV*E zQ6-F{I|XPED7h?>@Xm_%(!YtV;sq%r^OxE5y8&3+x zokZ7zo0>UE5a0pbh-YK-7RMWC+<4?@g&(UgUQ zY`Y<0{q&DCyB`Yycc4_#^x9}w{H)&gobK+9=s4D0YiO54<3 zB*EEV%z%`VOlb#%kwAhZvWbZa#nkdb$$ENn)-lXuDJcm3(?f&x0;^aNA7mhbqlyO- zm+T|_g2o7KJ}8x6BdvQYeSl?!uBOG_h=J|);;D%xYq^qIchz`)OaN@1jCM!+ivPJq z;GzM7%)JgR(v72Nd_&+sqvOOnJ)gF-H8N{W8|rXG2+_)|`3S_u9g-#vO^`7bqGN)$2|em`#A9 zN8WyeEw`v{(``J%EezwI@AbowU6n@H?n$6Q1J}lPI2p2BM889O-qvF+`u*zUK02aw z4xj%bp)9Gay{#=&Qy6l&(GRd+Aku6#N^vo@&8A}48(vgYGP0y>`FO=zXpkSygQtu6 zwap#fKmdX-|M}6CQ1rBO+xNpv(W7Ec%0n;T=@kvuyf?K1vM`_Qh7g_mh*~9LDOvFvgsc@Bk1~I;FI#e1xOPs*d^jdQbxJJ z`$-4R@#_mH^K??qtJq@re`X@rcge_Dko|`lX})(-zOLB$6~PoKC`rko71R**N3j1N z(GE?{4pCrhwDQL|1aM!WoNQy83w#S|D0@!w;DDOEl90b#0T6SI8|~o7kMwgq+YSrTS$6c6<<~(Bu2+|?B%cBYd1h&?5gtoR3RVNpE1WU#tqBU2`OF*&QWzT8R z)Ai*Eh_v>Hsr_%EVOPs3b4PbjXiADXTg#FJVFY)>TOL6HUX#fs@xl@ErNN!T2FFTx z2xhA2BLwTM1@v_K#JvS2@LAt8oq^um7kOyx+RZ6a!V$-8gh<3&6YWGAlT3NMM0rLeDnD3WITV+X{+|%ma|Db!SRHqUKap|@R*2$n3sfh&Itse~E z6F9K~QoDb9y|Ok{`Si9)O{1sXJz#w-G zHn&f_d^nqH^Q^=|ja#H!A7E`3u?K454;HJ(S3EhNHq8S8``3PdI>w2FZHnXTrf1)B!=F1%{yh}P%|Go&P4CEFWJ=cielqIsYngm&d&D`mqX+qEW z&(!lLuH+}V(m%>Ydi?EB`l)t8_x5|e*+=bfe>d~5T!6!A87=V*&8-`)Xj>L$hSfnS z_gsO_P!nno3uu_J`khdI6aHqgeRb+b`frePJIv6wWl<#|@{D$96wJUiYjWZsP)5fQ z`b}Ac?IBRive+My+a#e9?c6=&?1|i9l3L2$aE#WaJvxC~IkU{!6B0 z_V0?C-ZMgby#0{gbKXR5W}oZ}Gdmb-iHf}tOrfEMD1%*Ljz8l(q(_t53F=5Yf~<)+ zr~&^F;OAZT1lO<+S9jgeb}VEA3i8M_4qJdO{cC2|gEHBj7XSOq{ud$DUDI0^??TkC zwD+t2#5lLRscT#*Jx8+5F|Mq?lm%W>KVlDRj5g5P8?h|VoiJ~4UgPy-w9Xt#aLyeH zGSct=g7duXATt>2{9wqA-H_G8-YLZ7>%bE^<;}b1a=??pHpSWYu8U6&WZwAQs@`nunR z3>>L=4!82!=zZ~;=At3T-D5P7Dta~XInmI%H4@2^m+JW)U;%uzcjLQZ^5GKu8eeq$ zy(R2|e%gLeLj^wJoGVnc2ft=0I_ngmV!q)983BS3(pUe*d{US$S6-xRgetXfMl^Tk zVE4+!meP@|*GTnP*rFCMD=P6qT}gzOm;LyFkH7B9-?iFYT$zZg zmSUjYy=fEaB$9kQ&FK2-a7t!uCQ)Z>nOY=ByzpP8-)tv7)0XcuP~9Rn?B13+0+Kzx z|2N~tb`p>M%vS0ELbsn=M3AA zhK+|d-r&GpGw|v|kuz>8q7fG@x7mbz^_v}VbNyOCq=%oQhUmv6_JG@y8q(6M>oYUi z$o;k0qvNx+-&a)xnvH<$rzZ66zlTJT`~jl3=&FVPXvPF!@|aN`5|;CR8TRZknT2BS zb(_qg`W~{b8|cTC-Dh&&tZ(-ew9sGD!VqvNbBBi5e~wakb#pU%WIQ(hB}GJ4=4Q>D zN@zBi^Hn}0q^4F{4R0I)qnQ}>IPylF=fd%t6|!Z1x2_S*MgraiujKdAC(mqFamvF2 z?Bq-7f#=-%HEtmws##3Wv7K!rSe^8HJ(NQZ2DXfg!3_6yWsur~&ET~ytk^&2tA+03IqzXV#Wl4=T>MXDJxlmlbzd?Z6McIA{ za$E3V!Wr)Ah3XmkwU{m{9kq4sceR@+EJ?W(IBD_7ZsyQdr#LsJxV2;;=k^+srT9$8Td3ORA=g*)y&e|{720(4GU zT5~u2aD+uvJ2?GWiRFn9u7vvn8KBJkVK!xU(Q^r4Ax4WQ3pKq?1lZo)H`D^{aQjT%OHMN%Dqi6PyXe|||Z zwHu2T`$&yb%xV79Rf@?jF7NY;i?z}3RVe~4ACng{$QK((yiqA!KLa;oEau98`U)ZN z!gADbAsV*bx~0S?9qo1XT;}ZfmzR>#Lri%>OuW^F-!yC8X1rQNo>w!2y*Mh>jhI0E zTIL+xrYz~!*ljO)`e=wU(yl44Y4v+K?R0G41iO)qD+?Yp(DlI)vRVqgm9a!%}|x+l8S4!MeiJ|hPCD5w}v1fWR7xFtic5qf)58v&4psR z>snF(2aZ^YzSo@qLRxl!}NfM z2uJ~)D5ZHj+zpH`Fk7R4G2(uIwD2V4i^C1%#-;iNUrU1I2g8RV@|407Fv#gNE8koh zmTf%K?t|oQAZf7LW430N;Bu8}cG}koh?yffBvD@F_xsjPy+QSt#y7MP zG;UCBrbG)+JzYG9E#B+@vM^dZ_Of0R-#-8)y{qWX31_~7O6!1KKO}3I{i`6=JT5-E zcOpyj(qZ4v>mtzFVl1wxGaeKZU-EF;_m}prKRhGvq6FobwT+CrH_H}{;1LJap%Xz? zt$y#EY*zrh1StDc0b5o`sd|@v=p&FL6U@SN>A4nSp5kD=YctMovmnX$>@qF{F01)vSK)o6QV}#8`e{DEm?>5OCrafN9ps`@ z@6C5}^Z)0vMt!vMSvs~y^|(vYZDcbS(VRtHNz+_8L(6i=61SPJ_WN?&0qdipArF;U zd#+hw_sFvP@kOaa`f+~%;Kh_D{qI*6>f*KEm4MUSW5!PkBB*m|t$o&Mc4P%G$x=`> z`#G-w!eeOV^zr`F2-s4tMsb%&;QbS)vUB&QO9czXrjx(t;$n}yoFbaLF&? z(>SpEJ(J4d}HokHp>50e*j-Gr~0obpIA1{ z|M?3HwcZAGh(pq$zLk!R6lBG<7w`HEKoo` zN4zx*{{9Eb(+Heo+?p2eBmMhUfNpTIzJPfrE4lrtFZlkzfLsv~qJU!azd)*ZwGWoD z|LBN6e`SpT4zmC6A6D7fSlK<;aHLC8Y|e~D&kMdhyt)_WKBb@VA7rGE!kQpU3V$b? zydoJ%b=7x&?8>%R@nunI3OzEYLscel7a;$2FkX0c5}vHTPF$l>vpG^pqN6cC?aG3C zaz;A4r)p!8Y1HFq;am>r>=wvPOqqvT&_)sx%9hqozrXy-j;WlS)D8+&stFyR^?r|g zPSy|orWwU4`R>KP+B!>k7M4izC^T%$5B!>QqJeyd3PwAYBN7r}Bv;Rz`g;lRU7JBF z7Chz~ZSdE8sAm~Qyz8Ck1FiD3Bld1S0nn!OUwMc{I~aC?dzA+nH4fh}RSs4jrYz5da!JTdcZE@XkA8 zq5aT&DsdBq2*^lQnVdS6-wy`+kRTvn4> zQ7TJ5BSNPqD>8nw@wXnG_Niu@*K_}iv9}Cpqx-gpky4~M55=K)aVt)s#hv2r?(Ptz z#flYoXmKg-?(XjHuEB!*r_cS}dtdwT@`YqFnaP|pv(MVH*48KQ$tb9<&k^H*#wUsicaV#9~C=Cq57od^7@qA8XMO%n!>dJ}JlZd8d-0eLZbOq^)a}JLji>a+&+|75nf^Unxq& z(!Wj*nD4IUlvwtamS)2-%TG#cayQ&G5leXG8HnW4KfFG0NIV>-Q3JeI>l9O~3W?FfB z?~BQf=r*nzEmFsTJCHAUJO(`lg93JDR&J zIRAvlDZrBfLs`Q`uGu0|^6!`cG{#}X^M3AI3>Y0=mk5B~)p-v!wA10V!~={Mf3Xw? zC>RRw050Sn<#&hKP?-WcZr{0|W^+;mO!ob5gUdl5&G5KSdHwBcfTWIc{a`z+PP|wj z6y9iF6Aq}O!p?J@B6&D`z(5TZFL&Q@I5L~_6D*hS^lt{SE`e?wNj-0IVWP+uI?wsr zws(H7>Tt_E>xTyS>CMO2!Ge%apdU%jFc1v7{a(h}Y)fjw=QNA~P9r}T_XwZ@DaHYW zJ+<*!^_n>OZAQn?0ibIcXKeyF#nTcl2pgpd5}HrlQu4dF&f8o*4FFZqeY;N%cngdV z^FDJ5()o$BhPiu`2@!9KcKzqgNH6C1TA!vT=URDbox_h*_GC$y!a6Z(4973SWi=DW z6abN0hdeu2Y(R0Uq~d@ND*wL9YMghei5GKj5lsOP5iDK4JX)J2@viDRyitX!m6VXZ zHH65`%`KVby;Zf%T#JbIlAH&rzca&J?De8_nOWBM9^`mQ41nNK&7RME>eb_pj^^~- zKjdnVZVFT3&v&P08mU*_Oh3H)`y}(!Sx7N(voNQZE+Y|rd$~X%X@+cT zEUC5WJ<%6Lo$MYZEa_VZVY=665919E-A8sea|4-}7{Jfyofyg(f#iiYei*93 zZ-I6xSaTq z2x}4tR^#qa2mRzx@2|fA%U{6|O2V(qnvemcQlNw)OCTjX>*IspF0z zt9hzOlOLY8q1!M|hnu$tgq!dmwr;IvOAw>l*6^p%W7{vy0YfUh0=eRTiC{bZ0Bo)~ zniWZ75{+C~LFFxs}=bmQ~=-(@)s0dzwp z&ulBX-k2!j&1Lb|L+khrB6{%McgIg(^{jYRUb0F1TfeZXozj&3+ZsqI0<~DbT~Fb0 zzctZ)nPqXG^F8|l-0R1~xUc}x8MWmx&?G;PlYu%4 z0(({==G~u@k#*jfHXL(zdo8HTl-atXt`{iqUw|;mcjtre->TjIH!46FvtIYH{nCFU z01)YPMUUF}ZMa=Ch^7QoS^%~{7pLi-$C17~bbJ2?pbir%zGE?lE3f}?51f@4SeD z3eQLqt&hQ^3*3|dA38?!dU{~M*!sZNXrig%c#~wFHUaWN!7Uv9|DKbIg*GnGS<|EL zG|*SMl?$5bJpNXERhtX`L-XgoZJaH=pSr(fXKemPW6qco!ZqfCv^kJ#0Zk&FgaUz` z-OU+~zcHCb&RN6tY~J~+_|MIl!k?!ykt1S0_S;Dwmqf;d2t@jk!(a1p!fn#H=P?$|E)_ z`vT(dcw^yEqNDc6!cZJ{{|qnP&3qkoAY$aNm6#B3D-4Jf<9Nf^t<}ub0a^OuwD|<5 zjxwojOZobENABDRoUJtnk%v<5N@XzguBuNt_J4D=|G}-I)c^ld%YQTH|6loVVvx7y z-|Y2&!@nL9$BTI~o*MI<|ALl!w~l~n%=F&`GCyn6Ib;8r)Gq*D9@|Rt_rI1}6CxtKTlcNEcTU>-=Pb})ymcEA!ylsKMQpY_Kee~TLbLkRCKED+g`>V*TvL0nR z-+UUFd}Bv3c;4P+aJ&=iwb%XqQPX^;oQ^C3B(XGqRUd4vUL;X}>3k7pFx4VD_guK9 zV5{4IJE~uKl*5zVY=u`AAk-66^C{-L#&J*7cfo98b(7c6$Ed*ke)Z~;ZFxs__0zj7 zxFB%+q^uLiN7F(FMINE}-Sg_4S7y6G@ttdW$J3nSL{3Uh7*(zIlIL z&7I?{87Fc5A*t9(8W0JHDd-!Y{SPHi9mzB1zyKa5{{~lc&46M)C9b-GPH88+#q8qu>GR_W!6YJ0s%UH|G|z~tjkl%u~K4H-$!=!mF~Tg)A0 z=#F!=r;crLd27n&JTtpc=HfP`)>a>L;|9lJMq6<~RI>AU6;iLsM;!=*0ZNWY=k5^=m4-8t$1qNaD~o2}VkLJ>oPc5vG1@j_+JoZ7MG@9yvk znJl_JlKUE{A5T?RH_HBl6Nvx%S^M}+WUUVme?Uq#PI6Xy@#qYa;gDaeq>IX%-*@w` z7VLWCh)SCr@2z8#dH?Q&ce%)HyT|lbrTRirn6n(~au%qe4w5sO3hxe!IFh@Ioz{pO z|9wlE^@Ivqg^^8PtO{KK_UzU-=<`E2cr#INx{%x8)CoBf!&#DQRA*pF7~f;)=X4&_ zvdsr7gsOo#tfjMTa~g+qO4}xiZ#V3->$1`9mlXzHTY74iJG}FV%o1^Gj6o_^-1Mw+ z(skDHf(I+&X!2~MyVX=Yfb~MB)}H|B&XI-WoO~iF8D=>@O3O5}R_*TNBHjo;!N7sS z4^Lwj&Lvw?Huc443WTA{zQD5y@5+!SbB+dPu2Ff__5iaqSI-Dd#c4J^z@$hh#HP4* zS|KQ8ZwnbDK$H#Ft-Y}m(7$}$v#g>cE9}Pb`8@d1n0gred1WJvs#LMQ@qrLjtTXCp$b_eHi8yG(sO31L=I95`3qdIe6#;!L!eazJd6x z6p0O%m7)@FEQhfC9>v7sTNFNPys3<%#V*mbQ)v)-JJWX@l4aJDTp5A?2qy)V8~VnM zA)r$QPJr&?^!OLz=l8UCHDKYMG%H}jXH(!~o)n!dhideJ$ix-MW_&XwEhKj1CYzHo z6^6a2{fspQ13MKHA*G4$uGPm~QI%=>3(wVy_p>s8nKSqkbB%ii3>5US8y#$m(voz% zIp7y#t(E;Cx`|7yw)w!IxNB~>snGuSoC;KRBbO@T4`ZlNKMWl5mj4l+;Wp09BR!Gs zFeq6|Cy$2x8C+)Y2c}kkG{1bG zmEH_}n5orf&adkUUxwGL$lmG*tlu`8+nHyod!wCov4~k4&_fq`d8}T2ILs>wACcUB z1u0p~JCCYoSNpCeGudkSUJnE`D8+ zc0nWedJ^r6tk~5?cQO_89jqsuWsg11Bfy4^|X1>yc_n^dy15My5@?O30CYZvC_|s}GNPJZ8P^eL&lct3_`P{5v01<+VqDRr_MJ zBSRdi#}}OUV^PK{p-90Xp%_hj-(R1YOLkRmWE@4^Ri&jx>MhKA``Heq{8^0(dNrTX zHU>B7$STgJ>!p&3yDK)?WK*8G7}|8d=En-q2}Ws7>l?mE=*Qfs5pRp5ny;MHKb#yB z><)+SWx6mI&Fo)n3caX%O0!=|v=pxh9dK)|Cf#df=LBQ&^F}?8(VZCKeY=X?nV?ubp>{`XW_j0ZUc<`2e`eg{SO&qAc_1WF!EemS zJ?k(C_+^TY%bO@CWCiW=U%*~(8;3;^jy2r#Qa#@zBQO1EN&q`LCkydMX$ZwUuZIHCbgg&9bGY_jGpT^)YEK_k6@3@tq^b*8=@)RiXXBn)oR~_u z%nYXBlLwxkz12)$3n8Y_ANr!vI+yiI-TAzx&L7nA5d>c5;A-0}izJF&g~~sO%XydP za!0Ny9%9-5cAsH@B20)X$VjwSMeLT*=l*!SIT4jEDLfMNLSPfL<5|@%ns$VFTaMW$ z=a<*lN(ljMLX;O=FtdTqYu$ldUaLG@W@^u|-&edF0tI26PoP%RWOnyTt6eQFGA_D) z&u2JGI)>7r0Te+*NM=v-UHH>!(dYESKYAF?TiC<%^QUO{kVvyRXyp)uDi_DjO5x*n z(esl|RozxMPU3k8gC2%Kt%7Y>I-tsgR7uS>8-;EcuVLB<5*n ztzBqZMa>OIB}8W!6pY#BR@=N!h|e$9xbIa|bJ-?%)@aj6TlAE<$F=GVGyzex_wCA$ z-ea|PLhS}tU1spxcS20d0l%7@{l^`!1@+~ov>jzV!AO&Ucg5+61Zz@3Z^)-S_wihH z^;u?jzX)5id4NIb)xJq5_5?D68awaT)wPHdMk4rqVUR=P{&1hsv*;$-=3zxTQClE{ zZXAQw?TR?I&9}zqYt_ngOKFw@WIuPs+(dH^iji!j*+-lDpr4reYB;Vywb`nmaW~n+ zj=>NX+&jnDzy$FO9y5H|LCVX7EOV)q>I~b-P|wA8-(t+1#d6iUGeCy1Z5=lndCdk1ASxomkZDCmKjy*=k0HohxS)jdQx(RoIF2L zMm(QJrmmkODP~czYN>0US6K}sxE7*=>`1FCV+80m`JFtMj} z#2OT%g==XMC!3M(`@?{2c&g_I1f#wP)1ItZTV4e0B9bu#e|$4^7%s~lV>79BRi>6{X;qlhVv7@QIRHoSX1+H`37JQnj6o9vv89wJn$4yFSU z%hNn=_7mLSGUw^IIQ3#2oLF_Jm~=JxA;gRqy0-fj(4&n-*&Y_HjriRr6=*gT)JXj6 zBavnFnX8=}F}cZ#K@Lr(e!%!@XZQ73%uVD7)hiZt3q8SQg3&cIWZK^Bi<b5)_yc=?B zANX3?ap)x|m~OSkrqQox(z7cr(^#K;1~G5-2udworNGFckJ3G^ z=!{`|8~hy~Q=vg={as3El^#_B>cMy1{UZmZ1LnxsW#7xHlMC7=Oz zC+}))xLVQ!s!JoMI>fK_fKzNYTz7khE9>PFrbPDkZ)+JsQmGw%wfHk>UZ5s8WwN%$ z_VP9pHA3+WS@@jea}2Wb9M_tMFfSP!OG&>!Ei5(U>QWSZ+NKSMvJE+PzdxraA9ypG z;7r|<0c5GXE*6g|3F-})nxpkPEI9jNtyucS&WjYx&`vb2Onlt54nM356=()yTvK&S zl@`B`SUkd+(`pWpw6BS)dH`^ec79g`$z8_MRdCQ+oQ<{?gf}F zDh23*=rf7-j7!~^r!6r&lqdd%4uRZZPgF^&+vE0jeZlvJDrKK}dc@1qY2mff>qJwA z$3HL^)DL_~Q_&umP7qnk4?t&ZzZ5rW)INf)7C5euOk>}iFDD`7#r{PB-Bu>v@V zXgLH015(O;N^XC=aaY(yK0Ud6NeZH)l_^J%FS`d}c2(@_lOOg^8LHB+w|76KSH-x} z4NGOVCa!7{{L%aQJ4)7|qg1E#60TV&vu|W^I-W?XWAK!oY)mk$%9`fNjhfZj4ZDQX zdZ|xxX-JbQ&&H(pL}8XZ|$5Q-+iI zRp5M%kIxqh!%fX{J}^fZuaG3OhvR%*Et`QUygzrOOsLjM&nSHqW{qC)dLRXO3lz&% zAKM0a`De-djFYiXw8DR*l%f?BWHkjj(>_eFyQr&Sr&wjEpW~U#8tXgm-9^vp#qu%= zQoKiE;B(^Y@vfH!%JpSM-H{UR;^2G2{1AW&W~r z6aHZ9w@z5(B2+UA!Nk@?zs{*+6jhqne)u_@X@3w?K+pH!5a)g0Tf+u$K$cK)AJy>0 zE!(Dx`=1@sfi|X9L?dXciR@2o8Ai8q=v8r5U%|R`94(@FQx~sv;!Y1I>g)X{ z7Dpg>cKMrKzbt9VTVZ_#nU>kv=z+qlZ(M1wOgFEkz{0~y8L|A2px#ePmGF8T|zbUr!aem);Bgk;4%O)gXQM1O$B9Yzp7 z@l;w-&li^M&-tMrx5=6ew(xA}xLxk)lbUaEOc&9gF!q$v05-e{*Q$!*;=mHiz^MIK3N*_UiyQ=%f&xQp#&U>m-hCIX`3hG z9WRDQCERO#rf~N%V)H}H61aeDy-e~aGe$OJvltbc?jBVcO2vemdqDe^JR$(LEDvW= zIqeQ&vY}B+0e*%HsgJkC*Kfa&0gtH>F$*kVV-ky-G#XL{H6)dqo_^tcJa{iTvw(M4 zG}h`s^X%=Gq*=$lUIi^gW}Ucx5Z&wVXY)I>Fg+^|C$M2j8J9I=tU*XoaHL{gj!OFe>GWRXF^T^yV613F^na*t1g z*IUWFEHQZB(EMXWwAB}UUX9|yhb}UK_();*bn}$`O~X`?k-#jD1#pYU#!&VpjpRhO zp5Db$FITl?4e@=+B>Ye*wwo0lxli?ghn^Ns8W^*^9j|yN`0OW3GMSW!7av0%>&8I& zH z4&ybSwd5qxS&U%S+qS%GWW#f*o!o{hky?{dgCUM+oaZD>p9UYwv^x_Aai)eui6<~ZRsIuh{wm~=|1w$Ha*i8SzO^bE2w zUuhRkiK`ts7E#j>!Y%&&#;E&1(@f9?Knzx!=aXr8d0R|9B(WpoA+zgc#vfvQXWurS zxWv|?y;nssG{951`86WtLLb6WfPQg3WMbCBT4(a(`w^PqOO1t>*Wu#kMAwNw zTi_*m#WD#mK6$t@aL-SCWPw1t5UTv9P1O2y$k22kpU>F7V{*EvuQ<;_K~lbJ&Wjru z_SF_wm|kwKyH;lOX>(N9eRx*Uy{SB|KRCoXTtRCX-`Or;6$S)5nh?}LysE0J?F(HV zH`=BjhbAcrYDG(TFj(Ny5N1(&U~>o?jujLwD)!iv%<8BuF_4I$Zvxz8Fkvi zYjSw;!(|QZJ4&tJD4X?6;tYV#sl}g7kr~{GKX5D;`T%(9VJ%&8Qbqr+Y=xtLU|6tB zuI=5dJ;_RmAK6nn>@)&9hck~e(Bu*5sAbeCgX*0Z7?ORQ*V-J6qO55I$;{D{+lchM z_N26A0-CihZik*&z$l}qCCSoq3B-OaDf&>XDrH{CSK`#J=vC782nR1APiZ$OJ!dPs zTd|ZwTK_@MN9=U7-4?4dS{xgCL=mfIx3j-4A8Rgr2n-srW8xjo7P zEl}W z<=d5Cl#{safP<4VZ6P8)p=5oO?d7#w_BN*u(DD|X&iJ5r;~9Qvme#`C8=bw=EXVx(Vo)s3!I~$XBbv=+1E@1k?6L*XlPg;AiB|? z@_4Q_Id=H`c`ZE4vj%rK0U59?3>`2ovs~0)k5?I0hT=*0o_1 z5Q4){c!DC(^tKO&3|}8lEDI)VOP8X4({7&HSo^9TKUMG3sDtaCRd?@dBQtJrtoMQU zA1y7o9}!jr>S133hF0)`ihKl|2p#QdD=GAwzxqy0PbuXl5nGGh9LUgs!~3*t)5iGS z@g+@{ks*&;(+Tb0H51{>3mCgi=60S+cPhN3xEnm3$W1AamAR?}+l97Ehh8BC3{Ck6 zms>l4k&HI~Zq~&$9Tw?hmg47d!L#+}RvF0=O7)yQ_G+;^@?vj$@|^wHjYPqBwD~Bd z^dgZlY&K$&-;$Aw>S$j5xn-umB2vQ7=!M?&Wq*p#)3$V@B2UoPp_Y)`0O>3dZJ)Td zRPpgWA;hM^sdVs3k;KxJk&Q)xmRym*sdCLmNK;Aw?zS>jo73h~IRG0Nz$lfzu!$9x z?T@DyUj9-n32V+CXi>%*+>o7J(+Wi#Np9fL^<7}AvmH#&FlSFNytm4of`(5W@1&q= zr!~ZQqmqlBuAtrNK)?!0?Eoevz&$J18LzHmI~$))Y5b4^$-koHQeg@#s>HhIO_hW; zeB!};7%?$pR`>d}!Au()_||ei2J(&JoSls{UdAsl#CZj{KPYPt(@S!tH%7M5QAj= z_1tbpE<<*NLxJOB6EC@429SOpo+yZ%{8uwq0TfN_Dl=nGBH(6$X?JTNE2nn;t`8fU zq&dR!5UB*cJp)v~)Rvjj1rBwpJQc*BlCCnV`_K@(kSHie_mH zhS<}x?(SBihS!hS78hu84||6O8$nb3E+Mh=|EY^w!)5Fi`bGO;55Dfzf}l*8WbPg7 z*@NDpcve-YYC9e7PyYhJj|4xA50#}q&$^i8ERDW?^gNmff3&02fLKxt>eFH`k19o8 z=R&nPl9lX)kA9TuHz+Pjc>4vnmb;89c=*9~^U-xjo$4i^?6AIi51Y&f^ne@vzu)uh zvG_*9l6F9d>&goY0}PRe-9HB9Yp**p`q{tb*xvss!1ji_Bl#AG3@Bz8O5L>)D*h*X z4dYd4mv~eUC<}N>Sc6S;*4(}W4feJFG4So8P?|X<3Y56^ilD6sm!&))(LLne#7P{9 zGB$fceGeuT^Vj3O9f3FHq7kLc{7l94@j3_(F{Pr__$<#1kn0#eO5~cIno!_3Eu-$Agek&a;Kk@<>g*#kJ}NiQPL;d zu>tw8-Pf*Unewnn4|)4%wVTn$onpJMKjxD>os6u6`|L07Y4OB!<*{gT4*`ZbAK|Mh z(Zq7i(POLg!Yk7_vsdXo8Z?uIw91~SXk)4LT`5P9oh~glf6d6zEb9g?_HctGPJ$v@ zoi_b+<_&ZiNQt!i?<&U|Bf#r@SvoyFB+VFjvMgu4&M}|)oyzRs5$-fy>lNVT+ZhLI zU#3>3t`F`tkDI+#chapM6&`jSDB*TIm#4aYP?wHcciF+qd-Ah~Z)Pc;ukp0!*nPmy zZ-M@}A{mf2_e==^MdtdZb#&DWsm@ITH3rA_Nf8$xzBfe1g6UJfNyl`*oZFs1!+kbG zZ=`W;_@hu-PB~O-^5DHa9EGG*^zNTx5h!63C}-yR&(V%S2SZC>Q(c^pV)eFUGAOe% zl`%Xhj3{9vRU1jUUFWm#6KbLlZw&aTn?i`C+^S*SiU)zM$mG6dYQDJsQ%<|ew!<6( zwdAT}13Pgg#PbPATWKNC98+0i(KRP{lRd+;G5YD3mMdT=m+@=1|7_9G{A76;ZITLu z`le@uuCYLijZ)Zhr+46t<$Xd{^c+KkCNQ(kw{VH=zs4-pT>Ry5Q4vp`JC;WxvP5+9 zx)&J@2cw5vS4z)CP5nyC<2;)+^g7)bXz%cak;31)jjM)x$1b~%tAgT8Eu_j3lX;nf z7Y3arqtwc=@GmlnhYPJ(Tn|n3Sn2LWSQ1;+%AkBP4}M`C!*ftPhS?0-Ue1Q!PvaZ8 zQ8Lzzh+T2u@{f0)9TTAj%5Q~e=Qo_8R(O_TFt%YH#&5U#6ihZ-Dq=&zkA@PRojNCR z+oNm=E^H{S3wp8nUvTpUi{xt^*s{8JrbkMF@}TZeO}>zZOSXtKu9$O(S-QZo95QZ)W8G`I_>Fx`+DD z&V|@S^TRX?eIP7GLw37Itn0giI0H-3m+?<#R0U4?NH=-`sG&c8=}W-t2n;qMOebp& zUYL~VXN-mLJF)6=13e(bO;nJ;qY1{Tm(p^X#cw64H|@!)&I++kW}W}R3QhBhGSe8v zP;b7I`};u&t~5NTe~g@&F(l~n$omL%NQwni=;4EWQNPA_kCTlvfh2@K@d1@!Gwd>X z?lbC-pH2bnpNM|w2>Q1Ajy_pCSE-TuIx+drKGHOi!!dqWp}Jlrxk17Ry(=laZ!@b= ze?O+xePqz^?1KcKI3@`e(&wo}c>HpBJ@Otwq1-EW=TVn@s~VLU;K&w zR$gqbuazp==VxsfunFzvUbU49TplmHKUW)a2uh&~Pqt9yQf=@X1YGc#!^>l`?uO(w zp3{e)xQA9t8=InQ`g=?aF(WjPFU03`xSryU87G%cXC@eN2y3?1X7Q7v{R-~vh^%x#t40_dl+!F z1BIAP^>s`BedE=P^Ny~5v!{f>q`eGJ4)1Ul%L@ycb|Kj5x_e zD<8{dWNTo{1l@h^mR^DY5n4^o`<1HL42>z`PV`aC<1yGQ2w+zJjiZ{Dm zx0n^H`8;L?9$rsYJ#}VWUpg-F__lt13-k{RB&I+irvUy!_H3;zO)Yop63@y`ZyA=I zxaW0jXGfRN>ucv$Yi3R>np-YED|29AUyFYJ@?}>^96uNq>^Fjnsvkgrj&bqqBe16k z7F@S(8@Y3CZC@WxQc_M+E4P?lS~gy<@s?QD=VuOE ze#O8{*`Ely24~xT?tHd08l!1Pk-25RU6Nt0>j4j=?v=^-!^Q?b+;T>FIIhd5^~m{3 zPi;zZam)=P40YxEr$}x)s4U;snc4oyQl4z$^^KLsY7F2@szovpeH&g^(1DTj#@Ude1DNr;G^ zS6xduPsvKZM%uPzy|&+FMo&06UT@3lStyoyjF~?7U%KoHtc#BuRPIXGvgww14Ih7E z4ct8Q3crdSy6Rl(t70^85LhDwdA2`wkZTu|6bCgDmo_sNJ={keC#o?_!@cd_R@%m;I%gmEgUz)}471>h(*tTfXnq}sTK&QYZuEk^xR~V%cBK`9PhUWl z)}|7djqipSOSBg2$uDY?=mtk@PHq&5D@I~vg$QzdJMn-3H zXhJ%v;D|pwL$L&`l!I&-t{qobX zfuiQ$tF^~95cs^_@Gp5aId27t#UCHPu4y8k+o{B>{{n%at1%46#H8lukl*Sw~aREaWg9gMRLH)BWzY4QhO*-|MW?*9@ld z;OxPb&Rn{1rbn)`maEHMoWvYIu&qIJ(Mj`)`-DPkqkk|Oz%Qo+jS^SOpgG3^B} z`A;*K-mJd*sfAgK_-n0iSziv@9X;mvPnItN*q4jvN|!h0%`uAxy(jfJ6(W{A0RB^r z)v9=k4WMMD>*=&cIF`XTzVNX>Z-3bh*UxsXd>4B)85nGv5r-Q1t#?ku&dg-BHtAFB2QM zufzjUyuxY{XGeeVtJ!kypFjVkI{373k?INV=r)RO&ZcqfF$GN0paBX=XDZnEtGe5x@wZ z0W?Dbn|WC2eapnz5yc^k#7mA{H-Z8kZSS5D5jTy>X+3v&%!a~RN+b<#(2d%}Cu3W{ zxW@`At<^c&C*{ec+fktX2e%>A_cECBq0-p+rBfA!TO?cDlu!6-^E#3MK|wSaayViy zeoq%C2J(4%FpG5_RbK2>2OB}%y4o%Tnv3OpY3)j!K=VH!2jR5`$7I!b!l7wKvwKud z!yRkRtP104!*JL8n#Nz_SEvNFVjHSCR=R&xpwZ7&{3MM-7X=ifh>CaPZb z6k@t3^$D^)&amn~HP!4tU_{eb6Q0<48v%6b>j9jcu0Pa5Y$%D-taK}|({?zHqe??|O^VMf*RKQdp>AzQq|37qMdgP{Bo0HgqhVMPL6n`n%4Wlot?&RN$ zg6RRS^?wsxL^{=thGl;M1MIMn<1xZ?kHT+ywv z3`Q+h{r}s(l5((<(Q1Aj5+y<#I$o(xj(=GLiq(Lw?Zd&xSNWem#DMXqrqbWj*H%gx z1cw?cM%b&JIxIQdf$)`4^r~p+h@nMVg2})AcPisaBti z>ZCG;ID7j4>qV`2+PQxQ$nAEuD}O#YF|VZ}hS+ia6*fDgUlnM*xP_EFebn5WtxYe_ zT1O+JHU-NK^E7$U$Ju>=Rr(*QdFb-^V3Bbgf@p%Vcx@yJ-LnF=H_%htL^y_-hsLGNInU8#jzQH$Z!q4J&ulx;}9a_74-~q zB+wTU%Ji{%|NI!D4n2`v&}t;{-+$$~3pRzBnvQ^4ud9h!Xz)bYWv0t0?5{L75l!rm z5hNWFEn%?O3q5+&_NTzr(d;{rD-`JHAUWb4=2n^;dEhCr=$fAw>6q5Z`K&C(C6sS# zRet%qEO>L{od9AUI!r2Nnx^wjoOYB(ADi>@4DJ+$Yyd8fT6&1+ucKe&=GGLrrn^Hn zB}+W6{wNdrgUV)MW<9@s5&4=te!3sFun*IKJHI(VE_N5%&Og$Rr~^0G8HIe=>cOlZ zzAj&Fud1;u)B=J?b+$A(8`#v}+3M?aam)6WULlJoxGfjSA8fcOaIw+N9YM0l-WH$f zmKgZt1N^WG>)F&W3|R`hgRjESAXPBT6w{hL;Sx&(u(5aq1NAI?ZS7l0BOb5o$4;YY z;n2iF^##`Sm8O{Zd)%gM>mi#o6_uHG-BnBgM15p#ifY^0i|~{&s6mFyDV8U7IXDQ_ zQidoJ)EuDsFA>nHo6j6f91g?c9x>}q@YHqwnl*gjP(l=)Y%hRsOle&46p1AipbXiV zKSNn;^fEQeQO)eW{nfm7v)tY~;^!<8z8(;xJo68YxZ8(okGFDuVTjn=z7&Fw-MRXT zd#k=90h9HD%l&I)ep~yLA@>ghKK99xN@-Z&cm6iKuPJ#xcQBuK9L)m66E&}qBGz`QN-K{P>sJ3Ea~dz6KlFOmxcl$f-(Z{F)~cH|GqQU+|hUzd;oj zBYLgt{C#E-H}+M1ytWdGQy={Xu(dh{Ge$NYZK%33*3LzrcwF7SlsdxbVg(NkpV8bc z5-t~Db&2nF94L6##d^{p+uPee2H(O@N1nK{0rGQMyyHPP@k1RJURqY8#u8i+EO~i^ z_D$S~;_pyUB>9|jmkNIU@`nrmW$Wcdcz9@;@IYrieK>i()`uvrN|?^){s(-zK9S%0 zK;NHqe$QUF_ww~yRD0ElYTc^#5l^!PDsqIH<(%?i6w+Z=g&+;$=PA2-^azE8wvLYTi~>o$V@USYpO zo9Q?R3zXl)W$AawjE@-f;cz5#9wX?1!Sr`7#cdh~Hwd!fWtnsFc1ySXy{x;vo+Bxo z;hOEKy(jzoZ0265`9#AL4c=Spt=7BkX^BLcYkD5^W4o(o!e(#xr<6*3CSYtn$xM3ZnuYfrntVj z9yw|l(r=ksQHhmi@6GKq4W`|}Pow|m*mY^k?RZG-cb`3!<?cR=KDX@AkeGK2e zqO@CL%#Unv9I8n@a2ktqV~(tG?&XrALpR9I)>nqG-Rh^-WtOz}o#}gb-q*_w zD{mgI;;Qe$p+v4`VG6@wvIY}4@3ZY&ZZGtonClsh{bBPcE@qiK5LC`cEgR3t4a*!$ zZ)9PG77>hiS#deP2t?COj^(f9A63t0SNal?6B^u|tv;R3wpa2ya}&Bo5!4`LVQiZAmG}lw9VRq$H)J6ph&7XDQ02J-l%gRF&ec zvrPMJd+M$Vlara!56r94T)7pC@j3C2m#@ZA>*oJV+ga473Cj7t5FPH#Ic}*rTwdh4 zzj`MOjw)TFN8FwU>hIqU?P#BE`0Z6&bZVhn{r=#|6#jn1e3UdC0rf5Fm|X+>eZ!^| z>&$(P>}~KaBXs|5RRV!@CQdS^Umse)Y_Rg?;-VHjXJp2!PS+3`w_0H&79W_gmA5lB z@116y3%N`pSQZ<0g}aYf@-0%wbnjM{+?ko}`nLv&;LUOcv(c#sMoob%u@dwg2x;Gr z;gpfR-Cu+l+BPfV#I39gI~n0D=S{LkDCdxIsy|$nXZ;|Svi1*L6XCFhqR_m2Tn%La znLCxi=^oJ2tAS`i>Hm=SR$*~9TiY%Y+=DxW0KwfgXc9cQySuvu3m)8^;O;KL9fCWd z@y1;m_-Ed=KHLAd_sKr#gYLd&UtP0m%uzM&=NZy~3EZbHGPE|r{1J^+|9>XHsYKhX z_k*J}jL7M5?rmRnrH!1)#J2|^naGi}PN2*E5o9g*s#g;qG&+!xmsv2_eG)t5yo01j z+0V2+R#FS=*cm^3IWSloE6MP7#F22g1&a)CjO$}gDKk&`6i7IfuFq%Y`I1crkJ)78 zS4ah~&iB`vFv6vetu>fTGUBj>a;5I~oq)N?z3>cPY4WXibr?B~PQpMDE{bN|lE$E! z0EWqBq3CYDd!ZDm|EO1T#_Aaop(uRD_^Y3^9#a}PehLt%o*X?R9Xf_CKE6=hrWAIk9)W%4$UWw$9A@%H77Z8o^s`JBk8*o6r3nN)?~=by zL#=>kKTU9>aY)FUPHKo;Z}fQ{1lFz*lg8+Wk;G5#z>r;&y_LWF&0(|YjwcjA8Ftm` z72!PP_~$EPf_~;`>H((UYU8gFj`;7rgP;SgrZ7-iz-Z*TboNf3V?isAA0pN>j75@v z;#KReXrkq_MgBKki+OL(Sa}jTmu|bAoX>k-YWccvPc}JjsIw92%nNmX;QHM%O;9PA#XjJcrUs z0CZ#PP`Nm^>`!ZYbF$W0tUeMzq}mmp&#FgWDTZzN_tuD$lyw23GO zbJXWE(8WbTI2lPq`wPwx%K>9m-%1sS^o~xBupN)2_6xhZl{9NcHvQX70k0tB( zI5O&+lElAVN=IS2>1dOl_$w5w;o8a6OGRc1`L6NnTjq$cTLjG2REsh}L3ZkBe;gGd z5im#0N$F_oqD;#qUv_4w!ih<~xDS<+D2%Yl?{3h2|Bgv}8GjcaZ{8+z!K1)7+Woz6 zsIE-z&Q5Nz#zW9Gf`8hI6u|?-c=S4qkjIGEF{lse(v9L-f=F_gdrm1oUiK%$_-Syd z%lgo`*#fM1_@$fFPq#ex!i$wkde|f8Rr*Qe;kLRq-d-fp06nhf}6Bx@>q@e1yliFTv zyG_Z28PwZ!+nrVy!&=(@Y}Kh(z0n;`{caA!_qKM_4UTsf97rh0?fCuF$N8 zNnfV@=htG2cVcrz!fB>nRA5)fp&z9#aPH$OZP3B!D7oVDZ&&>%*=ckYsIIK6t90e5 z1$frJVNKyq>R!)3(#ij0Uk-=S7Kz9c?u$aFi}e%^?2tNU=YF|BLU8JJ|V z$lLT6@AY9=p(97D?U4^!t%-v&n4*$s}IgkE@k%Z6Ca4 z5#GO^cVam4wg!DmMxeKA8kGMD#+{CJi`Ww-!Y+I$IA;Ksl+}N-dbJ`&lDUJ zwSj+HtTx*{n)mVE2+&%Ou~Hamk2P?L5?IzwuC?g(N%ryi_RWoGC3#1^vYiVyh$cX> zxxfdKqk25-OyhN|0U{+c;$l#@|5MQm-PjhrI}`Z&rYIJp|3^bSHA-^rUSY%wD2HMalG!y1-RMma zXNz%u;jdOh^EvD(0D%Di&5r>sG~0(W>kS`QBnAwNi*V0Wr590i>A8TsIj4YS z(Z@iECbA}vZ3U2Y^S6nL$0_EPgUYj!WBzzcP!eD~SP?*e&sETdSF>L~!8uR~m2c_!bidI=D>1J$n}!AY z^636*FjM}z)sB)4R4i+q+M=xy&a$CDC+7;ljen3N9DovqCw?6H31d9`qXM|XU&X; z-jh-bFMZ9_+Z=_C$wHHh=ca!c=_C^)SgBSIf zh(PcHbyPU^xH}p=>rpsTa59q#^Ab>md_!Nn&2FNluPqwwzX}y zR`nvoDlT!Ub3Ct`&dY`qhGHrfoItR<gPbc)<#9v{hTuXIVC|nXNBM;Ev0eY+Cjjzykzd;ssamN z=1PG&w49JS-0Q6RS$xPJ#S9WU=d#1 zQzcRG>Wrk}L(#3Y_K9QLPH#IK`Q^W_io+B%_nSH)lQ`a}{MkTqV%FXxoYo!9rx1t) zQzswN=w&&I$=og*fx4%DI(^;`P^K+g=v4a_C?@~%tMb9bFHSyMN?OeIpKR=vyYZGTNWAijx~sQA|a(aZ+grL-R*fkt&a z_WP*|XSvJOdlY zgx2|d4%dY@x5@B{o6V%g-k1nJV&wCuozw-a*W4B2B|d3v*+6~l6Qm|v;g28Ph`J%E z^3X(?0g2qCUaqk4 zwx<+cGuC0pB;<35wiAtZ)n_cbm0>RS1P*fNHa=^QHH$63Onmvq)7klOWQ{w`7)GZ`Akp=yS-I?T>0jOx+cHZDj)?I{!^T^1_t-^4?JI8sv z=$WDUZx+KHUAYLDdP>26750owLK-pI{$jDCQ6zNrmgRX<<@7KZ&9?~EtxCIhRNC#scjmEQ0U^|}^l#qchRhGOXP8*2QS#x}GR_yax$y>@bVxSa~>d0kNN2@+_4 zbdJ$>=O&?%nbk9@*PwX$v?p1sTAXW$VBtnGfI2}2qh?DmkI1r&83RxLS-F>1rTncP zYKuRO^WLQKWF!ll`S%C;f)!JHjmbJyB)L0z$}GGw^auhJu^gjMRIUPQbv6T`Rtmvo-V+A5tSqSJ&Gm;amHvp1 z8S9S$C28bUq`03K-)hvm@Ck50x8+-6z&+r>rR2?dWVzqgYm{2f-|^4mX>!tyO)TaO zz@V9)@Z0PMM8MQ3s8_~2SCY-DuXvEE`(Lh!At220zU41Hx-Ys`$(v0fx)KmruX(;^ z+1QzQB`rzbBYR?;ed3KzxV~ZP^4x~X;C^3lLNeB%34|Vq=Xf?zP1YS~bZ* z$;QGwt-k&oD)Zn$H@MSU)tjS8r511mTWS#p;}X!inqBoehc(MwU#Dk`W!ykVMEkWa z+NPPaNe?J!Y2hYxnhTM4*9TJr9k3&!i)Y*rw_cu%6Hyj=0?Mr4V4F{ZsArdkI(3uA zKK!h`%8}8iFA)A1)HYu|wHxRUVTH7-{+!d2(*iqlwJA{fqB}lmz<<&f9&?|aGFPpW zBp$VTdREgZ)@n$SlYmu$s*DavwLN)S8EOXWZ3Z^k=21*-n?+qi;NM7DZ+5FjlZdb^ z`KKnTLCZq=P^E-eLkJZl|L8e{8q=vXb%U)oQ5VD zUQR9_c9)Qt_x7EghUk+!nQ(yZvEcS2GQEt^b}KA&pHq*KRVNtfUt? z)#jE_qHQ@*@{67rEC;Lq!NwBz`F?|pXp04pT45xao|DBZRJCLiqxQcY9e==J&Fg4pp|rb}2=rqNSoKb^2t-i*isd1=BkcrHS=?NP>Rzq+l{>nDQcllS0**ee*I3Cs*jIsn0fyjDLQi0 zG9{{2rhi3K8e!wqi*1|jans0;GLwOg$5U@I`L(<0ba4CnFAPf*stiW9Lm0K&GKF|N z4+@Hem7{%`DT);lN|Z8+I1xl~C0t~2^3-5kxUXA*lkqs=uol?kjn0gpGJ481n?eJ4< z(D#Yv=T|eFmsiy_XHPtumkWzU2tO;D&j1$_%9H z@}&E7;h;i&?p!z5s^=oXkeCw^(Q0b-unS!jScp}L;+H%U&a4jEkUChaXj||5T^0CH z`HvFtsmU0Kz52!!&nk6JGem`P1+G2*J#Qcaga{-mS&x`UCy0FbrETHE25n=joI=%K zg0#@KLW@WVBCeumM@nG6qj1ahsYqMWO|1_#>YK}bJRuZg;Ti#%D`s&Y6e*zXH1CNVckFz+_f5E{ z7*e;uStKYLhj10Zs0@~_Dx6Y3II%5hl8N?#KT!6cn6g4ysjtc1gngNn;ISa(laJWQ zKURjrJOY2Gyb6|lnt6Pw++EyDnT@!$FQyQK6`Ur(fkll|PBZ4i+>o%b-qdz(A6;OK z`KfNNj_H8WyFs*)3NP;!oz`1HS8D+!u17>CQWyu@PJB{E$__ha zfMu3)ZG)`UibX~oR6fYby~P)`k=6Sn%Zd$FoynI*9gPUp;!2Ju7a{6hgUs%irt-=u zW8<9CL1A^|13$r!Ga=>1oU(U4h*$(8Y z<@_X+Ms+w)@@y0iHYCzNWeepMK|uZ;qm@`|cTHwO#B<6SW#B$6%M&xTves1{(s(n= zvL6)Yxh5B4uJU65oM0pL5^4px_rwsvP=RN%q>Z`=M}f2A)E)Pw1)q2qXdUrV+9fQY zpKw-xz$PxJ)QT4OMq9DwMmyw=Aa!_O82|wn(ZPcA{##+#WDE%f%mP_N@o@4%)CICY z>R_OF*Oqd!bKsc^91>TxzCWGb#^1*Qamh^1hW(qyn1 zGTq`~wEfG{-H>I-Ne~I?B+&oN8xakSnvpZ{`*+IzYs;aF96=9*mxL&c!*A(LE@*6% zn)Z4;)EHF8W~kSZalbu7seaX0Kej(rg{K0Ojgdp2?8!%`;|jvVNyKzSc#!=bgSo<62NPz6 zq6kDjExQtcgjP0!`yW28$j4+a#``rP-qrN>5A->k9hxLo|&ba{z@DL_2Y2(DUXl^Pg$ zyEuh*5_eOvnt*c9`ok?6{SMV+=?5^BM&ZaVd0XdR!W)ihPX9VBhI@Y^Fv%5#YcBB} z+jGSG^GLgtm7BKDJ0n6|p5X5X9=`;1&GsLscs(0QTBN+bV4cSG5%RboV$ERPt7rZh zW8qHF)^V*uE9Q2(_TlrSWtpw==VZ&PDkKw}Y;3pT55l6fa{(8cJ-?vf;4NN$FeJ_7 zx-B%Z=JWpi;9=U=XkVV+ZY?mmkE5)olsDc)KR%V_-N9eO#TSO+*|a{O2J8chhfl~x zd<(*`un1)e-T8*NeSTpq58C2o@kv&>C$fpqCtKvGWo|U2qZLOzxW2|rc$e<-D`$+R z@>eMv6A~*#84()ufxr2yAnum#Ba+-+xs_*na>qZ@51w)t~SeHml5uM#215ntoYWcsKQS*<$uRHAK*%R z2aWQ-BYQ>Xn!{2%YDv7ZG-<}5w%f5VdPfx=MJS~|1aoTxiAvOQrc{+CrKWfNy~M%R zfdW87^>>M-=vB1j&9^4I;Fg5|0QciNUe~FqukUkce3|uMX|#)8{Z24iWl6X6oWS0` z(V&j%a*8drgK4}>M7q;+NUo6A__b+%t|=xwlD2>M9fJJMhGLWCK&+3XkpbYgR8teq~YI;E4AQ8^XpS z9GmWvH#vI>u9j8a))|>oosU$i?MFS5dRVsgsh#lf1F!jK-?+(`w!bSxu>xM#7rZ(- zI*orF*zuZI#14>-cz$|QhEC+vMDtZi?R8RtxENg#`jO$LC18Mb=s!@85tTo43Hs+0 z0Pw`ubS4miY@0?{PNPZEy%rMyElxm$ZB0UqcqNrpP|G0=TydSaV*fNf{^e>*mq_#Jq54O}b?Fa8u)L9CwPOdb-9JqvF%1W}c`x&0w)-S(5 zweHEE2Uvg}*@BSV0JWaJfB%b2DDe@S-G{NtyH=w4;+d7-`oN0#z z6AH(h%2dl2c-=4Fy*j}^#=kRl3tBdmmKvSHu?BBSt2H|p>L0`EOmq&HiX)BMtjl>#==hx`|p#K4e)zQ;w~hlAM%BB zdcyRx*_t*h8~-`M?($QzBCGuNj*y=-`FjcFn-sGjRc?8zp^ncm88O{4&l&4AB%+|9 zsYro0m@BIp8@;}WC_uz^fx?Ot|9;JAWPb@){?9X|Wl-8rQs^3)I5a_n7WDY z+}F~q0&I2WOzM53GWPNjJ~x;AmIMCB_|RC-#evD?Mww@65O`~8CS2Z5cX;%Zx)~SW zzffU}POnzpn$zXLW08a@^2IEGTBQ{H?rnzmQoL!hYZeLbF|4w;cGfcq$G~UN>QSju z(#~ttl4zf)`Sm0!6hfruP$9NHKxkdUJ-C}M2pib_oge3`{c9Gixtkd1(%bR=n zo2c?3WR=K#t_?!-j9w;hS5lQW89DH9pQD?^XOH_l%n)i05n(Ikj(!ql!%F`IQa(7dYeol8yQMVF&Poyl}=p(eYdeu9Q z*+kmeZ@sC}K~ha-6tu>hnXVAZM$eK?C#R@zthKDatW53J7SZR#^b33a=ciSX zq-C_l;mb;9?j+lVTYIK%wS^;2+&>)I1Foy=++3cE!wa{aT+OvMjSE#4Hc!hdw_p?T zPBPZCWhH#gE~{#G={3s#&hTbzcI#oFZ>*Pms225xM6t_oQcc>epC1tprWK%hcW>5& zT2i}|)__GKDU-S*3!4z5fA z*B|y9IpyPBbVpq=wjJ`;3=CGp`Z3wPV|@m*-;zk?TJqR*N==^{6T7+V>-@5wszCP{ z65>NsRH=~tyc$D7aI6Yb1(|RLALVj|m#^=Q`4W*!5~oM#rY-W*(fz~HFs~SI~oz5Do*n!$^kD2Gy-uK2rGQLa9>}CqJH^Q z`xmr>@gN$gwiS3e)B|QN7sbLB$B?FvGJ*3q)~nKvH&U$&$zq)}z-9u~6L0@#@|C(R zA+>H-(wAQ3%A}Z(MjuKW!|8=*fyQL}gq7<0L5?{#R>jUt);S!U0hh`RR#Ahm`>sVF z!ie%GPqfX4%iBQ)Os}`>x$Pz=eAUIyL0g!OBLS2+rf7F)zWn|+ z6_3YW->1{};>In=c~$d)+V7b@mnYZ0 z&=URzTrAO;osPpKZb{11QE9w7fQ`!2Bnn zY7y9Aa;iXa*w9!U05nEMccn}N5R@^rNHkZvw89CnrhSY;h3-^#=LpWQ(6@X->xIA> zGTjx>Vr%C=glQ%K-CQ5M+l~HDm=?gdTikB?2Kv;~055sD$$>un#VWsl6R7?i`3QZF zE^0I=loAQV3&kAWgfr~Oe*a03BF;c0H}5zBPAF*v0IYYsLIyZ?x{CiYJ^%;%UkvVl z0Jcp3o8$e(|Nl4B@&B@XIbtR4>>%bvfCB_QZI#L5{~flL`MF8nR`0DsrL?y|oc91K zLQ#A_l~jm8w-Y(cM7wWq_Bg!Ym8B5q6Re9I%e!9X(MPG%H7Fcs%`x!{KXGT7M{C6a z^`GsSf+G_pdzEskGg)~h<c04m3}NVEz(2 zc*eUK9M>H8^CM&F<_!v5Pn6jpGn-@T^oiI23OGYFGHQ`@yDy}v zx&w;6@8134+eR`|APyVNU1H zEhbLjXPkC*WT|-=+WF zJLnFAmoPZl@UKOir7D21LWO4IBh0yrQc)x8(_)>?p_ZRgQ6x>m51%xDTT8-!+8;R1 z5~2d_J{%Cv9sA}DdUAD&l#uya+=TyfXfxlaeNz62p53$EGvK#tHjGqg8LYgPk1czR z#7)P6?EdKP_#Vuz2WEEoaIQZs)aYBEI9osPtm;k+12gaE_#5R;t{h1j5t@;$0pRi$ z$rvvfsiH1VCfv3Y>U*5}U1VG$l*G5JvyXi7X)T$O)d0c;T-{|k?Eopky2L-E<>uDs zNw^sOOp&G;YQok>hco6HBRxnt%)RDm^$u8M@mclODafWq#=Jmkl%S1Y$NoF%A2@rZ z!A`?wacm&abqd|&HO=Rx<+PB!-H$(n?IObbT-bX z2OznGWLw}_f%;f5N8a4%j6oz z4U*IoZvp)j?d}rpTA$!2&Gslcy1&F!7%kTNI$`u^z^NAA6Hk@VlqD?Yzz)9gc^;Zo zBl7jY!DzI$+-76&%~NHiy&U=A^QMN5LY3+KAgXeD==oti($v#8+|5jQfFx5e{t}EY zZ#9i;Qgc<#MIvbK=_A^UbZhM!e(M}+UHouc&`s_8jb72LLQ7 z^^M8YUMW)tm0ijExzGKc`~|>h?L)8@{NqJqpl3JT-%b|CPIG@i_NSf3Ev%dAIKwkr zZV-Lv#mlx!F5*O4eRQW!{?im!MTJv0?rj zOfywwc;bH3b<0kjwp{ppk^^M&l2co|dOuE)hVUxh(|N&9-KrTLm>IkRsmL47XBt*E znl}oxxCbb7p8)cZ79q-3e=1JazmWVWLBD2ubvA(Gj{+p!!j;{6V^uch)x=EUrql`p z=IBp-8g}09n7TUQs#p=XN=XJ4!R3fx^04}*RzoH1AnP+ALurCNNfIw4fPbqM*OUMH zK0hLT@|cqT7Gh0}P@R@c&YEOiUk%kbiX%YVx|x z{%v27kV73pR@0CAcJ-@WXy-2EDYp95wM8_v5R;cobl69OPpMPsLPe6_iq4ON?p9(&B1XEwF=}A(lHx#R@7QU6PGo;q;m}N;5A-H2jz6mZ7?=vrsc*c5AQ=*Giaf_OWuInx z7-H>ufzCRFNjln&1$Evf7KI$Sy)egi|X z-;dSMg>`)j1y`D(*|?x`=N4gbX-su-1{G>u3dscZin>ecs?!B1O^HO&II@BL*`e5E zGel4s!KDX;9fGLbX-B>tVQij0B9JT+CBg{07N6j1HG7&_ABbzT@0{Y9w{UJrBv22X zA6zfXA-`j*Pg{_NrAL^fhvm;I?U*Jroe<&4_1vK@a*-?bV-_o7-}l3if3YXr%!)C( zj5O4FrcAPfov@9>wt=|#?k>r#gF=;L{n*}|1d^=v!Ct5kuhM|k?Mehi8W+cw7VI$P zezh+`4T?PTpr1Wv`VtE(nrY9sxe`MjQQvWSvaFQz0rJiI`uYM@F!!iQhKtL|Xu{Qk z!|B||E>Hg|AXFVevR1c>VqyV6i5_*u=PSDZQYfetOSc7I4E}3riWk~mg&A65M zvkhZ?Dem0d;l+jNkjgdvd*-@pRzxu6Od(O~7wAz2^ZB&H-Cu8>s$=WwzFPdcxLEP< zc=+MI`!+<~elpd)FF)d!7{E%Widr0EsdWnHP?MAOPFusw4z2PtB_TM1j2)(yfBnP8 ztZ4Zy@aH;yu=^%PexB2FeGy0yuU2Q$`&NxC=-{dN?Z%%$h$%{P4f|*5gL{3BQF4l) zXTbKv$r>JB4l2)1`r%%rd?{{U_IjzdCF4r{KM}2?`3eAXuSwJy66}3mZ#>p( z45mJNpL>N-;)am(zpzxsLqs|9TZn*m7~scXO<>Z(qW_{pR@~IfbzNK|B<+aVy+P%t zZ?8$DT6j!~2oQe$2hkMjUjry%&Q(Jh3dpkLEfC6LSpF9#3knUEJX7@S*9oIOe~x5z z_4^U#V1yy{b$VB;f29?@V)1g}M`2l0DP-+q0;jwD5m$aJ@Mn2BR^=6PiLzQTFPO`#1=^Voft! z!xLK5Za!=2^Z{fMIp+n2b9HP?P>$`O69YP_?3v|NxPeOzF&yn}x*cY@RZg{!t?-zz zP$0br%7=11h1RutOtio>+E767XoSeK3lqP zWxMpHzJz4Qfq&K<<&7$R(In)yds)$ng>?t>;l?yvS9*jBn2Lx=Sfedr=L&u9h$3HV zld;JHd1S%+vqK0IKRNw`y%KRMsa`1ey2AcuWb)IUI zheg_+O`SV0EFM?JmzB;6A6vGu=9KBN?psYb&OT{Ozh|fnzcN4Z*0Rn40hLv4c@FSc z34+miM^g%J~G6q{pIHQnap5<>D#S3APVs@9%H=^c{EnmC(>WW->TOQeJraX@_1@~7*mTilWvQN4t8ko6PpK~2*;^L+k6~2uS!@f@Ry^?NE@qqQ z1B{PD>V>@5e}KxitTDsFAew5)JXvs9({3>Ul0oXlEp*}8J>xCQApBs{)fF_cHJ=5) z{e`uii;LV5*~l3On+bM2CXL|??K;O;01DiL*0l!hLwRh?9)19eX!NGPCp|U9CNT9& zvCmqx1ZJT=Wc7W`J4)h3XjliKj`SNkc*qkPdyKO-m|R0iGt9@lLcFVeYXmT{VHkr! zXbfUIuqgUVA#!)NFR$D;0Qb^0wbnlu={p6lHwVK@XN3J!`GL@*A-q;h?Ba|F=C;|7 z9t-0TREidH%Fpd@hi3x+h`|Ub;BL=RO%HCR<5!W2dOtK=pks@}Tk;K9a5i;;N-f#s+@vg=XgGFzlTM@aKCdP`B>W!?u>gx91@?VIiXJ5c(^q zO=|1jTN&yLsq^w%^`o@T*g`@bwy(P779te>~;0W{FoYrkxVL)&GOIVK?vkyZSGe>C|r*wxO8p5lsCTft1p;Tm(uDMG1nK0tw> zHw&4sNg$9U3tIQm>o!xFtRtNMo6;XEbh}j_@tR*8zqd)t;>hs$2#Fyu}4=E}UJtDaSw6(6!W03kq1^Z4}`YI#|+&|dHh z4LWby#;)}VKir3wAF{nKDs>~2&!TvA_{cq7Sr@vKYL5u6%2*_YCF1`^_T*sK76ywe zceP9V#sPS->ER9BvL)owm>jgWt5ea=aWiq+uIS3**veN(n zQ?NPNo=qU~Yc3+ikKv6;6EYcAp^0f+Iqx2!rpsow{ecf_t^))|sa#y%eNM@hUqq$x z1iC-J@3ci!&h|)7*<;!Isjon>)Px7;;wB+S$+wK>KmHYVE4X|hlJ)3O`JYCJoGSkX zow+rc%F`Y7V9CDr3F@Uky>-^}NmYzC6&4jnaL>%m4I-|R1cpkQFIt82a}JZ?@^5NJ zZ&yldc0mH+UBgqQwAFpMoz$Y$@6kcq8Z|<*mTh(V?8Agdpyx1c^|iM))A&g zi8o~%iXV@5kvl|u9kol{nXwTzb5VgK+27t+?ldl)Mh;K!Z`QRmr7DuE9|xfvLWGZ&RwmI zH)Y@^7$UPdeo{{vob+cloIq!4a2Az>Ud9t=;afyz!sk}{!zT>?8r45aXUp`HHg)q^ zC`S@0nWvFh(KXs&d)%|bgvh(Kxm>bI&X_#;3@>M@%p?+2|Mjd7n%nl1)GA-HWxd|k z?!#585Eov~ZNx6|XK$wRBl4ev-66=ULuhD4u8BugDYh0oh>i?P_X5@H6Qtz(#nlh& znql6@IA$Uuu_><*grb4licGetmHXDXyoHZ*ZPm)ebIT$%gBXi35rx)B6`E}w?B}bP zs-Rh}^3mq6VMvZ)R=^pgX>7>(Tuqq(i(Mo@8(l7UuW+)~yC5;R7bkeO<}Q3I;1~Jk z4G?qjvX{F)mkP8sN92(Y9`?_tvgL9g<1U2^^@2qNgLr2PUI3Eo18@JA*;*FuwshN4 zJ`?;Uw`(4C!cZrNBa;grw*?IsM3vx8k-$J(`I<`rchYVP-bJWZVX{^=UPKr+xT|GU z5jI4PBKuV1OC)l_{(6EubJ*)(=e%ogy!^YG&x3A1w6{5(FPDyQRQ*+FI3G^N!}-Pw z{ZQ~bu|S^H(MFp^BN8$K7Ety+mM%Ge}9L?<2z$WG?zN>de5iuc{ZzBT`r(0mc& zVSQpfb%DWA2FB%nF$QFh%!bQ%rFhSwMx&@(bG=3{=yOI^r)0*=sd#s_iwic=Y%DzGBqT}+XVVW=KFBn_S8nkQ z^ga;rida$|<_;IQ~7ljHdzw# z(8nF*WJbN=x}Rgq%C-kT^?ajubB^CB;|{lk*K*IR!buMB zELa2J-M1Ol&sGSg_bY{QpitMvzwtSLH-pz1Ny#`zg=SML8j`hneBsDR&wK*Gux|qA2c8s-E$6@*MQs zF}{gX>><9S^q`NRP$^A*xM;&smsJ##`ou&VR3dUQSXt*{c+~fG&G53yanDZtu;PF` zYT_a58zS1dsBwuxKn>35$JvpwxqBzk%37bCH<*A*h-H#yvPi8`lb(!O+@GE)GqCDvkT1qX9Vxuh*loIxKd9QAm^ScK}xEzj^H zd%Ww~$6|hEn`DksAT7a@9wKfe>yK~#%iCL2o_+|BSo}?xB&c*c#6IHl=v14iVnp#< zPv?S;a6wSPG+q>)IJWu|=cQ3cP9e6|DOdAxMm|R5RaOdVDxKDhKMw9iIW0!IZESLi z513Z=%b>)3N!d)YDc~rck@-9@ZP1(GQE4=4PewB~y(HMC=_0X3=D9u?QwVfpYBhc| z6@O|o7C@M`9+@|^$w50W=Vp!b<0BXRCcCNr7h00UN`s$=6+2(VN$~rciN-~hoapP&rYpr%PK(BhVvD0qG>R8eM6K0}_6p%m0;}9FEXw|MXvAQ?60mqym z&U5Zq;)QXoP9695kKeWX_clGLhK8)QH<61|OUGa4H9+@ux5$T2vGJ9#Np$Q+wvaU< zKq<+!0OTUGPM1EyEg56b)2M{9COlj}i{v#vDzUX7V3rj@uQm^UAvLRCdHS1z{9m>o zHl-WH=Ko#qTdPYGxTqU zGgqqR+O8J_hbQ9U#C++g!;}(tj@y?n3io>@mVNk>jsrr$_xEuU(&&_4Kk!32HtYSy z;I8{wZU2h^3#X-F&Q5J{SkYaQ`C_`ODC*|PBPQ1OJv3lX#DS;Z(^Zb<_t|~@HW<%ynb~1@K&^PN>czDEnCkKkhtF)zd zFKo=Uxa%BwLrAhFP6Nif2L1QqpzZK*LfL(=0Pw&CKdL3_=l`;2y=zHyaskt3HkDjO z{IC2h27GdkcvI-2>;wJ^VU%u0y2GK~xBxG?qRq1>lX7EjLU|arEl|2miU!a%W zs=K|ERB+2oWkUMR6N{pT4Oj?p3TJ z=BF1&J%H5<^Uks#OB!bW?teF1p)*d?(-*co85v!4tb{f#Y;j&{Z#6JC5Bp;!r@S>k zQRQmACe^R#CMVN-YIP+wuW-j|q~rW~EpLEjl6=x`-@wit(2$y(@{bRUNm_`#`~ICo ztUp)dmu@|u^F;LNdeDJR^6ZphJ3lB32sz%zmn^aJ<3fhpV&Ar^q3a09xIG!X z1E4Z&uHb_W_}|P+v#qL9jc#p}1-L-1;{d>66wq}_WUEU%3*FJ*y_-&Oll)BbdMHbJ zV%?JOOjGDjFcU--PQZ@1C{wEAd=jB=Pu}1i|4DA5KvnU*IG|a_;tJmSjrx0@xnXo^ zMCr*xD#V>a=mKnIJIP=rlK<$T=;$U=x>%Fvt0(Q3^$h!u3^%bO1?SuM12oz@(y*Qn zVx-JJJa#fKATEqzi?##>0KNy_54|1sS~#g?%&@Y~=}VF&1$eyZlf@DZBY4*W$T4@< zl9`~t*wX0#fSg*_{_ST|+X~BYDvEw0Fg-I6)FE?2_^EUoUl36pLqF!^DJEZRx8bx2 zOoSIJVmW`5rl$8N`1U}oyyBTeL}O?Px~QwO)g~&Yr+XV!-_MZDy{!H0?4gIm?{)eU zL;#8uA`3#l=)X@Pjx7eF0`)o~InGokFht1iAc!Z^DUq$JF8(D2#?6TN?26FNx&xml z_S<~~OFWS3df)epphVrU){=?;7mg~ZICeqkcIMy8>hi&aviX69BVSvh32FA{*J7#~1BT|TVy4Hezw zI9*zK?%wfOs7h=Iq?asx@G76GJuG11)3+IG`4hcgy>j+*(u*+S$cl&a*qp`c&(2D; z-g8)->*<>`^Z3MNwMMIXqJ3t)9sNmB{jY`Nb~XQ@8#aE>(_1ZB<@?QY3thFRw=80OJIg1}#GWNpY6w?@I%SBt| z6+c%@cT;-}pbzdu3U{V7Uo>QP!=M^emj2xZ%53~CQsS0ii54-L`L5af{arMfl!OUm z+{P}+>N%2#3Goo})&GWyb~j_J*!a^datYhILRLoao9o6033%0#!LBzL60?W6+^Dq8 zcJHbELaIt$3SRY~pDp$e#z%ry{~AEm7GMuZF3zN^WwO}|m@hKlkVjVs8B5dhx4Kd? zfcClrFK>p#KXtlf`O_BC)mL0PIU20j!!G;NqZ+Qx$aLIB?<#9HMWQ)@9IS*1=e?`)(`Me>^jG~K(%w2Qs;=uB zClo|N5a|+??v_?kr9nES8|h|1Ku}7gyFt1gdT6ADp}QM~p>yUvc-_~1U-$Dqzt8WF z_xv^Iz}frkT5IjKzw5i=sJb?b&^&)yn=O8BoixaQ!IM%0xm8P=KjF2v#L<+uv%us{ zdo}AX)WGigbNG&fB2sp^i02Qt#0_h(iuVzsl$9Nl(5YYuY`QO=hRRQ3K(oRxs6jS1$&)!D?+ZZ0WB z;hOXu4-_tP^k`yFWn(<>5Bh3*>irMl!jX0LMFW!_v7M50m8^r6&pXz3m-Myi9WxVA zA7*Yod;`Yyde>fe%-3gPOvPgK{XT{Y3CsYU6nV1vvT4`(%E|3N2OewFMg;14~5nB2p7L0F|9A6A^U4-BhGnOF7?UuBOPPg&gbhi;knn_BqkXWK<`mTRzr$xv2>`9IHVj4#DL&Pql2J_f{Iiyk~1 zfEjxJ`AGu-yhPrt_@rD67Ptof0?U(U=hfWTqZrVy@}0E6QV8M>xIzX*A~h^91FGx?OA z*mv4z!ZU~eBoC|mZg{vd8{ccTye-}Q{rS|!PM382KSM%GUjA2y+W%Hno^&&juzEVO(io1Gg>EIy(W!=m4BILX z&DYflM|}Or`hfgKnsio*%hz#W_yuD3USQH-A+{Hz_6G+(VTyjofk% zM$lYc4$z1v5%VPgdElmxODu?i*S75I_faQB1IEt&aa0-s%sYy{``aUVROxABl*3J* zvr0a=ha-h)BLr#a0FuGt7!qt$(xpk2B)Xuwew@7zIHpPc_i4#@uBMM`RKa<^Mvu(m z@g;aE_R+n6znhQWwHRip4*HQGM^=DxctBJJv~$j^0W%2oBhzXP_wLWDo#uq8>swG< zFFz9p&o|$Ldx~ZHj;Dv}09l^SH%0v>$DO^dyyF5!7P~2hQ-BDO!?5N2`yZmqfRXMQ zzddf0M|ivo@aI96N%W6SP>j5fh|Dyq-L8yjNshmgJy{wMO(zH1w8B0zu05FOm&P=a z;CN`;!mS(XRUhwygHkXR(8mhw>G%iC66@s&v$MDrX|if^6I%mvm@|lhiQzlfj1I^8 zx@LVK&2F)v4b&>+>Vb*k8!}gZH)v0NO{h_p#-h{uN*YJ?sYq{qkoTP)%GH&FEI!|F zWCq|HY^}|{dStWZJPGCThnEO>Gq-B6*Q6jyI@vl)c@W_W`{|w~4GJRW+d}q24y%1s zF=mu%YErPto2NsLk12^Hc*Tbt(H(KHa$oX<0e=IE{E=|PjqWYrhw^IR7IN292N-T{ zo$|Z}NO61Z;sLv}?{1)Ae&oZ~=(2?P{O^<4f-F(dBv!E^DLYP!>j8ybIMBi}KOUgc zoRtc0Q9?CB9Ge+VoN;RI}iK>RxX+7Ui!k2E?c4=BCKEp^d@$IV^-31`-Je&4O?!@z9HXRrb-n zxCF8(c$Y9rc&boBW@z+Xt%F)i{>$M$drByK7b{eKI9}_|-1GjgP^0?*<*0=PW~naU z0YTvji}Vvc4l`vAlFugxK@e}@uP5t3XoICE?GsuZF?ANwE!cp=XKx>M&D&Q}!|il8cT;i3BqnO^=jh;=Z>!i&Jf$9t$@k36Vq% z3iXg3*&+m+L;QD4rhO#k%b1>&#f+@n8oheU=8L*o1{TAfT4N^RWS+mdCG9S*&E<~E zvmvpPA-}yXX~qaK#T}U6osu3s{K0552a^X*R&)R)kEMhcbiJ+eWseNXxB4BPnx0`s zlF|fzMXF4+6V0PPn%QzOAd;PWP2YO%+MH(?Az#2g4{x-<4BPh|!Tjed z71Np@kck8ks2OIfkvIShE5FW$7jm-+Oaw}GYP;*hNG=n2ekVL(s-_qK2s|4#O5=J@ zzq@aXNC$gf%=0{eEHTbCoMyf^9H^b!lkv3xLgq&L%82ICG4Vz1F>XUd(OQA>RxSeA z`lNSDvsWgnZcaZ$YY23wcrvw|_tVTx%0n;#TuHjo;blHHl6NGA8c4qzAd{A$WKRyRi70PP@re=gl! zAVb}pFHtw-C(glsdwNVgX|hqK=et-E2)T8tD&dmX7gS%X7-BfUeJ?*ggC~orz7CnB zPv7j2T8*8FAo?3c6&44WIR7@p*cq#L2HSeD&%Zpk+?;LZF5R0sweAPt0{asUft8OQ zDaX2{@0I^lPEA*_OZxorkDVl)gI=<7=JXFM$@*I;K@_*qR8Aa{ooQ6nfLeuont$Kl=LV?Kq%Aa9|6|o$`raY$mDWth#~)kjo6T zQc$Oldk;uy7zgSTyAK=|p5Jz&t{b0Du&`2-xYM|Sj?@3kNKqskD?!gG7pIok$=kgV zrUfW~jLy%tWDJ1KdIXEz{Bbx2SJ0T)jE9OyQArw;%?8^)V>Pe%81 zK2CVD>lPHd=i5_C9}yK-m-m32>a;Vf(|P0QMIS61<;->&%iY-af^&wO7bre6(AfeF zGqSx}_5q;9Eaw|`^7&|`jy260>40#R&0)&r&Hy$&4%?f-v$@VzfNnDWdrVyS0t{I^ zU|nHwyRuEdO?rON?D)~;j>PxiV^{E{X@+?3^jdSOP?LVryb<6P?i}w(;LvRP0Zco9 z{<&lD0f0c3Qb?9F0*G?S-=XEO{q?AG{>{ICOG)w;wEh6x_PKQTW!6=m{Ri3n*Dt9s zhC3SK|DdJ+xIzD4A5H=zhMaMV4_fQLJ2kL$!s<=9oQ!{OR`rjHz+LGMi<@K1RDb;o z00T8T2g6umQu55HNJYK3h5q1&0kVqTJcE`YfU;rXZUs~rWMlupvj5;^ zr?{ypfuorJt2IFt!2sjwKPmOaP@Vn` zaK1}KJpAoCU4vdgZcvDm`G1T;5dO~!XSaRTEO}d;o=8O(r(FpRnFe5OH2;$y8?aRh zfdiVA$vn|vkjHyqar`5`hLPotw6~M(|65Rymq;!?6*|GqO#?F&MN-91OFVGYB`Y(? z+0vt*^b>2V&4^+F#A#b21<^W@WmJaB9Y^qVDTRS($4h4yFkqF@+86$l^Lrz)Ki^xTBv8(sJ%l@P4BuG+$mhzM zzvB&^0*v>Xixo7gPrPpOkSW=c2c!zdNoIhul=mMKnv$AHIeM;VPm zljGh~zfs%wXV!N{4zQ%>kB;dcf9GHUh|d3B(vx>l6|_=eE4-N;JNEj&>8TjR;r`sZ z4`>fOF27&`_p^cXp^mDDnDQn;jqyC6jEceGhcE`6lHDyd*%=P z?8`o$`Kb@=P~drJ{LWgI-I{qGQ;Z!kJ~?LVa3jU}!E4&iv_YSU^Bs$(4T=>&cNSU&s@4{0{21q?YVJ9xCaZ(Om$8H zb9*j-b}YRRpY$6!GMBT5ug3FBxv$Hf|1HSwx!Jw1vEO`UwV?iIXCj>EscAJogxbi@ ze0TqV7DWM*1m)pzbsVsBBHq$zqeGc-=QX%?=ee8me`E>}{mO$-0vz1HYE7n5)7H@b zdG{UJ0xCAojDgnF7S=ouslq{u#tP5NsD=en3@`=De612)m*P!QI7 z^d?0^8GbfD&|a^z@9|^(e@)xFIWlW@^lZZ6HZpBo40hDSbbSfZ9KJr=mnEV=A?;cj zF5&?DZF{I0;5$BFCF9~wkB%07vO8~`C34*TZsPV>@Ut#`h!l^psmOB~Ji5Lf?woDh zWU&l&L%+5mech?Xo{Qy9lsP7 zfEo2~P}wekG%Ag8pV7l_n(CXlonyHR;36U1jM>6seg}0r`84w#@q@xsRi47$P&S_A ziOhPS1ty%R_T+oI=w6~IXH<oi}eU%iSt@L0s>s?}GsmHeY_c_Ty@ zTZVzW0kOT{g{4ic^Wo|62YW&Je9GqUz>`aaf?vcrd?R0fRX?CU-Gcze_6)EazdUMA zxKqggiP1XX%s@mFjKCa!{62v>r-Xh@Z!_*Pj-&h(dgXhXyIuL(vWP)@s`DC~~BI$7wty>Z3Af`|n9lV&g5DnAmfr+|x!!Rr^wy5$Y;3Ni7e2NBF+h-hW~U`CL}_K+=lr z=RtP%+$-%uJjufJBYNSAl=5tOrNSW>gE*SB$f_!*b!PWar@qr*;0?D9$K5hRX-z%V zs72ZPL}q!h+peTI3Kxx!2bDVSU3pU=H9>n6>DqC|eQ%ZMvHB?LSK=v#tMTXXiRQJ63zbmJT}qtoiA^{qZj3_~h1E8)yeYbJ*0eS&p*Y&Os2Sr67HJ*ZQFLGAsuPFW*BG^! z^{004Ld_%ZR%iNRVnyR`U<~hL$FzP)hw?UyHCqI~_*j$b{w$}`?zQDTF{;5$HVQkN z)iEn9N|BBwo5vRq);(JSX`mGL_{o8fX*MQymcOV{Mc?j7(< z&FPjr&wa_A82l>KdEJK2iQQJ!)P7~fO!UfF)GM%l-ypDUxN>NYo~^IUI?!A8@K?XX zQB7gFk&W*B8NaK7FLv0^vM!d_%;4?EeJ2*H+t!@5DY~;c+XTZr<;mc}_s3$?o~!y$ z_mLRyp&GUx#e3nYz8x%FD7+t5$wstoY%LN3nq?WIZnj|}#(gw3hp_JbTBtL;1U|Qx zon??@VFQWNI?t0u7xp$>3RO zBfA@CdQfZ-ach?Ftpe%*@)m)*AXvhO$l?6ro-K>dcNKLrY|IgVHJtGZ8HqBzmn_gl zT91Kv%xrJ=i>oN)!Jxg#uy!q|eiR+t+Oyi7L^SC0s-?~{clV~i${%Vz=GqV?} zeRCEPr(~@$PlmvVl5uFiX@H)!uTm!WPr zYxeN7=AVw!==Kmk8d@4*px*q#5ut=_d4*?Y9%LmAOfImDG@03FRj9ZZx5q$p_}6W# z8;hg`(lHjcw#-SDB95$q%pT>Uf|7w(()HVK6`Sz_TY>=LvPzgqWEM;Ij&YnwH5;la zN<{vo_XpQGx3&C>=0Phnp_41szW#njj&U^&9SLnKrggLR_sWXW+E{sf_qO^@Cz;qv zscqRfplTXL1p`JR{BlI=7^{^DPqJAW6s-!|>XanEaJ|&F&YV-O_4AY-=g0L*+g6iE z%Bndc?Z+`AzVrf0k5F2Sr(lGVs97`e`MN_rOV7A60s)C9^iE-xk%ceKFt7_+v%Vgi za;BS9@Zcdu#t%NEo&2swteX9Aim1l1lX>~gDP^5^+S-Ea`3!<`r`-jPED%urwgc9U z*EIPA$QIGmS3Y=M^5cg{F$E>EJz=|5)K_|0R^)| zWrtrHrtP~*5zp-}Uaq9;lnoT|-i~B-7kY$6Rqqt`MA#h`v@E-jEs=IUP%~_s_~}Jt zW4D9x?)I}R{lL;lp;}^!_WBw*sk_k8ACVbK^b?pZV-M%#py9&oZ7k@E~Nr!cp5-xf(v`JEglYOyU$;& zF)Xc^qeWqAeL$%u7Q-=q{DG`t0VxDwelOfiMLJ}~`i?)O zu^v2eNF>M)c=)hd@G4O76h*dLK1*SkI-HXsMek-E{S5Wnpm1US>->qcIGCa6$(e(v zahs6aZED2C(TI-sYD_Z7;41ah%61Ymu(rG!N`ZJuW4THu#-09_ho-CJWUNeq`eD~( zjSnsLK_-S?Xp)FpJoJlG)5y-t&7FEu z$0AeCl3kPgz_*fEUnKwz%<4#0U#_bao2?o^zj8>Gv;0hCo($}X2X$pVGF%;p46Khs zTDoU$4J%h1O%(J#)YKr#y`4tw8{ zHvpME)mn(NIG7~Ld8kr`Nb$!=4Ed-F%6`NbP%12{Y1aDnx~|4T04W@sj24o2Wqi`G z8`tw=l1OKbL`BslVr6&3Syk25AgL31>nc{RaXql{TFpL~sZOUz-~a+NqPYxz-jywn zZgK>`zjW3t7B!){(0TRY&}|J~^Oo!CSzZb2QWJ;ZYz><` z+6r?F>hJW+Xk^lrtTrtF{b|lkgi=ECH99+27+{UPThQfCs2{?-n1%HaoGnU(`%a7zSi$|1Z*mKRz%LCWpHX554P>O=MDFMtXoeLLDD9f7?olrjXpW;y`p^ z)`ePh=`mNzFc{P`v>!&gjI}jj7E7o9bB;1=8L| zcNfr7Nax8{l;sN_#8v$)Gd_S(TXg;=kiWX6D4Fp|)q`e+HoNmii52Nd_NKi()~GTA zawD>(^?&ko_C!>woH(kmTTmqU14LwrV(4>~91|27Q_)GMfQ1}i#0b^IpaxM8`1{(k z3+|(ligYyjceS6ISzN(jhU?(y$Xq+ER^RB3EgA$`M!KtMYdGY+n6ITljT?24i&_u^ zb2y8}*kjRRPEzzwW@dbAQJ{!H%t=nABtJFQ4^9{OR}jZUiJF^}Zv3kdC6g>CuDpln zpH+|53IvK6*(AzY^7G3lV_OiLXa$Ug9`rL=wZ$~-)Ga{nnBA)g4oC&>O_W2G{VA|x9JKCul6q4-^PLNQ ztB6~%!@G3qTmlgNkfL4*jG#0Sg_d+q=9CtT0=tRG3mjriu`H`WMg!#Pi!1M`j`3BWmXj#?uhg%{X7jh}4_*vI z`iDvvE1h>62ly0{Qd~Uoy=?pD1*=9{c~pJ$2(Q$T42NaTwwC06ab3kva*}u!8v%dU1>2|!!pY7 zqJ#ML$fj1x1rrP6igvVc@K|jZZNBvV)_QQNs7~Y+jOgd zcKvvga|WAIGQ-@^7`P$4*q}MrJa9OKEv}?cb)TN^U&InYz%NNfJhsYhrjjiCl;n*T zk)APU%SU}nrnRTw(&hcW)4^*cqs2%N{EQ-_${A^o2fPKQIY;@F1m+L$r5U#H??R;C z!erbK=YqCgjC*;=Dp)^QtERnXznS84BO+v5#}lE@192A!!2{CCeg?r;82iCax@#~c zqs`CAD$6QCq~L`81qq9o;?i$K#Lap6v38@KK**VfXyx^9pi)@JWz)s7%vsVI1W|sj zDN3f3Ql+{Y+tGL2Xs2*`%Rpbj0k^_<`3c4dt(TWK+LznF9dGdq&Pr63jEL48p-k)J zX-P)=c&gk&0Zlt>d3|xR6{#cu#bI{sowFZ_r|f~2Hw_i{8*HX2;qgZO^ip_KzG%B) z1qEphos2wBM65Vz_*Mt{if0JSb6Q0WPZcB;Ybk^_CjHy%2p-IeP={_OZ!6I(`AX}g zQ76m$Jo07~9^OFmv?y(kFr=?vLduPc2cdAZd&?iZR8_s*1j<1vQp8Im>d+sBKT+_^ zlNSnR6eTD7+(8Y{@sXPKIa|Es7&}MBu!S~iZ%oh?l7E5Mm(yR^(S9Rm{JVEPcaDrB zqT8R_)Oky3cgh{j=tWxMyV^9b*Qv2p0iTmLq-OX!`@Iv0Vm^;81h$T_`r4Cd8r zo@ZFL30;LSjiU}Gw#epej}oWX-ZMXsjb6oyikE1$W-QJQ*A7MBoN*|j93F@Cys14m zZ_P1L=nsL<(zexzfx&rH>Cyt&>aq5R zGV5-Up7I`#j~{MXuR>V63# zRd|~8=$Y_`WnK#1k_UCkh;!pBfmU0MB>Uu4mjj9guq%?rik~ej)3Z^6J+iy&L6ow zZAIC09*(Y{5fe&`T;TF=``S7c+zc|?W|*BiY0J1|BHT+;=(r_ws1VhTvdJLmHUj8r zf;_ebE8WdbakXZUSaQfC44Qq+i&W6f`f$_DOhW2CML*|nH4-0By-0tp6xhv9)hs{s zLb)a4x0~wD$DN(>`~5Nr;Q+(!t~?|)M6P=)I(;rj9?$D&sZVrF zbl2QxD6Pe$B}8rau*~E_mb2hFbD2;$3@r}dzfPAz_EAQTIKeLxe7p`RgM+mxZ`b4$ z^+zRi!|Gk~n9VnbXyNJa#-6utjOYEUY|s1<3LUdMu0nDL>Z{0UOr?$4S{Ci|^>_K0DGoOu3x-E2z*aOO zN#{tx49)mbQ@FC45zn#3gk4#SUMWAv=4?j9{-*!+qS<`a&Z0EP9J8Mltd3EKYu>ga z-R*W(UNp430Y+8*#m2qB>zPql4hJS&$<-=~iFXrcidqlrPlir?#lQ3G0;ev0>3Dd= zO5bXJ_AIouM4lr|t9J@wi$KKG7MsldJ`^=0IRed;a^)Uf(anLM%Ra;gjsO~+iEE67{sbul;WMOizpW9DBA?krlIAyp)8~p~vhCJ2j zgn?h^*GaSW0!Aw`UAG;HXOY8EU0}1=7EVeQ5cS-0_+aT&FpMn{@RfHCsw;XHww)lA zIO`HbgRKgT0M|r=N96nF&kL3Lp7shSpJx_#ZZjMn#jKLb-{6p*DvLL)Q*Z1Ftz=7y z4TOPwBYU2q*0_YF4rM!Tc%?r=4uB&=&AF0!)`8hUJFcZC_7lAT}03=F9a(RrCBhJ^=$+>hh9-mbDB`|+q-Q8e3)*J3@@{ zvQFgi)1NuM36YJ{#@+ZQ=aMJ*Gycli*MC8>>+^L#N>}G%?R?Q^5&k-?uMDKO?rXsf zC~ua3YtC;4x-Q_pjyxN{)b+?^Sfg(l@C6AOMXwI=APst-2b_JhCsa{etoT(NCLSo` zl{I2saf<2udm;@14{#aP&DYMIHQb!rKu`Vot5DtOUPmK1#Cp~t)xQnyVe<;Key>btLH6d9zxCv0wFB;B}`zuG&R$dH&~D|8byl-P?nAC3$o zw2voX=Fb+Pprgmob`wwUV>z6mw`-V_q&|LE=&OeOL$HfkB>kZc5_-e1A<$&AaJK4D zd3?S=dD}1L2Rk7a&cNPi52;SPIwLc}Kq~m4L00>}q?~@S3J%e(G~sVn(#|InZFCce zsz2Bqer{+)h;q&iTZF|Qs1lX#LLdTj-h)Y9@>`+d&xgh4-@z( z$sZX8EAccRj~)4hz-I&;xZXQBmNlCnp4r^iA1ym*)h!z=EQn;Z31<8T?KnX(B6epT zu7Q@4w_OERt~GHbYd3Wb6)cIvLH1F<#zda31cG&7BuI^7-WyZtQyGr-Q{I<=;} zmErLPd~Smoy)jq^)(vW+;2EE0job={W4x}+cl;r4j8N9mQaod=myZ6W0=5uGMPx-_ zX^Q)p*ehf68L(4*Nr!laYqjyxT4w-2FadR#`y74gGggs)w?Q3pBA@RuM*GMrGLW(? zx#fT;5k`+(z~O`lK#O3PTLr9JL>)#6tdV}2r=yMT=jk$w?1?J<2L%GJerpj&JnH5; z$HAgNahIiJdLc0H5!ki_=_Hscv+kGiVnpv@ZzjWerGybgQSkMacN?ef9uM3RI#$UQ zNcOuY<>!?l3MtV+gb9l%Nz_ei`mOk(nhqAb@3V*=G#{@m9Ivp`Mlot>2U*d=x2~c` zMTyVOQ;={qjFT3D$CKfP7dm85>?%6;-uB)f<|Iy{211<19$;M8bv(%9u@3>Kjr2+-RSc6k;bD}leQFC1w}B(?&=4e!^z)gRZiL;_pdnh zQJdOH@q>U_c{NsQ>Bb#1Ra>9FZ-U+JW@TrmursXzj0viQgK0aqDY7RW*gdOla0$yBUgugOrqnxvwJT_vGZ4_6g2znhtyyY6g@XKdewRPJ`VA9HKt1 ztue@Xd0eJ%5yAio9;J{fa%T>u@zR9K0MVK=_Cp*^J=mz9~1<9Xt!8 z?gm^1hx-o5jAr4eu%ZRzO9a~I501+C%CFwP@66+skZ@*@W9uhw-p`9G1vlJQrA&|8 z7_gPv)=!_BYj4|e=-N5<@~X)>)7I)ay_SA0Elqa68`TdqfP*6&kNR4BVYnwSl-R3 zX2`ep_8ss3G1Z1=swtLC^H*6}?0MJ+ax!F7a|s=fqd!Z?Z9l|yWSVbBJ36kaoGykr zv!M89Y!s7tf4iJPp=9}ewX`0m1n*gW!`^TrGH?yC=%S(fdG9H6C5N3NyYs{hw3wsh z0+=XX#rNP6=s`e}SXSJrX`R`Sv{&nS2A0AbW8@3s5-3w*3Tp^|YnM97reR zk>Ue~yPloG+h*5bTZB{hApR)wEiE3xHexLS7xyt%X{JcU#X<7)hCTcX^x$>PNv;DL zjw3iTOP1sQHtA$~q$5Fv9l@~J-b1uzIJ*~>{6*r;;VE`IM+HGknJ+m+exZv{TMb-g zHk-kUTzo=}NO0CDiV`0&R37CmtZ)2&GG>kd`>svK6=B0vq4cA_d#-mH_TH8 zQc`PrBjG9p1dpt`e}s-%By;JATLQmAJCX)S6+&Ts8#y6-CN7nhguP4^kaE-nPxCtV_y=)62v_ctGi^aDk(jw z=QnJT>$bP(6okJSA5{1HCe1tdQaDrx`^zX$_K#pZ0bL4MtK#Y9E_V#5pvM+Rz*~5OXbeEzIl_cmi-ABm_Jvv(A$6? zZ@QYAn&C%h-JHHPZih$h{U0;r%Zm!K3XYw`kk)f~p`oFv-VbG@|5{GDZOTuXp8?nC zM+$Z{x*vP`p}rZ&EnZ{Lx#}jrwI9MVLF89MCyC z2dH+IvrPm!NN}50QPp9$-^mVX-w|6#X4!3l3CB#<`Cy_xNSJC7t2xq(YHw&guOE^2tA^auk+7U;BEp0sc;ls{hL|}VXxepaR)4opt*x6<^sC4J zc6d2ql4QGBF{f4Z&EXYisn`7AL8;Bf&S@o3*BFnatf<1S;@g3Akxw}xXXO1vJA!M9 z?ayWpFKtZ1^Q=jxxj#P-!TaXR+OJhaeAjCtnMUuWc7*TeoCe);|$^Ge`k;brvl|1J=g!PFBEg6BH zlGt>s>~(uJulwlB)r_Z2KHWG{Ik7yrvKNO8AmWpeYmB(j*Wn{QG))=ZuV3SAE%r>6 zzHd}@KZZC!aw;mm&DG$TXzzTJiAZ@&P9WKx&er`s^q2PfzFoF<{~6_6W5is{xPS@o z_Gn74nAf+`-2`$uB5r|)Ssnf%LlT@bUl7l|+jg{BZ#%Svo!dkmOh1N+4j@hG)Wekf zXbe-5WsJFIdk00N%BVQCAK+nsf{JZwg&vHgRXi_#Gi~l_Q_lA6i^2iUEOB-75e*58 zQ9H!*CBh)~sNs!WmQjsX8lDWM{_oWJ5XGwFB`lM-Qn@;~TchEhK1mSYX!y!m0euxc zBh_Ykt6cnPqzrsGRqJ?>F*ndS4AP2NPWZBw^~h;Am^N|GZI7NX|5HxM<(B$}L$MSu zsWaaryyNvSl8x`?CF9n<^{(wG#_wwO%@Cd1B}^k4!ojXouNsw~{t)$Luc@y2+CwGo z^lZ~1qqQ;4kWZ2P!MRaQtJ_0=mFM>Nmky#M#rvz+&CsJ-ZX6zBKdeM#(P`~geLEzs zvCU}Bpz!1VGreeN3}@Wx6VoU(jYZ?FST(f^I{RPx7(EJsvWPqWc&qiqJrZ=zdM_su z&Jt#0zDMjkZxU`B)3?VQwcAf)$gsc{-8NF;J91A<59@@otFB*1V#b&rQa@mq!heF? z#qB(}3e{fJKx{hZGgH%?n_eWg9In3KKp2;#1=O*S@|6e^&mXbe5QoK}N|Y|B?HFdbh)jcOz%xbNS;Z>wGXORLC@k1E`Yf@0@;Fa;jt6Qby9 zV|aHU>)glRQ>?(UHwzUlr$fj_ekUXCxJ-)}=lS*#9c(}|TV~PBaMw^tx`{d%f|wt3;D)>>P zg6^Au6Wm^Ju*V}TiY&4$Ye9a8ran#I>xVU*8Q)Q0Jla;A$}(#bGJ6b7PwQNpjE^Rp zNn^p3X*fMzyXYezkq)X<7-}fpjju4WP?TlLlk`w6{{m4B5ABUY2amX_`+`ydu}y#OULZ?!-2c z!UFYM#MIlkVdgEe3*$;3O~%f?YXuDiM zY~!Q{axHNbF*=X;9NN=U(C=$Z_#vsoCTQ-rIxT2cGUaHhda=O(biT^-VR)whNE%jo6Sac8 zHlqP4eMZ(2Y1rVvN8Emv=|w5|C>h^IeLvc^8CW!TZ)%}Uxr}7jqbB@RxWrfp@#zbZ z!_j=vgSXkH`{{DMMtSn~Ge`4<^iT*luAO`|LU$Opemr^2+SpU*s;cWC< zEWgRvj}o$6!0-Uw86IQAe0(Nj{r%VuG=?VCvPG`lKb&tUX{dvG{bI*r@B3WLn{Q5y zb3KRFIMSWPzwPA3*v-QA;kuSn)z|Ec6GpIukOhOJ5sk^$Wp={>(#Yjw*SHHG-R=t^ zc-a>hQx2|65U1zayT>rE@Z;vE4IgVwx-AEY!W*(|Mx+z8DI$1e`mb;B_1ZSkh4Cdq zia$F}Uq;_cp`A<~jLg_LytLlDXv&5=M(KK#81`r9Fi{IMzKklDmHVV%G&s|HR4&F$ zE-tGFe=5L#8MD5js5irWZ%=l*#RJb-lji<+QbZJOXRcpw&G`h;K3B;uO186RbXBQM zSC57D+ZEi7t$F5qVZDk6a2LNzd;Wg#bXe59Fbarg%Go&;nSXXPUk!r`5A8dU$sm8= z-!$)==9R8>oD;_sE+egmT)G)w`5~BobRVyp{W0kPonNRcWcbQgj?b3M+J|(Gg zO~ErV3atiPx|B$ileNAqEG@U@{+cg@iDwY;N_WckIMFoy1mDgQLFYs;4@HS^l7IzX z^~S)5i8-@K<=B@PQFX}|YP(6d?f3E#4x(pxnmkUHsV0~N=wu55pTdKoi6V|@YT|S9 z8kzSYBC?F{_Qu00XRy5cSo0NAyBl@oT_}V;Ax2QVFU<}p&Dmzc<{!)YTwsX#IP=Rz z2L1N_h5zgqt(h}M{)GP4s#hEN`D*q{!fQWwZ0UZJhppnz0;v6pFAoPNJp||yLc_{d z-%XaxKkgBT(@x1euLXabs;g;$UnpFJ)G`<&Wd|0%^?UjXM_*nxvWh>E@KOD=x_o;J zkJ|6;k_Aqh3JSPwYU2`H1UO$#xcaXj!vw#p_;$RxJ)@%+zkO`Dhih^;`233lPDT2q z^9HBQkoBm4jq6axe5&Ap65P}~`a5snd%(06dJ>gkAu1kFh@5=@@2IZ?=Sh`sO?)Up zj`AUtmXeF9R*?^T7O4+t{PO|rr-ha;4k^-O)QQf~HVqCUd`(Hvrfo{;Se(aZasZT^ zT>Xve4CUq)_qA(%1`YF-56TH;x8Y+}+glIv7zRa?Gk)PseX#)mrL+R)VpWV+su`>{4!prw^kLZ)yiU5gt?Z<8^y@}5@HXK;K1oFlh29!uT-rg>f>*xB3s2$702Rp6Vn5dW1PF&q2ZXC^q5Z;3Mg+{_wo(-tK6C zOMIhP_ln^OjC+6wv=wAzgbEk!o<^*!h8@hbbi&Qp4PJlc3y-Iw zJUxn{c^VR~-}q}P^>sX7;C^A_8`1B9y~F!sfxaB#AaH*@(#*CBgJ5>L@id0vaqs8uKVUwm_C9;9z1LdLdiFkRXRSF0bs=ytE_e|OHB|U} z^0Y+pxJrB=dXE*ULUj?fp~+jL!Lqq6(s->GT;WWxqd@d5%n9G#;<5pw;b>CIkp2N8 z6D)agM)Js*_p@IHxhHMp+(8FNPiy@uGdZ0}n0J09e%9hSx(paFG2K~$sTP?SKFuY1 zmjR`?Ow)|uiIXHe5DIqK7E|2sM9#H)zCPGEWA{t$D7N<#@4Y23^$n4kdOxazEDKE+ z?AN~Q@l7Xdjy;+`Q*K%Bc`bMB#kNkZHq>CQd$M)6Ypj8TEFwm4h0~Rv&~~y3>z~N3 zc7S-ew0_k2Urz;^NpoCYVIEltzsW0bJUhsN*dZWX>FdoAuLl>|d09~^AL9cbhMIc6_|a)M z{HV`c9uCtX<#{&U&!QcTJE&) z+7FUmPPeokS>Z=Ie$Tj=?rTD{ZDHG*Mk5)B{k^M^tQR6K7-3N{hkcWM;Yo|%xQR@( z0`JR3Q=aYnt;=$de_cEhE1DIdm}2VG;}4etMX!gk0iB@FkH-Y@UL##zFZp6O)a}>H zc4a>?(QM2Ij27V}%mPKztI`+*HyjBSoM4EYoh6lRjSWZ-``OL?LBjK5FUPHTv~A-O6~guxJ0SU)M+Czxp|D2) z&Mz1Jsi4vAr{DV=$eUooPPuSE;%2Q&y9Y|b1MLwHruS&iU98FtSUCx_oC_3W{g58{cG}F~*;NnF> zb)aUnw;#xbihG}~)qbY`>WkELr3v?FX4dQQBfmmEPq%y3>28%&uw3UNT}+eG1qm|) zZ)|kx;R;LsZT}L0qAMHkI^sd*Er)GSDB-`31{-%o6kPfUCcZ9xw9`m{+k85jc^Sur z&feoPXJ4yh-q0M{(eUYTP$fJxbd0>YtRdv_{T|R-)wV~parzET$zIf~N4<5?>;o0f zt5XwXqI8a;omt<(6R!oIn)NIJ!P=E!WM9=_18z(o6j^o$|d9Ny>G0~XAB@No6h4xH_kiy zmuqB#N~uxYJ?iOHL+msTE$j5j36yy!(SlzU(KBVqB4R5G9xJ7t3mlzBSRZgLK>X#p zwiXhvlIq&-EKkz0<+fpA#Ze4d<9jm@@~M7>=YE`|ye8sWi8I zsq`x9QickAfgE=uBSPaC}P}&|DcQB&XhX2KjQiZnu2IBVw|fo?ybMu$a0n zTdJl^#vW~gGc5Qa@>mz7R8N0_coR$W{+2zrojY2Y?A)36 zMg0A56Kj@}!}g>09zJaN%qOLoZZub#>d%N}In_|}=xL{s*)Mza$UJ)M74sZb4N7lz zXF57Y&O{0E!LP~tlDQNEs~!qOeIzm&29GhjfFJ-~VPjk*swXV`0dq(0%8@}+gl|&y zTQ#qjUaa(TH8mq*4mv1+J|GU;k^KaFJYj>8ZH=rgw|+r#%G>+)eotp*gz2_BfvwTP z_G^_PUgAAXkW?`YZ*Ti2er=lZX6#tZRZq_`G(%co59)K-roJQi76o0=Xfst@DMt9Y z?rcnF`NJzga)n~G49mpUNRjk-mTw_DoZE3PV}ybGvgG#$Z`o+q;90i`D}BRCUpT_P z3oKPuY8CFpw^==^$KXvztw$SniL#fn0Uw3N#Lzw!&dkjfK89jDRAwXgn0T~W`HLf=HhVY_B-Apc?`qG^2Ia&In6U8-K9%Px=K%A6oa_FQ0uenVT6?a>*~huLKyd`(1^y$m%F% zDX4uJ?9`tMl)IoEt+QuI&Cux0=TrpD4`r?*c0LH7oq*{JYwMx)3;120V9|0i`iqzs zu5Vs`0dd}sTIs&E&O(Y6>)Z)}gnV3M;`4G~scnh}IVHHtHburyFrHr3&jXMjfRq!Q zHbxtVz^YuKM#j_5Xz;UT14>=E&LEzz*T=B}2E&xr{VVeq>5PlV2ZfFZ{kF5loJ#>v zR>-+)7X|6ibXzS#!wa_YO8!ccd*{H!&h&hFyYPca`(qZ<)v&>1TXb!|v!G{85#ZKN zAgB$oGcZ7wil?--da8p7Mc5)UM38N5P9h@WPIHzqx`#(?*|Sd?+~hvHmytg|sZTCP z72qSsmw-gGX(Sc!CY>;O`^fLTjci=Dx;V9$V|_9^aZXr)#;Dzz7VmGj>ad-u<9dz~ z=XLtUJ(35qHuOt-y;2}b;N-dR^N3Gb$VJ|lql3>~AFXu*!V`%bq(DavYb?Y}I)+Lu zTwQLY{3J&9t2QAA*`o;_we|7~Go-9(U}Ix%LaNY(yjJ`LMx2$kY=2~Zx44g= z%LoscjZZ-OjIr;E-VPLu9l8G!73_gM6b@aJ(7&P(m!D;#+#pt(IXSnY*}SJrB5#b z!d8V#{*WrW^|zc6|X3X+^jShT% z4x{JHgkC?%%l|FT)j$UunH*Sa#++J?7a#uAoC!rvxt#pP>OviLx1xPHLw%DW&Fzt>yF$N_mShgExXpBI=d~6{2T=S8D<)+-F+9^NKudXk~6) zq-5n%LP&M^KvjiB_t-Utt@1T}m433E zRoseKc&!`vxRJi>Xr)5B}a3iaSBYe!<9@@G7l?S4P_c26+d+J}|&LYzQg zJigI`D*to{ z5Ls!`f>5JxTT+|}eRSZei*AaE4SSk<)$}=7^sOLoOZXOE6-AK|UHI~q_-d9)Hp2MW z!uvX{QKV21lf_Vw17-L+DV%VQctb8wU-|}@vab5gr$g|~TRZQ*#jIu7K645edy8WI zBAadyT-d(OkM#_|?{EB&h}BMrz+?rPXYk7J1wWT3OEHhRlbSI;N}698NEb$~xL zVry3!qzVhBHuzXYZXu)YvGx8174-^V0k7GgVAljdih-Q5f&`VE0{@Zp~X)$apEFx*JMH~mkM zZvGqAUx)VBWviK=EK;HI8-qKhh4D?}fFB*zkOsoEKZz?K+Y@HJn;|>&)SVbO_Ei`7 z;RJ0~jv8=P0x}N&Tm_^qXOCjGWnkovXCxToomH3aImqs}$9pz5WqJJiZBft4C_z(t zP#}dt{?Z1+*!$dwnW8Hy$sNe;jamMaC%bOcO5`0d&CddD#x{^B*MSZ4mw26>jEl0d zH`6$0_|(MP4@3}#7k(?^c2HXz*U8|s+=x(nOB)t_4HY?~FYQ1c(KYqI^aS5CqJ@^; zY5@2Ru6Xm>$_weLP6{lpwRKDWR!XP&_dQbN&FIU6_A$V z*$7x86>FMcB#h8-F;T;|xJoPcqKaA1tx!6=U29k;j_c!A|6jcQEOZ3t8 zv#=&ItdPtSK4^2>2(5VTD*e~`z8CtFV}goFA0?6seJ9YKmocvI<)_vI$ynoiBX#T< zRl7x4LowjG;EE3Tbc0((h0cB34df)8&`-wR>vZ?^&>3h4aiUlvagzCvg5!hQ8le}` zlu-TghLor|hUE)?r`Z#Rh7%M!QcqO%$4!w}&Dfi>{O`sF8hJGrd#+xK>mQ)FKc;BU z^ToD=oZ-$hr7BRFT9z&wYg?TbQM^E4(NnWiQYXJtyM&huwKOGH2M?RBtnQx5 z<+kn1UBZ%A434P}V(@aO=Kqc#LTgG9?aC&5wKw;`Utqs zyWpXsA3w{s#;tFx`{>wyuiE6MZ6o+oN@!oQX_>|OrA!f{t4EydA()oZ-rurYf zD-T7Q5*-P!331yKxfV#|#+d zWtRK+wSIz8Jvaqy-f0;}%oLJ7FCdB;y#E1C&z@4oCFheQ!%wZ7(&FUA+s7ZCZjcvk zr-)nwz$G+~q_x*QGRy{2IuN+#NuTU9Y`1%kF@LnKV4@>4pAn7oM2?l|8y$|a9GtXR zls28D#+7Bu%{oXvwvbsHqNr#R|4y$|NB8epCzifE-kUO`B+^A;g~+;VL`irFHs(@U z>mw(7SA($>u6FmA{avIPG|bNK*0$tY>m~8tRr#bQi$C&Uy{4_&UI+lkMD6&C@$G;3 z`u0pk@MGD`^^bIIPW3!>K44cWdDZFWhE}#(>;QMoI^43Z(x`rh-u=|s`L5<2oL>(W zzBJYX)WiT>YeFW0?;^7NtvyEce%ou42(WZ&%iUIl#Eoqsunk2v4 zq+?GYWB*O@a>yyx(^qy2w`|O= zS_=+6!y1#OUQpKvV$5JqbpCYlVcQdi)6317f4BfUZa1Tb8WKdb%&e(BDN5UNVZE3V zAn`ChV7}pQcmqGkz0JP>_UKPpNk21M!zn?mF&R708C4p{>I11?ML^s`M_W=Nab{bc z36sYi3sW#oz7@#R?~WRd${)}oyki6fOum^;R4J#N?f~eEK**lESzPSubTbCSjIh_n zqleM_o$f5Jjkd0@7*LpnrMt3&p#tv`pU<;~Uy(BooECc#{Lm-Ndl z*;HD{1D8r5ehT-8xXaMs_pV$-i!IA(e#R!&`mFxwU?*6@sDi?2TeUX4)*hP>g_|M7 z`Ie}3Q=x6xq67T&;Z7pqk}d>%gBOrGZo7bX0JP7{G1~q$K0-*A<(tj4GcJ`*8HL8h zagG#*zk=y^*FK$|9d{9vQyxgStgdDpX=;uH7%f~mpMOp{nGv6-Sm5DzXZDjmcu?%QPDPb~i!&d3{H30Bh>vy~Ce6hb z60)?q8g!{qTcPrlg@U3T{gfK{J)tB^wq*M`mD2i$W104`DbckYQ`|Lx?16Xq$nIf~ zfq=?4bEZ{B42!X3n(SCozH+bzHeiODntktC&c}U0hZ*$=0O|9Tq$C1>i3u``IYYne zXxO^V>*gV%J;}_S$nZ3}?F+>k=zUKo4-D1Wc?k}Li2hXX;cAta($)>9_85(cm2Ky4 zqzND885=cf{dhFoV~6Q7uWo{5^5YGGcNhVc8dr0)HX=H2eqHgD@CCT9lj}IihycOU zFNCtU%n4?fciGXwDRk|PxO!YX`f#HZg_r&;Hau&A)%};S4X8jV^+B!3Clb*~yZgtx z67LYJ;F6rN$yC z1+98`a8Pij-#r&(#q*&!!Nt9^5Gaslj_-F&EGtAX?&&uw;}W!^WNuNQB~?Id>@jb|RLS z zpT(y+R)^OeG1_|r7X+45I7}eK${ofW7*}oebq5#OpElnFDZ!JD|(kXc?hHr7)wRo>3gOUd$xcNfAU5{MO*Y4 zgfLA=N#i5tyE^rZ=zHRwVtVt@$nBt@Xe2xVk9Zou@h{jMrT4sUTz}E*IfbjR__N@< zBmjZSHQL^S^_*5g$)aq;$m#l#ncI2=QDV;`sj-Se0#YS!f6Q?0*06H;%dT5V?9+s;%(Gc|mconcG9T1KYL(?i?d3g|SY0rvhg_*#Tr=lulUlga&r3*Gho! z(sX(qms!rm*tf(r*rBI#8a(HG@YzvGT}#M|cA%8( z-j7Tkkrk6IJ!5r1O~%?Uwy!`31_NmUp~hN`*J?i7K?|}ODkqjx#sAlU zN8AQFZKY&uk}us=Af^)L5Dj?!p?`su>j7W=ZbLd%pCeQf3c;WO|v zaCzC8)yTWwb=zrVMDDmK{M`f-9V1=m1vm>7H}>@DFD8p=;rAju2opfm>8irQE!9vb zHUXZ}PA|q$jnc|}eCYlmw}M=c)Ik*TR z`T(PNk4YKOyD^#vlG`C$4<&UOGiFA=%Sqif(lW9#V!H;IU%=TI_=>#wS{yMZ2F3}D z=mFLqp!U+XZ-F}jb2TsA0$St1hUD?sHXi4C7Dsbhk=iguNHe(~df=m<+L?j@ zRN*4|pctl-5Gswf1x`}{kckrz3zfOt>!t;hbzX&P4FqYRF#uzCE$w0a2%Gf*;`8Ky zbe$%|<+T`;Wc055a937{)AFJ#R(aKkjxa2KkDjYRShnCgTRk&keO^74Dk-qAV!NK_%~h2nQsQfUN<34ElqWOz-AF0?MSO6h z!$Vb4wtbJsDwct9%H~;5%bSoSX7ENCplYs^Jg^~c?^%|5(Oq^mq!SdRbuJ}trDyo+ z(zC4SsLdkHEQ;`FUdj!+RBbyEiXd*dXN-&`P31WNgSGoMfu0y`3ZsBOUMEb;dt7Or z^_h|dbKy+}3g9NtFZ|5MU0F38D&5r-a$+_in~{1W{T+DJN(n$;Zo?jlG){p>Bxx|`-89E~W3mDO%00-LN_c5d*?Elbw>(YQE{>q#&-n$N(`ORuLm$w-BizacPN$fMi`QkdJ*X zZ{M?=7`b*>8QQ6|l#}&enq4ph4|TcWgEPP7r3bucz+g8A&&q^+kc-UYFCIkyg5f8C z@*__-b+|Sq01c~17`1_yxd}yM!>6psE-2_@oL6L`;jIs7mNL3ne2uAhE=IU>ut?k~ zp6#hzYGUVQ_sPhCGktF()!qUYHB{wCt19m44$ew$9*LZi)U>VV&O&`yc_H^5PAg!x z2^}kxd@Xw44*&H`^n#U54P zB>J^bQDxqXva^bw3fScuIg+M3aNr3mm4s5d;%shovA&36z(!jakEFQG4&W$*z$ut zI8%pCfP%bT4}6>ci|(b(hoIU4vMNcu=x}LOfMQ%CPN$_@B{4_FpBXYe8KS@x(9J@( z;t@SJB>|*T#e>d9>rHXkfyHoj+l3X3CokVaaZha+ZZT;aSdogqvSSV^IBo>4kg>89oOV39QJK0#+(G;*L?JS9te>4E%3=A=KR8&NarM9id!j2dvF34%gF=DJfiH+ zlUO|0V*_4q*e`VZz-JysPl^}&6#(&dsBncKRT6lSUfo=2eR$Eez_@X39Nc9CKvj3x zJ8l3$9CBgEig)(lj@@hRUb|Cm z^~%q#lmk~)LoJI;AU&EukJBDc%HR#_V$fWowsD_!xO9gRdy3_3HQzuEWR$m>5AJ5O zxHxYMvf~2XG&@+b)^IOtJ8Di`ym#EWJEJ;WjuMdF1oL^akoF{T4>-F^^!~s|T}Jb1J)qPBz6iXEHkik@S6{v3I^hniDcFusY`uRvmNvx0ov_~t zPm6p#%kla3@Xq*1)72%j8?jl9-jg4;A@q0E z9eg%(=F|e}zcj>_2(Y64Rqo_}mc+$hQT}NCrTiuBdx>1bLtsq>WFa!C4)c0s?haA1 zEPt#>8#BJT^@`IPPYQ^>P;BY}jKcQaG9orq+SJ4IeM}F5ek%&VC%;Zl5_#7e)Q$Jw z_f>R}Z^GR-f;3R2>KR|45le7M@G2E;y z(X5Nd-ifz)<-Fj^`$;%%Tt|EkK)+Yy>Tr6(2(TZiB6}_bT*~@qMUOzLw z7h&T(VII~HVMEjrD~8@Jti};fv2Dt#mFPA=!UkZ-pfxH8W;z|$Lr(((ZJ<65{LGfx zLqI+_+6t7)GeQ{FOJ}_^Yc^#1smT%vsPcr=ZhniSdJS0Pu&~^#Nwy^2h{!7A znIDW01T5n9f`lgJAd~e>sv{5b)DQFJo^ZIKl}#IXfr2Ie86yFfrNsm2`O5m5k%(O$ zCC`vX;@!@^zBFJ)JHUL{zGBVXP_8GQ%l-fZgU__@8EnN-fmGynPz_8>9SC#)eA&!~ z*~tPKt<>>RSZt3AP?J%0j;Pl_pMC-c8t_JWb5-TSvs=IZ7$ql%YI6cAdJ}8>d0Or~G3#fFo*rKS^Z#>Y^SVM^tAMD| zzrDF4ByOZ^;ICAst&wk#pj57ZyHF!9=?D|I082I-`Nt*I}arDUa=11yAGGU z{nz7SgI~23a^A}u_#Y=O6x#~NyiNaaCszNL6R&|gtnvPDCo=p0XB~3D9om`yvsD+x zGl9ba2%n~=CZONUb&cvUG#lM6E%iM;Jy%v&GbUQxpk58G5j?v$%enr_WAHW$3(JGU zqo{##5X18l&uqYcC0|}z4u5@we!AD?xAm!l(P^;xa>D-wBje&=;}x*<>#vsx(4MsI zo6jkLD(jaA;ZfAF&Vwu4az}-UJCG_2ZM)J69j33oppBv?K|sma;OE!Q(Z5Rpa7wTTE<3rxQv=*Z@l(g!w^7+r ze4?oTB_F_hr=p2xx%K{idbhA0qiQkneaUa&7{^|-j*;h6fV=VU^1{dxyZ=SnqwND@ zkW$UpSm3DuF2PxO=NCmGwd-G({iY!+E?4iPEKKe%0T%$*7FQS?uiR$Bo|mv?2;@)jP> zPKud=3{?_En?Oqz-(Rk*t~{9+4F4hd>U!1FzX~Vi`>23>_T&a+pSz>$|GXmPHV7YmQ-o|STxJunZu&aT8_hnI?Ue>vN z)Be(~HwAa1xYdWcBpzWJoEAU7GdP+7ds@!`UmGwr_Vf*Kxs^GZa}l~89ASQrr4D`f zL-Oq6{MYKL=?;~Wai+JKU{DZRavi+S=T0a>;T_tN>6O(r!p3y2_2-0KhiobPr(X|LeI-zmFHfy;-~7mM z7c}_q`YVKL-$djI_65aBM*;LF$^|s_gExH zG3)o3`~CK@#HX$9^FK&f?3!&qRXiV|v`EeWEv9rlAo(}zkEGje_5Wm7^heg_8Gi#d zK8D@Kva_!)u+~4h-tL-D5uj=XlC@M%DDLg=*~|)jas5F0FASYWzjo_jFgFith7Ai2bMx!S&hLRHtEy`yd*W zt1B21b@6;%pFI)f>}1jvM00a<5e?FN{ZZ4w(gZ{jSS9%=05BMT0km|1y(SKfcMun< zn>lZ#1PGrWWh9Q^71_=*tMaO=Go=@UZvy{VXL*XdKtJS0|11ZR+@fuT(zeU0Tlz!v zES#0s6oS~9q2;H=-ry1LuZ4QVig4Ap+30ZCXSvmkV$7%4f?1`romDOyG*nyjR51lG zCy8q`W7Flg;j`k42`SknlY(L8q0TPF(c@uw8ZeA4hGR?sXSy3mhxfOc)AXP+Z+oxNDZq7x>Z(|rHujb7)`vynrQycA^`1K?3wHp^L$Okcyh zN9s)-@s|;e9_%#DmTN|8EuIk-_0pCnaqy^Y?<|vx=bgAbp8G!%h_sW|KvJ{^I~-m5 zEY0U=Bf-sLuBSYZsHpDLmftvY>Qy*=Ho%1d6BHK=&Pe!rlp zuqx(b$|hBk8IUBF9V2f7KtabG99IKRjpS8VFeE^Nqyqtk&&|i3K-lM0`t_)F`c|ph zi-2(y0j4#rGM}F8+%xir^f#kPa8;N*<80r7zvBHS5B2ajZ~KPXW8xPsi}|fr1K-po za=ugYcTa8P=kWi~AtxM5>!V1uX8cqY|58wv(z(3t~JibKy=+LY{|#T_vvMaIjDm?Q|~$x$Lv!W!sX{CO3Fla#?i9;RRU# zT+-e315mTFAalw&BL%WLJqmclVIX*Od21AD->>xs0l(}+UJbk@VGUzZZ|>++!;K2m zS-;H5@(QWRih4$F;n?xK1!3tqd@3iL-1+)jIA+`E0#wnjJUs?Xq%CA1c$Ku58=oD) zx24(QA_PNYaUMGFs!;+5+m&S>=xp`DTKCg=*U^0pBvgGNb4WT<|Ca2NBw(g z8jzRr%|qo`O3mOLkrk(${K{tAxMbwsrYaFoj6O zw@m;^1{UkUmhp!3WqNe+ku$jq?vf5EUVZqq{IPix2c($UdDau}0c`C{KK6cf@37rO zDobR?3;=rSDGa8UZ6|%pO1;v(OT($4Um7mQAfnjI1 zUJVgNq?pDf?qW+|hl`A9e z#8ZE?6kZ&+x$72`cyJ)%v4S%}f%)u;9>%o>tJQZQwZL4e-<$g|+i&tZ3Y322NTGklxpoWGuOr&qL&*;#>=jZRFu3}p?w$QQ(;|Ev-d6^ELZ0`XkoB2grgoN60>}I=t}J274&0Oh z1ui@zIwMo(g(K2`XLu<;K-BOK1Tl*_ED|u)+%=%3-&)r0JG&?vTFfj*0vRj0M zAJ14Tm0MQASZXWnPpl&^gvX)p8D@}beyKn!6=&Y5B+eo67Y4nfx?aF5eyN)$UH~3Y zkuvwU6^JJWZzkrBRu8H@<03@ekJ7aD5d!;Hn+fy7L9E!D#5)!_paJjppKP102ZB7ID7R%ouPy$yC91ha?Az&kumv1~s>h^3_ zppNibcD1XdtlF255Ae;wh40nHDFc1TKKeT53*8&vcf4rj%t>USg+CZ__t9raP~E`~ z203XCiDC0GUP%vj^0Ke8kDU65?EQJOR8!vSTp$RVj@BNw=r}PP&ib zN-TdYL*jWJ{Thbmw%RLYUOsW+xe`uTM+R(O^DyMJReiRaV-zYn5DwE%#UyVCwwsJ% zIY@a0%*=!GhMpWa56ED}z`A6vpG|CLkE`~`#Iit!)o5zazUOpnRJ)c05^gitZl*ya zOY`&9_aQ@r5-3Qs3ju3he6CL+ddY2wsA5m&S$*jT96MzXaax^p`fC7}qtqJJw#?DD z^}9!q&@d5(7?yhF0@a=gx z$y^|Z=IM-ypA*Y1gVmpclM65bkOU3bJu{>Dn>3%X92TE35M~U%aU=g&6MQh|J$mhZ zxn^}XF~NKKUD6c@+1e^x#?7ClpVaZZ@rgD=^*n!qyL1|H@RgOAz<+&z5cPN`PCSWO z#$}WJ-CH}ic8x=4muIBKnNN^s)WnW1!7?QB&Z$)=Cew2-?kSxOf_#BMB^7vHX- zrjPyCZ;0byKE) z!62cRUUKJRBZ&1f;_{68;d*v!0~0|~mM$mB;)#^7;icnb$p?8GZUZ>&;QD-;-Q0rF zbj#(Az?~;uol^GMF$=^X{p~`L@-bh=EU8Sg_OyDYC<3AtH}=Y$OCT)ho?cR&m(bp= z*%AC9dZub;i$WU{f8VE(Ibp}_tqw7n_{@ZV72iM%z8L=#UmuYd3}g(G=a=WqF{iRQ zeIL=!TGd;q2Fe(CE>HYo)B?};CBR)e#+jr;FmFoR>}_~Y)4`XDx_I~j1@Erg?%1yx znX0ox({-hKyh00`WY3v#XR(yCU2xM%!=u|Tm+5k1>D-&(l!PT_Lzi=xgG(y6Ym?4V^CyJg1^LArOJL!*yx zeMX&&PY1sHrap%!!*<6LGF9is*XO4<_*K{QgWk`0g~1E7y2}jntSgddT5T=kavqmZ zuG|d9Ojm~n*p&N}s{W50rj(zxNk>O_s=q&r-Gwrpk)dpedOUGJu(m=S8SlHhhppFq z$bdgBJfe=Y2wFapwTDeV#DgwytiE=*j&m_p){-_utfy=+o?WGCTfAR}`DaDHnsT!< z;r{{AmidC3p#?Qxa5}TGrx~_2T72i6xQteJ7-LSub`ebnt}L@}>e}KYNuC|DNVGjt z0_0b^>Ss%H^r<*++g;xI6ooQRE+_J!fXCnUSMF8LhAWk5}FV%0(3t229# zn9V`W1PY(V;Qqx-gf-M@qSQCKMXK(j&o-X1^slTX)UQW)*@@aKI-lW zY97)>&a&jpz-9^cG*btH2EkcK3-xHGOBN(7N4vl&(!;sJnJ*%{+SNOnVZo(UYE#>% zmKmqpWPIJezUC%$9?L~TEUeVqa8o-C9lK9`cjTf_Y>nTwSqU(Mw$%5e)*>2K#l5(rbsA^wahuycfe8QbpH?!jXli|Kfxx;Ee~ zm*BpGLpa5W9LQg#q*`bAivG9C-(e;e7;W>WpIZaL%#pzTg?c;^Hl6d!zDNU+A%r<| zS~Ct=OzSD%1^HYM+$Y!xw&!JA8g|R*`(VAzJSd8SW7wABw@$@|*Q5KFI(tsmyzrzr zjMC>znwmPYzuFs5AdNt#EyIvW{>!4U1{%9M~zXZU7d<1te| z7SiZs(WWh^Tg}OAFtqlQ++dS1^PCE=H5Z=3*5kXAd>;xv8lPFUNf!<^YiwCcVT-PS zSi*TSi-kHW^2GQ6%1+V8YqY4h$vkY#bFYOX;`Rd|ThJJlC(Cz9;DLUjgD1PzlwiX= z{P>9k>^&Q){zgg0LcDOvO2O+cA+6`sqo*rz*tQ9P5+%7@R3y2035AEupC@~#G8>LP znPa#KO1|!9xSQQ_fY^SC%sZ@+T^YgX-*Jw1SEQ)&VkeKzf&X}1p0^SiWP~@)VEu$6 zQ9ftXKIS7L)ETPt4&A)sbROh0x9Uanedgg%*@zaklcm&Dlfxspg1008PjiDO0tBUo z6hEnoXorayk&{_!r)R1HH(uCmnQLD&Y1^}T_NZQU);~|hcIV;`DEV#2Lnk*A z01qEU)3$AFwn{UU@t^)u%JWsV{4zRs+|OvYLn!HyMtSA2$}QRa4IXjh`;M#f5q`@V zRSZZ_dD|>c9#w~vaxw5Q_>SEoJZrSaxt4o(Dqi>(u!-&ZEpVLcNs3fi{57%?GP7@8 z*Hob@0al-{tqIm<)pB{S?OM3e<#@QljZDdB!O_Id>$CpVN zV9x(4=8BkMvVIs4(s7ritP-y3;-gBOlyG9eWeUag;n+i0ii9et3X@`HdDoZ49<}M} ze#LfBGbM>_kmQieYdbKx-i;3^W06%t#y51GT$*|_iwVhu6t1t+IU{eb*9&}-RhPfH z>gLaY;W0V-QQxEWm3-D!1hyO!x*sQ^Hi`v{tE-$jg5F7c2gK zTrq*?S#?vYwC`U=pZH+v&!71pg1NZ>_76~`AHA@iC^UXw?O2t;0{c7)G)cHPjv%wWWb?)ZO`__6EHn>GY{~bSH z8e3k{)0u6=v^6$9;5sMn1C~bhzJR}Qvs)M(>hoVp%2L1H$c~EYwvNl&+xE>T#3;HA z&x!i}z`6(o0;RjJkK-mcL@XHJ6M{~DC4$Ye_4uWo9dp4>K_rkQx5Cqp2X;2PLMb_S z#pBTPyx%CaFm;V_=eBw8HP>lWnUS$HAAh~Z>VecB|Fa5RW#du4Ix{eq@wFC9XPo-` z`v+j)G*mc)AGchZk*6WgZkP;eTVv;frG!p+wvZa*%Z8e!V}y$FHAWrFU#uEp2g;|f z-^jV+cj=l@Q7?OH!=#t0o?8mYh@62Tc?aLy%iE~&jj?hOw%l}Wi*{8rg4J~(x&xF? zWi22kyG_4k3#vD{2>TMa~X(BlQ0-t#9d$!*Y3+&*DCDT`rS%n<5|sTP&}mEn!*N1#zkNW@;CB5!(1j|e!53lRU-C;!)(q81v2 zbqmJ?0DlTN!fG74`G8N+zAW?avlpTtf%36?XVk^i^*m;Bbk&WOV@E_Gho`DUyKP_D zSY6S+g4~J(a3d$Mci)>^8k(yHC>R%RlA9&s8_|C+$%<^d?iU7=Z&u}750vCE-r{`* zrpK{Ietv#UY;fJvJm_jl5AvE2yY-0{d>M~F%vA5a@(swg0vZc6y6-!BT3%3w>V5vG zSGjEcoOe?|S4tv;9U~*{YU%!K=bbAfW| z-9-+{R&m~MtRAWqW9RSAH`I#1`8r*&9vm=iwIaNIH2(@30b#f&67{~IU7@I0=hav5 zP^}b{aUtGu8)ZP5tAGr1=osxyre;tWMwICrJg<8U3vX96UMwnC-lA8R-XhOYqj~9y zslWxPUZW%YMx}ml>%q%D3=!S70>{LH+lrQGOV=%J`G ztlVWS0D6-IRf)Tv7!_|w(kf+YWDTl>0S)M%ABafM5AiN@@}CBT##|~_$`|OEsmREY6|13u8BYGwC?*StM7j=wQ&I8ZK83B?)?BNYvRUGK@(9m^R z3~}K5Sy8GO^DizML&y9|KUc#)Ln3B$IApjYsx<;JCHKg1`+NtfoplWh1Kz|o>_z(x zQ=7%L)GjIpU9X*8y*3FaVL<<2VGRMiULHVpsCK`vWwBhe1+Kk19^ZZ-mEm*yHj^R| z3CRQ4o8w=6tsHPLCwt}bdzoBe>1p5e^H#Fki|xrY&q2Yr%hlwz0k!|sYr@WUoVwjD z?3bREcxiFXj(g#Iar(t^nWi7U#Bq3Z6lDrj%l08BYrrFN9An`reB&07d@c$+g--og z(k}mk1QG&4f;+)o zg9dkky99R)u7TiAa1HL>xNGC??(Wuj&mno9@0)w)t*M%-`^VJXRnUF9Pj5d<_xi0M zN7`J35k}@R!GC<20C`G>hYu%P_UHqc(`AGiho)D~pn6;$i{M+Gz&Q>y_paup&T^#2e>>eHU(-<8@xm;^E|z#HWThI99z8E{ z)xkn)DhoLBv4kbqAK}rfd0p5Bn>jxwGvc^5+}QM;uJ%v8<-h!_2JAugxHjJ;cdzHKIi2sQcU@O!_MhW{FzMQvJ z)Nj6MP5~V|#D~OQw|qXOvmtSOXRtHQTLC%gMR@@D#!G;SJLz3D97xRfGyQ@ZP%CR* zd==*9iT<)()2{1KJzIpcbV^mJo!p+A^!9-}@;lu))CGBKd3k+lJ2tvVlQ(u&U@o#* zA63t&0n4A^p}}<*{pCTb%&@u^bsccrm{TU~?h{Rhx-Fzfz4u900NTWAj5skqN;~85 ze5?++fhH+Tq%xmWXJ8s0X|(M&<6_U$tg`{Uc8F2=WY&FT7lZy_{uH|qy-|J*y0x5Z{C!?YGsGYCylgyYJZmES2N2q)Z`8xX%A~tH~E~%Wvdg z_n;zwX!B-aqz<#o$8n=2i5jw@<8zA{dc7+f53Bi(#!tcw237N{cmX(^!7~1!_IgXQ z7x<4L!NVM7p7r#X2>Hv18(S15Q%;sx?i{$!fB5V>n{M5s%qIs@9pg&qR+?ASI>jJS z-f#(K0Yn%eX%yP3ZF!v274_M#vH^}PJm(;?lL7S05a`#@a)X_3dPxg51pxA`14`DJ z&~%g3qcuN7G!lmw*F=pEQV6BGZ!ELdThqfal952*vw7rwAu7hbbqZzrXC#3-p!uIr zhKCBc*6!+XFvC?0>e!Fk$l=qyJSXbbRW$ip(~xgdMs-9V@0Oso1uWSwGrO1sBwlB- z5TDKUHu8K9{CY)y+$}39#&q>Q(oX1!=!xEN!3!@|89P3snX~t0^p?ZW)7dlq5P4SD zK7@W)kAk~@d$;?H+!C5QRt?=c^@V>U98B@?lKvx2sH+MV{&?2Fdsu-8`H8J>zlr7W zYnMx>x$m=*VWKJherNw>H_eG~!kU$At8lzwb&=&%8Zr9akK5?=Dg^WH91Ag=pBZPd+Ia9O<$R#)iu%W#Ks#|b0B>e`wws$eG_2|O-w`itss0%;6l#sc8_JSd@bhWFYLaa`m@hL8@+EclhTN-7 zWallF$rY;TMK{dp{ZhkNhrMMM+GQ4=e+4fW9xpdNF^b5uNkRHOwKH^>d1N`Zhwd?h zVAao#Eb}2Li*bupxPYaGMm$7k!Vj@mBw(N$ehW;2jW#8ycJO>Pyy4j5dJlZZ=e~33yD+g-j%{{}=Ww|6WiPFz^82jvePi9V z3P8sG=#~`Ab#TJmvPw>ST7}elC;EsA3_lCPj%ALC?L|zWN()dao7aT7kqj5#K`9R_ zl|+9iLa{+xUaq)$J*al;1|=tE70;_K8Vw{TK4Yq&{=Bj>AJkgG+?_^|{W5`;<%WrM z=|fFz)6v7hKO^4qmUS%GaGTfh({_Q!sIl|QGLiV`u#PHR?=8_9t4CFiR=bQ13gP=; zO8t=(pyp6Jx<&rrW3xRqTgb^?!ovAZZq|znmx$SQKNKUCJq+_Kk8?W9)U{+GAcjHu z?-OilM!%Oo7E5cNKLB)4R;^W0t!GacQ9V?AUfX?pcevx4DLa~AyldRurTvPd-O9_h>|Zu^f`%J;m}C&=Z`)ObH7=egtMD?41AMl?HPdhk$-l`qt177 zIyYOSMQ_5pnwq*j1_;b}S;*mL_=oj3w2Q}w@PcF)YxmYnDv%1}uB(a{QhDV%%x7EM zP7g~?{O2hUo>%7Js|uT0*Txl%H#3?VrRf*P_IS8zHN+*q*6}jS{GFRbRvko%RcwD*i0F1vxRw%o zLQskEmduZvx_13pP8LU!tCY@CWfL5!k1u3#NInTMDNr=@k1a*6+3#bBQf@_iE1;n{ zcCS=18yq&2oHAP%^%behKxn%DAk7!(fZE8Z286!o`8;U>0A*r94iVlSd25Y}-EjI{pHxLWd znTU;TA6vHAIK%}<&l;hJEM2S_aYb)v+)f*I#2$0G!KlBMFbZAIA6|D}?H8$I2VGxR zng~kja@nMBjT^QHi4Fq36M%~GyvlqlsI27**~bC)PaN z`tJI#bY2*vNVI%u+Cg8h&)>cqP~ zb0LEllsde#&0}CuU%Yb-`%c{UX9l?Xd^ebm67`P>DbmhxIc&M_9>qkISxd}*$+|Zwbpbi*tC_|`A%xguJg0iQ!%`5>(BUvU>MJ3y8L{gTCBFgg+BMq0I2g*srsvX0bI3WAf-PFm$j8YZWW2q=6)ul)b&k?rypvsZo{kaw;lHB5?P-VaMU$m6!I)udW@Xnh#+Mqm6QK> z1HdE#3jQhI%Q$WzYi7$$VoW~VW>&A?i5sr`Uiwa<%3v%qEb$AI zojl&_-jt?)Iu~dA4Y#s zi7bh_{VXT=(_16m3vBJm@1xVy$>-!4akA>S-9CQe0D_7l`a1XW;}_EbM`cs3%r`M_{iN4li_^6mMM!ern8?(unflKXiX^-kjDInpGWXW-;yP-(LzBf$ zr}uojO74PGBY7eyz8d)_rdgHP7`<>fe!o2A9jlVyP`tM+#c~8}9Y;O|Of?9t(Hg?% za)sAZ7MNCX`q3*Po)GCwL9549!IZ98NHg@E-&^sVZRgMsp-85!C)|bqMP>kqHj8JOD+q%T-LN%HW#&-%ERqpzL_HsjA?BNmi9Rx?b zhM=q@skhai*M8oHL8hkR`bRGpGf8u2keej)kKQA_Ivyj1`Q3zynukyN zwHdExjK5)#!_~z}Elh_0*~HJJ-PsZf^yyOrqq5V68t1JT(k$$|E*5((GM)d{SyuN* zv*WMh*rPc8883G6XI-ze0L`3eY~|+W=Cu33aW<|x&WH@!uTm1#*+4cH+0?lKZQGco z5^LON7=n23v`gsqwgvZl@5oA2j|8&%v2F zk2{E8To>B)9)fZM(W@|Q@U+8YUnaFCR{h%4cJEAF4ExTs76e;Y-kV0v=D0SdM5x2t zchzMV-*8L*9=tLt0N?MuA^-pbDK{hHxx*uaiOO-D!m`0-xUhuYmn0idbI%!Gil%3kWoX$lN6Jda8HdL!}sFyL1j*5$X!6 z)CJ~HgqPWU(-zGdw#Uz7F@o!p9V}-IN$nQ_ba>WZX+K;a&oSA+32lk(17_gUXM7i# z10HuMx-j!$&WMdxIePQ|NclZFZO~BNzMKqyn;@N;Yfr`sBPxEMAbcxm&Z_}8OS0fG zotaALdO9Am&~~#02WyY`w_EM*Q;O77SC5F<;MX7Yq`qx^qCW^D$#{;l-REBF?P1{$iKy2@%J;X3 zPXg(#5JRa`2TfTqd^5cCoPA)J483+-6Sz%2V$@XY4DqhbfRBi02GDh1bN4WP5Ks8^ z9(A}(BFUD^9_g;~hZkJ^;{D6 zobHQ+A&(M0E^|9tlY5PnCha=`eJ5U|ug{eUojZ$>x-YG29S&ff+@TCXnD4Vokpi)tu)H8gXrPOiY>~faZPC zwb9*;<~yK6_&61bXqniJ0i375S=Nr21JPw#9tL`z3jE>IqKbl_(yFsc8iK(?l%Ltm zrIrQu!Vrsi&~;r9_0~LvQQ!w8IM>*Daq%S92*3`JHTGmf&P#J+B_AN{5J&mrp`Wkg z8PW!H22Ik*HMn*;gpq->sjaaTSX5Z#B0G3=vI;@C+d^H&ixLLq|k&&$LW8BdOF)>!BSZj*iZsSa-Ok zgRYt`D_HZEkbVF6{J#9?<9x&n;miFggQB^(7gM2bO{P|-9@r3~@~XfI?d^(1Q^yI< z#o&B}oUnuIbYPzA?y|liNa&mj`*7)BLv1K)6HfFn|Ctub)u>C^ zA;LZ+q(= zeH|}>;mj3J2W?%{<8bj7JzvDS&1dE+o&NG~ix(6xplj^0tH>!4b=e*lCX5=# z&WibF?5>?{E&aZ0SSQz%NgIR~gyijwUq?DB-s?BV+Glhn^<6fPZgX^Dg_fBv%LnMy zK?;&UNN3r%PL8Ett1Ea~j;}*^k4tsUs$F@BhimS>4;)^!9Ejg8U*k=E>D6F0CWip< z`z=w6eQb_69dr#*k8z7b^Hch?*wF)Ju2y7>tuv!iSAgm2IQc4#^;a=$$|3S$ZN41X zzTHOgt<)}Lz-S=loRKcXGy1o-)<$Z736TD(6z1cO%NJYocsk7fzd}1g`3%nBmd-Q+ z*^|tlq#t4OmW-Oid!^$pe_{=~uN)PHCJk-C--y`G@sKs*x8DM#9tlP zg?&KWiq-gj*u=%^z!e_q>N}KHP!- zg7@GX$Z>e@%LG_WHaBa(qLLOigm(OZqw}BZb>ToIF20dTNgerXG=DW5IYy@~?&*yb zxLa<LfhZ-+f3l2#I8Kcf|m2LQW zyb@S7{lCa*tNB02`8HocI`ocIHDx(4%l z6-*0CVH$&e-;E-$7Q=p&80J1S7;(myJ!~Pd9?k&7vnlHZ-GV#m1F$MTekqSyePxV$ zTc0orFO|Y1zgRaEEMPqFmFSsh2EaniY3AX06z(+^7Se4SmAZv9Y9D z!+ez~7e+jjd`yG(`2pp&@{%-~H(Oux}d`*pb_4wQh{-+^pmrc?tk9PPk^d_>>1RK-eC%_WJ`$3fS_#S0EaHci$zS)H7E*#IGKWbLe z)q|A+9EvNUM|3(@xO>!)pbCHp&Q;1g+Yb`zZx}XMvabd<>Ej{g z=y4(>BpgAMi;bZ-GYAB=CP{1j0bS*5Z@ex^91hH(9#2^uJO=0a=6+! z&8nRRO+#YUn^5267-k<>#FuOJw0f}Y)G>!0kUEb?+?+;S?(~CXJ?9cJD@IdSM=xeJ+ zSJu=&bcQ*^V{z%bMxpoCdY@X4l6^6^uc!3emNS#Ab1F^1vXV3nQ8N>xx?)F?%>FaY zqfY1kX9q;<#_y2_?RsZ+m$XJ79V^(zrL7w*1(RU-aU_oGS%+tO!l*x#Yu0`0EFS>GAFqN?E)gu+`{!R z)zXwMu-6cg##zrN_PE1rT#c!TKj8H5&pG6r{FrZ+P&uU}ro->)VJKz1Dux%}t?}fVzFsJCbq3$<_Fx_HiJzAydjxZdW_W_(23Be@c9#&Zjvsu| zT3piLYr=KKM+X!!sLZtOT)B@wf`6LOom_xs2bdhtD1g&h6w?>@P-`s<dgZQAJ+7l4N=OZ9r0T{d7xvwBy@Om+hQ<$FkTF*82Y;ou&p0YozSR+x z?^|s80#yNH)9aUA^+Il(nOb$)`x?NeU#aTR2e)v9qtAb$!8)Fp7=)Q!AjZ<(CT3e9 zk=h@b=SX6Kn`#z%W(UI6`k%-e; zsq%YaZOat;NSm7Pgz*F@RtY;Z4()n~VXs7u25*~Y^!!dbXFfR{98V)RdSWQ4+3sqW zIxHEX>ag^Ca{p}EFNdBwme?#T!McRo9@)V#@D0Fh>n%)Tcnw%>pWLMCyyj(21#--6 zM=qgPdqVIesH(Keh}pYSy9hfRHjkG9Zz+0NR|5huIBdJT9BHhhe%vB|t%>?pm(=9j z3#SeIutSmRVclJf<>L)!lntW>Qt$f-9B#07JsNVb zYOY=uZ{h!JKv3wjv~aHF9SYB*k0u9v%Zm-7ITj}Hsx=p0tXu&sWG+vjEPXL~Qdy(RKr zT4$%@dBpRpv&YB}_!|zbCon?Uh^!rU*pHON=)5=18)`s-GCafatF9o0X`{b}BX*i4 ziQ$etv1s3racxxwcRw&ZrK^vy*y~AD4Wi)N$cwwt(KiCe;BzWf+H!VYN>aXIF9vQt zSshpAfS5PnO5SX6+zLHc5qsEFuHIzczSl|#w8g8q97dy7Ef1lnjIG5@3wetJqk!LT7>IFYLMm442#}#k+Q&hscZZhGCm$6Sd^Va#NDIa;w z1lQK4blFVy{c&=grs6B#!fb^~5rKBfW%z!@ewPR!ojAChLEV+-8YAcbP}wnWy}ptw1N{lgp#-t}~n;B~GGtCk9ij+*NQAav%B6(Pm7&UVh9Yqo=BpUlTMd}&C& za5Z(h*>K}#=vOa?C;eT8h|}V1VHQBYHZwJF)ZCWXlY~1-x<6o;`W>L=Xg{*!!?4v2 zS?C$eM`?>~!|{fs^=9lu4!Erw*1tqM&b&^p3AMlv*WIAPY|U(Jaia$u)@(28oP$nP zgcWIMP*O>DZhy3=x^})1)DpGlTt4i>4Vvj$X!OFsOYI## z!W3$bnb5}Fo@=<=)A;__NS+qK__p~F9~6E(YzF-}6!}6?m9p}2Bt$l!?~T0heK3;8 z?0O?`cE-+&33ae0s(xki&GPp#BPoO1=isYY2eBt7!bCZm+n$<#6Jn}GTsu`-sbS08 zR({EF{nu9(zL!Z9lG=xjG4^*_B#sU_`NwWxwP`eVv0lW3(F9SQ!vVS*j>XfVeWxNV zP;cV!Pt2i04{zPw=?1&gP>|!W(M7+6Xf_%KlDNaYcTY`)#gPKuZEY0za*#dMnb-G$ zqXaB%sDorgUVlTC>N$joRq7ErIvy9{GQgmIx%Q>^qTx5iRx;vLGp}NL9zbX&nEqqN z1XiXBB95*n=u*&ERFRf~pFqk5vxIbmsyrR&74ar;IVJE$=#R^xAmuKqp49zhM}bEr z?g!1XQf<{S zv>!9bMVGa*W%j5CWk6~-DtVw4VZhwY_072h`%sGr=|zS*;)+?vR-;F;KKYw17zA1P z4yUu(wy=;j{L)Hme>At`?YUo^ANBz0q+TZB>lYqE-7L&Zz?A}z{9WQmP@YL=u|ETC zJ`|VoeSepanl2A00e{;0RyfFio+CUo(_C#WJ{9TxG*!aXR+NR%Av(5*hjyDNQongrHM+^4j zwUW-NcVHwKDu2gg{q^C(-X0bQbYzMg%s<}p@sQEVE_>;R3%gO4cKfCHGrVDY$xky) ztw5Zky$Ll!$C>}>< zu z4sdS!0)oF?aM#W7Z7h1crJHxCtPp?OM=tI;`k&*BfA9M9f8${8uOsWRip2N3o_3_^ z6CB86OVb@vSViL<_aidKOe;!M1Rn~Dn9f_?&i4dAnSb8D|LGMFKu$g*7aX1(dBra8 z0?KtE55Eu%-<^=&#o_9X1zqJU2rA&A7T&`}6A=+SYY*pp4&VEnP&LYzy6hyG*8$^b zfrRAcNSSksp+k?>@_uvaqJg0ioTAOp)QT!Ye;AM0)ynSf@I6V=Da)^8`~68J zZR|f49pSvpe0pOUoF%etx@33GOxiUa!6zRd-gZQgU&p4!iv`CdBh=KX3Q!rlGBJ^*7*Ols1C$2Y0 z;c+BFe*Qh{3}hZ2q;n_!U482Ey0jWpTgZc&^de$gS4t9JC6VlsX3>kUFLjG&Of6fw zs44oys!i{<%{HU5zJwchUm$i(O!vCy2ur*A+2({368xEq-!Vt;_H^&E>&m%zma2ZG z;w*}gei}9rkgcjiBxu!No)$Po*1V{Z5>Zt59MyOY^Lw#Ug*uVTpkU9`tSVn8vu=TM zw4-9O*Ipx1spgGCj0vAhVtw)@xe8dZ>^u*ZP1Y{eRT3#Hmx_j4)2AMPI!r*i1C#pA z@MCjo!ed!?C~hWbLLnRJ-Pn>$tXVLzxa#omBnWmaCU#CFg`bIKGtfUe=Cev{0WKxC zyJ|%z#k?tLi>j%$_UHPlF8+|68fXEdIHaxKXE9et|IN<0kBW* z3QXDk9+p=<6Q*eWkAUImA$9( z!z|O5Z9{vI?9+nR$b`xdjlc;}eu*TM<_MW4iN!D$@@PIg;~oDd6Hzo3=9iJxW0M$* zOaV5kQv+H5KyOheF~UgtoK_mCkW)-`76S=qbT)?<51J`d$Ug!mxaK+yQPIz<)P_(Apha5kuTSj{G7T5o#OEb?-WKspA*j0aAAxYyk%b&MSw69 zPn{C^2Ylyse^(d|k0L@X+WG8Yfd}AfcSDTBhZt2aP_Z`otpInpgtSO@^1Hab&_j=M zZZx7e9Dn!8Op)*HxLFbOVrM<>8FE8&=Vy7+cTiOV|4c#cBW@zdW>|%|ZBhSpR85@O zG<^7mjfs^>3@Xm*d{DI(El^#Ui!+JykkT!lM7GA`tCS-w?W7tR0ansa-F2VLCE5pm#>&fOX0Cs$B_a!@dbSJo;b_Q1zzsnVigSl2assT9G5G|%;U@|KFop>t0`$-ZTD7{Uo z`Dw{U;w^(3KPlNnE)MvrlkhXR>jD@(z%cj^I+$H*o9q#Oui=e0rb(Y`;vhI7hdi$l z%nq9NOs50>GWN`-KLAyLv|%^d7o`{;_&4)Qm@!J}EPo$pnt~iFzV4}^Y#{yf=CKZY5G)BjIczavz2y@Mn2bZIgLV7k$k)n!jq-e8! z|G9p!=u;<+TgwdtZJ}jb-Lw^#!QrNwzk|FS9KT}B(+!3meu0#vTx{&{P;xj+V+Tpz zDXvaQq~zZ);mfJH)T6k3kKx{?kiw12?^LO|Lc^PWCH6#cc8xA3I^kUi>_8}Y4ik}0 zq@NvU9m9C^m=Ev5P0p@Q`>cFawz=vmdo!HCnTy5;IZbx#fxUFg%g{18Xrxsx#oKGs&aVKO=Mv& zkIwj08a@+vB{6>{mTzlYygV(4sRw|H>JVZ08ZahUREWEWzFr1qQaRX`|%fA<@a-JN=YAU&$Z_nh#<2OPNcm;ii4p=GpH;C+o(DIEHSU zrC8YTJr~cDT8MoqZ(z?a1Wnn{Jl$2cQA>-qw<$TUy^ZuX+I%L3<*P@XK;&s`J`r(k9i z$sE7875Q${gp*oV%{g|tJikFa=@lxj#(bu8RolBv!c2xT(^)o1lxEQ!NN`+NgqVBC zn7N1hk0rO2MgyA;zG#`(4hL$PjrVRA_OB@nclc6NlX6M%(Z#O(nJJ{A%*7K~)H%Lw zs$JeGSM4|!DqzK?b5xGGx%z{tj7O>2yS02a^=WPbxrW;{CZwE3BC49hv;tfqQ2 zEL+OeHCRD$<8Pk z3^KN|8?JV@Rnd(}-d%1U%!%wk(YHC6(lTs?f#LJAQ{W(-%r6*FD3cj)e1&j%=A9s3 zEk+0#p6EwF-!oWN5*d22X{(kI&GP`8c>dzp;;52tCwu!7r3PkVl}@@=s&fnfgj6pR zhD*36y*7t~Si!_qiosbYcUGLoW7HPZynvrld_%6Nn+F>3jVjw>?tYR&^+6B1W|SDU zL6tpxnqn8SM$#2}8hVkXwR$ijbEsE@5uC*+VZLZ>X{R|Ad*CdL>Y|ebJ7fyM1Z`pLz zOyW{ZG1@jYG5d~Mu5P^eWa+d*JSZ?MY7#5E_A$zPee%;naRG6Z{lItbrBdOv`$rZU zwfI7VJ8CSR8-|+}X;3u4xF_dE7Zwo5-L8%wb3PcaVO0wJS;eL?45Fo~!U!F3A7fsPtd=!qRu29R2PM%QJ zZi>q#J$9#24v$Q{n&A_R%G9ZY%n;j0?FaA-K~kx#gnn3U?(#{w5D8TK-1NEI|3H86 zDDER>VqOKk#b3+& ziR<1}-4-&c-Xe~{h{+|e-h<^fX_kp{MWW1-#0#?`;Fhrs7U zWyQuIyjE$?Ktb6l|3>G2FRs>%Q~CYPAD7i#AMpuMV{Lr$|kKw5!;fRKrjaF zNQ1h2_MmfUvS-^QxD@N~!K?)RVXvm|Zr!SCXHuhhM#*k-V3W$ueYh*v$v5^Rp7wPu z!=84NP}+eZk|wZISEdT$`eiA$Rl~CwEueX#(jw{A`0S)L-uf|8y@BNw(N^Po{Bau` z*)NU{6`XUO-bz*VxaUa@X_aIBCZBaxS2)U3)>mkp;1yr3ilKyvP7NvL%jX^KVpAnm zX({e6kB+e|Pve~n)ZtPS$LJ@@K;4uRintHoZWSga4_jLDH2ZGHk*MFA57ukXJw7s8 zuvrZ4a@0#~+|Yv=sGo4w9C^umKYDfM{PX@vYQJ-P%N8i9D22pq*cENO zpRDf9<+4w^ATFp#kn#wv@4F8bxaNiE8(F7?$(*Ew3d#qg$O3Z*(zj9kW5>!Nm7?m91Xn;kY%P?&#K=1N87MxC zml@eS@6(}cP|R)rqJ>VLZaRjpB}#*3DfqHPmM$(c$36|hMcaXKfc|Fa>(G0X{ki9h zGs5amgh#FaNm7b)C#=aZ*r(($LtLij!q|*N9jh!=P^0kwq;VQ}8ibfXmYSkiad}W! zJ7evPBeD^(aev_Mu4XxUB4W7ji-EDXnKn3@l^)r>nuy9|-)!ZW=$q*7SyR}sO|6or zy1TU$8h{WpII*FXmb;K`hIJr3z5z+v+FDn4@-TyPB(jkyrFQIJ?6#SS*rhO;Be~5& zoT#I`ee`4G6`O;FOUr!o3`&2;ZLd}7RS&Sb*XMjKNli#(ff%_sw^1dsG0EF#4nDjG zmSmK*cBq#)JczfY9*Y37zDUU{n^_s9-#ZfB+o8YMq7ba@s#w>el`)u$2-ZI7(R;(t z;K)z}cRBm$pr?F$2S$pbnV3b4xM0w2xbvJfm>2o!X^~d=1bNjvMcSzm z(NMxa_AVwBPOdZ^KIpqro|9*GflDVq3y2 zvml4hfKj8;jnR$T*_IrQvt9lYr^=`4Hl)%8?W)Cv2INosB~7i2HW`LosX1n8vc!o= z;a00R#CrS+jV*Gq9!Ms_i5%iFMJb#~NsU9U8!CuhSbTCD|P%`OC zp>K>%idcYB@nZ6A2_-N+##RfYa}AFI?q$?6$F%+94gE`#yi-=c))TFE90{iLv9%5U z*gw$oCsI3DDmuw1IR@r+`|+|qrpVc(OGJrup|QLP501+&kQ|yr;ovF{LMZ%|+1Vgo z4(;SfoUWnJn~dP#$)kj=3v|bz9C~%w033Fl>!4JfEfL zK%1T&uh1#4ySYYBT`s@0C{{_$MT4d5OXXPaGut%^6;tpmY}}R!k?*OqJ*0~T*`)>1 z{WP^swBlKg`ki%cc@~Y)+Zb?R=)@IOv>J55O&dIjvE*lD+-=7zC+*$U@M7eVhLTH_ zZgDaS%C)fjpG_{2ChKMkwhZ#y`=4Y&;<#1hthEjF4P33R`YC6e737M;*gsRzlFDH6YY>=+94qbKEQvpA zRd@D%QOE)ieQgE5A9*S|LFoWU&Mu~Fz+{^&4;fO=(;2wC67nyzWcI;yza&U|}b zwfzovWS7Up#i{*cc9sDpcZttCkHk_I*v8&0&b&0a*H>R}!ZzpR9n>^$ji}mO6kb79 z^GwBZ`xb*F-A6dnl4J8jjxVP+JJQTHx54&p_f93mPCALNcUohdw_{HSjqboQGLa80 z?{n!^4k#ICB7%c*Fys_@vlQNo*9_y zZs&(7E)*10>X@p%d$w-DL4{50h59}Y_~*?;fO*Y8Igrlk#5UzDZ~LJm`t#f!G>wN` z6t;znAg)j_tVFC2i}wp=e!lqN$4tV-yfzZP?Ze-2N)nO)(rA&;+%_E^5<}#`JPksgB+``C*w&L_R zw#wM0Af%|`>BYC6B74G`R`*G0tXg&+Tnb?jbTgI~y4zy4YaoUU_HtxH+d^S>*A7@( zm_x1Yp_I$13diq_jgnB;(Uzos7;BN6G&(5VWmw>3Wu?*X{muE`JBxq|afO1h#r%oBHT`B-J-IrY$Ld8CbY#}N8|7Q zRszU=`6H>G=*H)G(C~q07O-vPC&hURr?+$$o3Z>H*bVgm8m7IxR^)D`=s^1f53?)BF?FFHmVE)o_5__BnkNJu6$UVCv0i;CuDWMpjS<&BVrg8`E? zV3`W5n%7}&RR4rq>LB5`Xh8c6j?Zz~K2CZnbvlBb%mpATeJzhws!_}~V!_F;l^dE!Nb0oXms~uuNOong`x>E^)NGf}t#ieL5T<~RurSlAM1j5^5;oOU z&n&!!f%(`8+0VuBxaCE~!Xk5YZwd-V>UCeTIC>3Z5mZ&F=y2Cm@%mMMPL7(@{T4KD zei2&Y@tBM;6#6LDn!w}WOv zd%~F_3Wnodt0nQcNPyqDS?n$HAcx#?u`SVxy?s8Gl3g{Y=pw93N->giqwFVMkKz}^ zhYI7RPV|{fI0ath8OUavPUEW($MQU|1Byp~rCmO_vJh~4Tw}*zcoLUKp&F0hQwcnH z-XRCDhUtKiheKa>wa=R2@K*gj(^B4$;Epz1HUt%uv%$e%$nWl?MOY;gRW2m&?u zJjJEKh46xZyfuKkhS}w;vOX7TeYwlq_@#k?oZG{xCJg`T@$Jg|Iueg7L_#6NUDSs_ zp_yI*<;RZw`w-U5@|Y~)pV_U4dJE{vIg`IV=7XD7x0B3}oR2^F{qy|R2^KvTM+nLo(7RKG^CJ+wEXPJwhR<6Lh`25m5IFk_mk@9uk)8$;N?e|?E+?1Ms_W% zcn+8uOKvZgj5dwb`RM2T5<29qlGykZsQ{j*_q#xtUopX3k@xvjbr+6t-+R;Bgbt}H zq7~pbXUwpvORW4df7^gz1LLG(%{Lyhp=V$v_{iDiQOcLOR2|NFTpux`E8Gf5@!Hkq zlZBa4tW(O^t(ce#%nQ+D5!@3NNPa1(aWHG2*k^z(77q0-F%0!B8@!?ApX>JSiDiV1 zH4J?f@4@y#Un`tpL*c`h+?tfKf5yc(KS595gq5YGZss2E!aa>3iH)E+h=uSi$*zgP zDlQ2gb?`^Fs@}7l<-rhF?(PVAsh26;4liGO%QC|$PKuUM%GbJN7e{*3HVu5qiR7KQ zVDe8QT6y_*Kwhsy+VZjBl_o{;7rP_Sq?lk6gttz{a4ZWVd5qKGgl*L0JCrKB+oyAw z(fg_ux2!8DLVaTmf5!haAJ*F@)S@h=)NU5ycH5>BYFJ#X=xT{o7aFahME?(e>*x5t tPSZcXBN+MrKX3axLI0OC;D0*_#V0&I_xIid4L*G$CL}Fb!ms1~KLBqhPF4T_ literal 0 HcmV?d00001 diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md new file mode 100644 index 0000000000..af849a9e43 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -0,0 +1,54 @@ +# Migrating from RavenDB to SQL Server or PostgreSQL + +This page covers what you can run today. How the migration works, and what is planned, is in the [migration overview](ravendb-to-sql-migration-overview.md) and the [system design diagram](migration-system-design-diagram.png). + +> [!NOTE] +> Copying data is not built yet. The one migration command available is the source report, which reads the RavenDB database and changes nothing. + +## Before you start + +The source is a ServiceControl error instance on RavenDB. Keep its RavenDB settings in its configuration: the migration reads RavenDB through them, including after `PersistenceType` is switched to SQL Server or PostgreSQL. + +| Setting | Environment variable | What it is | +| --- | --- | --- | +| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | An external RavenDB server. Leave unset for an embedded database | +| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The embedded database's data directory | +| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database, `primary` by default | +| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database, `throughput` by default | +| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | A secured external server's client certificate | +| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. Don't change it during the move | + +`ServiceControl/Migration/SourcePersistenceType` defaults to `RavenDB` and needs no setting. + +## Report on the source + +Run the instance's executable with `--migration-source-report`: + +```powershell +# Installed on Windows, from the instance's installation folder +.\ServiceControl.exe --migration-source-report +``` + +```shell +# Container, against an external RavenDB server +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-source-report +``` + +From source, build `src/ServiceControl` and run the same command from its output folder, as in [How to run/debug locally](../../README.md#how-to-rundebug-locally). + +The report prints the RavenDB server version, whether the source is embedded or external and where it is, both database names with the setting each came from, and a row count for every collection in both databases. + +- **External server:** run it while ServiceControl is running. It only reads. +- **Embedded database:** stop the ServiceControl service, run the report, then start the service again. The report starts its own RavenDB process against the data directory, which cannot happen while the instance holds it. +- **Container with an embedded database:** not supported, because the container image does not ship the RavenDB server. Point the instance at an external RavenDB server instead. + +## If the report fails + +The error names the setting to fix: + +- **"has no database named ..."**: the database name setting it quotes is wrong. +- **"refused its client certificate access ..."**: grant that certificate Read access to the database, or supply a certificate that has it. + +## Not available yet + +Copying the data (`MigrationMode`), the dry run, and the status and verify commands are planned but not built. The planned steps are in [Migration workflow](ravendb-to-sql-migration-overview.md#migration-workflow). diff --git a/docs/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md similarity index 89% rename from docs/ravendb-to-sql-migration-overview.md rename to docs/migration/ravendb-to-sql-migration-overview.md index 93306b9566..277a53c783 100644 --- a/docs/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -15,19 +15,17 @@ A customer can already point ServiceControl at SQL Server or PostgreSQL. They ca - **Minimal downtime**. Only the required data copies with ServiceControl closed. Optional data copies in the background while it serves traffic. - **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. - **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. -- **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and RavenDB is untouched: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. +- **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and the migration has written nothing to RavenDB: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. - **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. - **Every identifier anything depends on is carried across**. The event log and historic retry operations are renumbered, because nothing references their keys. - **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. -- **No silent loss**. A migration cannot end with a selected category incomplete, and skipped rows are counted and reported. +- **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. - **Bounded impact on a live instance**. Throttled behind normal ingestion and streamed, so memory does not track the size of the database. - **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. - **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. ## Deliberately not built, and not currently planned -*Each of these was considered and left out. None of them is scheduled: if one becomes a requirement it is new design work, not a later increment of this one.* - - **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. - **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB, so once the host has served traffic there is no rollback of any kind. - **Steerable while running.** No pause, resume, or abort. Changing anything means editing configuration and restarting. @@ -70,9 +68,9 @@ The copier runs inside the ServiceControl host, so every row and every message b ## Migration workflow 1. Upgrade ServiceControl as normal, still on RavenDB. -2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated) they want copied. +2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated-categories) they want copied. 3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. -4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the estimate is worth](#what-the-estimate-is-worth) before booking an outage around it. +4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the dry run reports](#dry-run) before booking an outage around its estimate. 5. Start ServiceControl (`MigrationMode=true`). 6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. 7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. @@ -113,7 +111,8 @@ flowchart TB target --> bodies ``` -- **No provider-specific code in the engine or the source.** The source is always RavenDB and the target is always an `IPersistence`. +- **The engine and the host know no store.** They deal in categories, cursors and counts. The source maps a category to what it reads and describes itself as labelled facts, the target maps a category to where it writes and how to count it, and each contributes its own startup checks. RavenDB to SQL is the only pair built, and another pair, such as SQL to RavenDB, would add a source or a target without changing the engine. +- **The source reads the instance's own RavenDB settings**, so an existing customer sets nothing new. Leave them in place when switching `PersistenceType`. - **Both persisters load into the same process**, each into its own `AssemblyLoadContext`. - **The engine references neither assembly.** It knows only `IMigrationSource` and `IMigrationTarget`, and treats the resume cursor as an opaque value it passes from one to the other, so it can be tested against fakes on either side. @@ -172,11 +171,11 @@ flowchart TB ### Not migrated - The fifteen RavenDB index definitions, which map to a much smaller set of ordinary SQL indexes, and two of which are dead already -- The transient in-flight collections, which are empty when nothing is running -- `ArchiveBatches`, which exists only because of how RavenDB works +- The transient in-flight collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations` +- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works +- `ConnectedApplications`, which only versions 6.0 and 6.1 wrote and nothing has read since - Integration events still waiting to be sent when you switch over are never sent - Broker and audit service version details, which refill on the throughput collector's next run -- The last computed licensed-endpoint count, which is recomputed from the throughput data that is being copied ## What does not come across @@ -187,22 +186,20 @@ flowchart TB - A failed message whose `UniqueMessageId` is not a GUID. The target column is a `uniqueidentifier` and the value is never regenerated, because it is simultaneously the primary key, the ServicePulse URL, the retry correlation key and the body lookup key. - A failed message with no processing attempts recorded against it. The SQL model keeps the newest attempt and derives the failure time, the failing endpoint and the exception from it, all of which are required columns, so a message with nothing to derive them from cannot be written at all rather than being written blank. - A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. -- A row already past the target's retention cutoff. The sweeper would delete it within the hour, so copying it would write a body to blob storage for nothing. -- A group comment whose failure group has no messages left in the target. RavenDB never expires comments, but the SQL sweeper reclaims orphans. +- A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. **Things that change shape, and are not counted as skips at all.** The dry run counts these before anything moves, so they are a number you see in advance rather than a discovery afterwards. They are also the ones to read twice: - **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, in the one category every customer copies. A message that failed five times arrives showing one attempt, and the other four are gone. - **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. -- **A subscription whose message type or transport address exceeds 200 characters cannot be stored at all.** -- **Throughput endpoint names that differ only in case merge onto one row**, because the target key is lower-cased. +- **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. - **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. **A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. ## The one point you can go back -While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL and RavenDB is untouched and still authoritative. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. +While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. Back up both RavenDB databases, or disable expiration on them, before you start. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. @@ -211,7 +208,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - A third RavenDB lifecycle opens the source: connect, check the version, stop. It never calls `DatabaseSetup.Execute`. - Both source databases must be on the same server or cluster (`LicensingDataStore.cs:35`). - The source has to be at a ServiceControl version this build can read, and nothing in RavenDB records one today. The only version check that exists compares the RavenDB server version to the RavenDB client version, and runs only for an external source. So a marker is stamped into the database on upgrade, and a source without one, or one from a newer major version, is refused by name rather than misread. -- Duration scales with distance to the source. The copier already holds the document from the stream, so each body costs **one** round trip rather than two, but it is one per message and they are not batched. Egress out of RavenDB Cloud is billed to the customer. See [how long the background copy actually takes](#how-long-the-background-copy-actually-takes). +- Duration scales with distance to the source. The copier already holds the document from the stream, so each body costs **one** round trip rather than two, but it is one per message and they are not batched. Egress out of RavenDB Cloud is billed to the customer. See [batching and throttling](#batching-and-throttling). ## Writing to SQL @@ -243,7 +240,7 @@ That window closes the moment ServiceControl opens. From then on new failed mess - Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. - A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. - The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone lets ten thousand failures pass on a five-million-row table as "only 0.2%". -- Verification therefore cannot treat any count difference as a fault. It accounts for all five skip rules, or it reports every successful migration as broken. +- Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. ## Dry run @@ -257,12 +254,11 @@ What it resolves and reports: - Rows per category, and message-body volume per category - A duration for the window while ServiceControl is closed, as a range -It runs the same seven checks that gate startup, so a missing setting surfaces before a customer books an outage. +It runs the same startup checks that gate startup, so a missing setting surfaces before a customer books an outage. It counts, before anything moves, the rows that cannot cross as they stand: - Documents whose `UniqueMessageId` will not parse as a GUID -- Throughput endpoint names that differ only in case, and so merge onto one row - Subscriptions that differ only in message-type version, and so merge onto one row - Subscriptions whose message type or transport address exceeds the 200-character key limit - Integration event dispatches still queued, which are not copied and will never be sent diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs index 8184bc21de..737a26bd4f 100644 --- a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs +++ b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs @@ -32,6 +32,20 @@ public void TearDown() [Test] public async Task The_report_names_the_databases_the_settings_and_the_collections() + { + var report = await RunReport(); + + Assert.Multiple(() => + { + Assert.That(report, Does.Match(@"Mode\s+: external")); + Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); + Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); + Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); + }); + } + + static async Task RunReport() { var settings = new Settings(persisterType: "RavenDB", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); @@ -48,15 +62,6 @@ public async Task The_report_names_the_databases_the_settings_and_the_collection Console.SetOut(original); } - var report = writer.ToString(); - - Assert.Multiple(() => - { - Assert.That(report, Does.Contain("(external)")); - Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); - Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); - Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); - Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); - }); + return writer.ToString(); } } diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs index af0c18df37..61958afec2 100644 --- a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -3,6 +3,7 @@ namespace ServiceControl.Migration.Tests; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.Loader; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -87,7 +88,7 @@ public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_ide { Assert.That(AssemblyLoadContext.GetLoadContext(description.GetType().Assembly), Is.SameAs(AssemblyLoadContext.Default), "A shared type produced inside the plugin context must arrive as the host's own type."); - Assert.That(description.PrimaryDatabase, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), "The source read its own RavenDB settings rather than the target's."); }); } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index edf971e7c2..cc3c59a998 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -2,9 +2,12 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; +using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; using ServiceControl.Persistence.DataMigration; @@ -18,24 +21,44 @@ public async Task Describe(CancellationToken cancell var settings = lifecycle.Settings; var build = await lifecycle.DocumentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); - return new MigrationSourceDescription( - settings.UseEmbeddedServer, - settings.UseEmbeddedServer ? settings.ServerUrl : settings.ConnectionString, - settings.DatabaseName, - settings.ThroughputDatabaseName, - build.ProductVersion); + var server = settings.UseEmbeddedServer + ? new MigrationSourceFact("Server", settings.ServerUrl, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseMaintenancePortKey}") + : new MigrationSourceFact("Server", settings.ConnectionString, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.ConnectionStringKey}"); + + return new MigrationSourceDescription(build.ProductVersion, + [ + new MigrationSourceFact("Mode", settings.UseEmbeddedServer ? "embedded" : "external"), + server, + new MigrationSourceFact("Primary database", settings.DatabaseName, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseNameKey}"), + new MigrationSourceFact("Throughput database", settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}") + ]); } - public async Task> CountCollections(MigrationSourceDatabase database, CancellationToken cancellationToken = default) + public async Task> Inventory(CancellationToken cancellationToken = default) { - var databaseName = database == MigrationSourceDatabase.Primary - ? lifecycle.Settings.DatabaseName - : lifecycle.Settings.ThroughputDatabaseName; + var entries = new List(); - var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + foreach (var databaseName in new[] { lifecycle.Settings.DatabaseName, lifecycle.Settings.ThroughputDatabaseName }) + { + var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + entries.AddRange(statistics.Collections.Select(collection => new MigrationSourceInventoryEntry(databaseName, collection.Key, collection.Value))); + } - return statistics.Collections; + return entries; } + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot count category {category.Id} yet"); + + public IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot read category {category.Id} yet"); + + public Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot read bodies for category {category.Id} yet"); + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs index d5ecc8694f..354587cb5e 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -13,13 +13,16 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using Raven.Client.Documents.Operations; using Raven.Client.Documents.Session; using Raven.Client.Exceptions.Database; -using Raven.Client.ServerWide.Operations; +using Raven.Client.Exceptions.Security; +using ServiceControl.Configuration; using ServiceControl.RavenDB; -sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings) : IAsyncDisposable +sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings, SettingsRootNamespace settingsRoot) : IAsyncDisposable { public RavenPersisterSettings Settings => settings; + public SettingsRootNamespace SettingsRoot => settingsRoot; + public IDocumentStore DocumentStore => documentStore ?? throw new InvalidOperationException($"The migration source is not open. Call {nameof(Open)} first."); public async Task Open(CancellationToken cancellationToken = default) @@ -40,9 +43,7 @@ public async Task Open(CancellationToken cancellationToken = default) await StartupChecks.EnsureServerVersion(documentStore, cancellationToken); } - // The persister cannot reach the host's Settings class for the root namespace, so it is spelled - // out here: a customer reads these two keys back out of app.config, not out of code. - await EnsureReadable(settings.DatabaseName, $"ServiceControl/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); + await EnsureReadable(settings.DatabaseName, $"{settingsRoot}/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); await EnsureReadable(settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}", cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -61,18 +62,6 @@ public IAsyncDocumentSession OpenSession(string databaseName) => DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName, NoTracking = true }); async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) - { - var record = await DocumentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken); - - if (record is null) - { - throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is."); - } - - await LoadDatabase(databaseName, settingKey, cancellationToken); - } - - async Task LoadDatabase(string databaseName, string settingKey, CancellationToken cancellationToken) { while (true) { @@ -93,6 +82,14 @@ async Task LoadDatabase(string databaseName, string settingKey, CancellationToke { throw; } + catch (DatabaseDoesNotExistException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is.", e); + } + catch (AuthorizationException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} refused its client certificate access to the database '{databaseName}'. Grant that certificate Read access to '{databaseName}', or supply one that has it in '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}'. If '{databaseName}' is the wrong name, correct the '{settingKey}' setting instead: RavenDB refuses a certificate that has no access to a database whether or not that database exists.", e); + } catch (Exception e) when (e is not DatabaseLoadTimeoutException) { throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it.", e); @@ -100,9 +97,11 @@ async Task LoadDatabase(string databaseName, string settingKey, CancellationToke } } - string Located() => settings.UseEmbeddedServer - ? $"{settings.ServerUrl} (embedded, data directory '{settings.DatabasePath}', from 'ServiceControl/DBPath')" - : settings.ConnectionString; + string Located() => Located(settings, settingsRoot); + + internal static string Located(RavenPersisterSettings sourceSettings, SettingsRootNamespace root) => sourceSettings.UseEmbeddedServer + ? $"{sourceSettings.ServerUrl} (embedded, data directory '{sourceSettings.DatabasePath}', from '{root}/{RavenBootstrapper.DatabasePathKey}')" + : sourceSettings.ConnectionString; string StartEmbedded() { @@ -147,7 +146,7 @@ static bool IsRead(HttpMethod method, string path) if (method == HttpMethod.Get || method == HttpMethod.Head) { // HiLo persists the id range it hands out, so it writes despite being a GET. - return !path.Contains("/hilo/", StringComparison.OrdinalIgnoreCase); + return !path.Contains(HiLoPathSegment, StringComparison.OrdinalIgnoreCase); } return method == HttpMethod.Post && Array.Exists(ReadOnlyPostPaths, suffix => path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); @@ -168,6 +167,7 @@ public async ValueTask DisposeAsync() } } + const string HiLoPathSegment = "/hilo/"; static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index b15f70cc04..5c74f8eb3e 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -92,7 +92,7 @@ public IPersistence Create(PersistenceSettings settings) return new RavenPersistence(specificSettings); } - public IMigrationSource CreateSource(PersistenceSettings settings) => - new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)settings)); + public IMigrationSource CreateSource(SettingsRootNamespace settingsRoot) => + new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)CreateSettings(settingsRoot), settingsRoot)); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs index e7ce5edc04..85b8fbd919 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -2,6 +2,8 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using System; using System.IO; +using System.Linq; +using System.Text; using System.Threading.Tasks; using NUnit.Framework; using Raven.Client.Documents; @@ -11,7 +13,9 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using Raven.Client.Documents.Session; using Raven.Client.ServerWide; using Raven.Client.ServerWide.Operations; +using ServiceControl.Configuration; using ServiceControl.MessageFailures; +using ServiceControl.Operations.BodyStorage.RavenAttachments; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB; using ServiceControl.Persistence.RavenDB.DataMigration; @@ -20,6 +24,8 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; [TestFixture] class ReadOnlySourceLifecycleTests { + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + string databaseName; IDocumentStore bootstrapStore; RavenPersisterSettings sourceSettings; @@ -61,7 +67,7 @@ public async Task SetUp() [Test] public async Task Opening_the_source_creates_no_index() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); @@ -74,7 +80,7 @@ public async Task Opening_the_source_creates_no_database() var absentThroughput = $"{databaseName}-absent"; sourceSettings.ThroughputDatabaseName = absentThroughput; - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); @@ -87,7 +93,7 @@ public async Task Opening_the_source_writes_no_database_settings() { var before = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; - await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings)) + await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)) { await lifecycle.Open(); } @@ -100,7 +106,7 @@ public async Task Opening_the_source_writes_no_database_settings() [Test] public async Task Opening_the_source_configures_no_expiration() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); @@ -111,7 +117,7 @@ public async Task Opening_the_source_configures_no_expiration() [Test] public async Task Writing_through_the_source_store_throws() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); @@ -126,7 +132,7 @@ public async Task Writing_through_the_source_store_throws() [Test] public async Task A_source_session_cannot_even_stage_a_write() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.OpenSession(databaseName); @@ -134,13 +140,13 @@ public async Task A_source_session_cannot_even_stage_a_write() var exception = Assert.ThrowsAsync(async () => await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def")); - Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the OnBeforeStore refusal. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); + Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the RefuseWrite request hook. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); } [Test] public async Task Failed_message_status_reads_back_as_stored() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.OpenSession(databaseName); @@ -154,7 +160,7 @@ public async Task A_failed_open_leaves_no_store_behind() { sourceSettings.ThroughputDatabaseName = $"{databaseName}-absent"; - var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); Assert.ThrowsAsync(async () => await lifecycle.Open()); @@ -165,29 +171,27 @@ public async Task A_failed_open_leaves_no_store_behind() await lifecycle.DisposeAsync(); } [Test] - public async Task The_RavenDB_configuration_opens_a_source_that_describes_itself() + public async Task A_source_describes_itself_as_facts_naming_the_setting_each_came_from() { - var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); - - await using var source = factory.CreateSource(sourceSettings); + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); await source.Open(); var description = await source.Describe(); + var facts = description.Facts.ToDictionary(fact => fact.Label); Assert.Multiple(() => { - Assert.That(description.Embedded, Is.False); - Assert.That(description.PrimaryDatabase, Is.EqualTo(databaseName)); - Assert.That(description.ThroughputDatabase, Is.EqualTo($"{databaseName}-throughput")); - Assert.That(description.ServerVersion, Does.StartWith("6.")); + Assert.That(description.Version, Does.StartWith("6.")); + Assert.That(facts["Mode"].Value, Is.EqualTo("external")); + Assert.That(facts["Primary database"].Value, Is.EqualTo(databaseName)); + Assert.That(facts["Primary database"].SettingKey, Is.EqualTo("ServiceControl/RavenDB/DatabaseName")); + Assert.That(facts["Throughput database"].Value, Is.EqualTo($"{databaseName}-throughput")); }); } [Test] public async Task An_unopened_source_refuses_to_describe_itself() { - var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); - - await using var source = factory.CreateSource(sourceSettings); + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); var exception = Assert.ThrowsAsync(async () => await source.Describe()); @@ -195,21 +199,29 @@ public async Task An_unopened_source_refuses_to_describe_itself() } [Test] - public async Task The_source_counts_the_collections_it_finds() + public async Task The_source_inventories_every_collection_by_database() { - var factory = (IMigrationSourceFactory)new RavenPersistenceConfiguration(); - - await using var source = factory.CreateSource(sourceSettings); + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); await source.Open(); - var collections = await source.CountCollections(MigrationSourceDatabase.Primary); + var inventory = await source.Inventory(); - Assert.That(collections["FailedMessages"], Is.EqualTo(1)); + Assert.That(inventory.Single(entry => entry.Scope == databaseName && entry.Name == "FailedMessages").Count, Is.EqualTo(1)); + } + + [Test] + public void An_embedded_source_names_the_data_directory_setting() + { + sourceSettings.ConnectionString = null; + + var location = RavenReadOnlySourceLifecycle.Located(sourceSettings, SettingsRoot); + + Assert.That(location, Does.Contain("'ServiceControl/DbPath'"), "An operator told the embedded source cannot be read needs the setting that points at its data directory."); } [Test] public async Task Opening_the_source_twice_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); @@ -223,7 +235,7 @@ public async Task An_embedded_open_that_cannot_start_leaves_nothing_behind() sourceSettings.ConnectionString = null; sourceSettings.DatabasePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("n")); - var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); Assert.CatchAsync(async () => await lifecycle.Open()); @@ -237,7 +249,7 @@ public async Task An_embedded_open_that_cannot_start_leaves_nothing_behind() [Test] public async Task Generating_an_id_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); @@ -251,7 +263,7 @@ public async Task Generating_an_id_is_refused() [Test] public async Task A_patch_against_the_source_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); Assert.CatchAsync(async () => @@ -264,7 +276,7 @@ await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( [Test] public async Task A_bulk_insert_into_the_source_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); Assert.CatchAsync(async () => @@ -279,7 +291,7 @@ public async Task A_bulk_insert_into_the_source_is_refused() [Test] public async Task Reconfiguring_expiry_on_the_source_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); Assert.CatchAsync(async () => @@ -294,7 +306,7 @@ await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync( [Test] public async Task Deleting_an_untracked_document_is_refused() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); @@ -308,7 +320,7 @@ public async Task Deleting_an_untracked_document_is_refused() [Test] public async Task Reading_the_source_still_works() { - await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings); + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); await lifecycle.Open(); using var session = lifecycle.OpenSession(databaseName); @@ -323,6 +335,31 @@ public async Task Reading_the_source_still_works() }); } + [Test] + public async Task A_source_session_reads_a_message_body_attachment() + { + using (var session = bootstrapStore.OpenAsyncSession()) + { + using var stored = new MemoryStream(Encoding.UTF8.GetBytes("1")); + session.Advanced.Attachments.Store("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName, stored, "text/xml"); + await session.SaveChangesAsync(); + } + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var sourceSession = lifecycle.OpenSession(databaseName); + using var attachment = await sourceSession.Advanced.Attachments.GetAsync("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName); + using var read = new MemoryStream(); + await attachment.Stream.CopyToAsync(read); + + Assert.Multiple(() => + { + Assert.That(Encoding.UTF8.GetString(read.ToArray()), Is.EqualTo("1"), "The body copy reads every attachment through this no-tracking session, so a read that needed tracking would need a second kind of source session."); + Assert.That(attachment.Details.ContentType, Is.EqualTo("text/xml")); + }); + } + async Task AssertAbsent(string documentId) { using var session = bootstrapStore.OpenAsyncSession(); diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs new file mode 100644 index 0000000000..3c098fffdc --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class HaltThreshold +{ + // Both must be exceeded: the floor protects a tiny category from one bad row, and the + // proportion protects a huge one from grinding through thousands of failures as "only a fraction". + public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) + { + if (skippedCount <= minimumFloor || totalCount == 0) + { + return false; + } + + var percent = skippedCount * 100m / totalCount; + return percent > percentThreshold; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs new file mode 100644 index 0000000000..5d20b9af13 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +public enum MigrationCategoryState +{ + NotStarted, + InProgress, + Complete, + CompleteWithErrors, + Halted, + Abandoned +} + +///