From 4bcd6eaea10eaacd0e6b8424debd125e13f1a68a Mon Sep 17 00:00:00 2001 From: SerhiyGreench Date: Wed, 18 Nov 2026 23:33:13 +0100 Subject: [PATCH] Keep a server that still answers, and never lose what it already delivered A cluster manager removed a server when its announcements lapsed for the alive timeout, and the clustered queue tore its connections down at once: destroy() removed the message listeners, then dropped the reader. A reply the reader had already taken from the broker but not yet read was lost, and its caller waited out its whole timeout. A busy process misses announcements as easily as a dead broker stops sending them, so a production gateway lost calls this way under load while its broker was healthy. - UDPWorker says whether a removal is a lapse or a down notice, and the UDPClusterManager hands that to the cluster as ServerRemoval.lapsed. - ClusteredRedisQueue PINGs a server whose announcements only lapsed, and keeps it where it answers within IMQ_LAPSED_ANSWER_TIMEOUT (2 s). - A server that is removed is drained first: RedisQueue.drain() releases the blocked reader with CLIENT UNBLOCK from the writer, so a reply already in its socket reaches the listeners before teardown; nothing further is taken. - RedisQueue.answers() says whether the broker answers a PING in time. Unit specs cover both, and test/integration/removedServer.spec.ts proves them against a real broker: a reply in the reader's socket while the event loop is held is delivered across the removal (lost on master), a lapsed broker that answers is kept, and a drained reader takes nothing more. --- AGENTS.md | 8 + src/ClusterManager.ts | 15 +- src/ClusteredRedisQueue.ts | 68 ++++++++- src/RedisQueue.ts | 125 +++++++++++++++- src/UDPClusterManager.ts | 15 +- src/UDPWorker.ts | 7 +- test/integration/removedServer.spec.ts | 197 +++++++++++++++++++++++++ test/mocks/redis.ts | 36 ++++- test/unit/ClusteredRedisQueue.spec.ts | 78 ++++++++++ test/unit/RedisQueue.spec.ts | 69 +++++++++ test/unit/UDPClusterManager.spec.ts | 33 +++++ test/unit/UDPWorker.spec.ts | 6 + 12 files changed, 640 insertions(+), 17 deletions(-) create mode 100644 test/integration/removedServer.spec.ts diff --git a/AGENTS.md b/AGENTS.md index 7ddae4f..5a2c488 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,6 +129,14 @@ not add these specs to a runner that CI invokes. this property in any queue change. `ClusteredRedisQueue` balances round-robin across instances. - Idle queues must consume no resources — do not introduce background intervals. +- **A server is dropped only when it is gone, and never with what it already + delivered.** A cluster manager says when only a server's announcements lapsed + (`ServerRemoval.lapsed`); `ClusteredRedisQueue` then PINGs it and keeps it if + it answers, since a busy process misses announcements as easily as a dead + broker stops sending them. A server that is removed is drained first: + `RedisQueue.drain()` releases the blocked reader with `CLIENT UNBLOCK` from + the writer, so a reply it already took reaches the listeners before teardown + removes them and drops the socket. ## Using this package correctly (for consumer-facing code an agent writes) diff --git a/src/ClusterManager.ts b/src/ClusterManager.ts index 6bf4f20..2d580a6 100644 --- a/src/ClusterManager.ts +++ b/src/ClusterManager.ts @@ -27,6 +27,17 @@ import { type IServerInput, } from './IMessageQueue.js'; +/** + * Why a {@link ClusterManager} removes a server, where it can say. + */ +export interface ServerRemoval { + /** + * True where the server's announcements stopped arriving and nothing said + * it went away: a missed heartbeat, not a `down` notice. + */ + lapsed?: boolean; +} + /** * Membership callbacks a clustered queue hands to a {@link ClusterManager} so * the manager can add and remove servers as it discovers them. @@ -43,8 +54,10 @@ export interface ICluster { add: (server: IServerInput) => IMessageQueueConnection; /** * Removes a server from the cluster. A no-op when the server is unknown. + * With `removal.lapsed`, only the server's announcements stopped arriving, + * and a cluster may keep a server that still answers on its connections. */ - remove: (server: IServerInput) => void; + remove: (server: IServerInput, removal?: ServerRemoval) => void; /** * Looks a server up, matching by `id` when both sides carry one and by host * and port otherwise. Returns `undefined` when it is not registered. diff --git a/src/ClusteredRedisQueue.ts b/src/ClusteredRedisQueue.ts index c7b149b..fecff79 100644 --- a/src/ClusteredRedisQueue.ts +++ b/src/ClusteredRedisQueue.ts @@ -22,8 +22,16 @@ * to get commercial licensing options. */ import { EventEmitter } from 'node:events'; -import { type InitializedCluster } from './ClusterManager.js'; -import { buildOptions, copyEventEmitter, errorCode } from './helpers/index.js'; +import { + type InitializedCluster, + type ServerRemoval, +} from './ClusterManager.js'; +import { + buildOptions, + copyEventEmitter, + envInt, + errorCode, +} from './helpers/index.js'; import { DEFAULT_IMQ_OPTIONS, type EventMap, @@ -55,6 +63,13 @@ const SYNC_RETRY_BASE_DELAY = 1000; /** Longest delay (ms) between subscription catch-up retries. */ const SYNC_RETRY_MAX_DELAY = 30000; +/** + * Time (ms) a server whose announcements lapsed is given to answer a PING + * before it is removed. Configurable via the IMQ_LAPSED_ANSWER_TIMEOUT + * environment variable, defaults to 2,000. + */ +const LAPSED_ANSWER_TIMEOUT = envInt('IMQ_LAPSED_ANSWER_TIMEOUT', 2000); + /** * A server registered in a {@link ClusteredRedisQueue}: its address, plus the * {@link RedisQueue} instance serving that host. @@ -1155,7 +1170,10 @@ export class ClusteredRedisQueue * Because the underlying queues are matched by host and port, all entries * sharing an address are removed together. */ - protected removeServer(server: IServerInput): void { + protected removeServer( + server: IServerInput, + removal?: ServerRemoval, + ): void { this.verbose(`Removing the server: ${JSON.stringify(server)}`); const remove = this.findServer(server); @@ -1164,6 +1182,45 @@ export class ClusteredRedisQueue return; } + // only its announcements lapsed - a busy process misses them as + // easily as a dead broker stops sending them - so the broker is + // asked first, and kept where it still answers + if (removal?.lapsed && remove.imq) { + const kept = remove.imq; + + void kept.answers(LAPSED_ANSWER_TIMEOUT).then(answers => { + if (answers) { + this.verbose( + `Kept the server ${kept.redisKey}: its announcements lapsed, but it answers`, + ); + + return; + } + + if (this.findServer(server)?.imq === kept) { + this.dropServer(server); + } + }); + + return; + } + + this.dropServer(server); + } + + /** + * Removes a server known to be gone: out of routing at once, its queue + * drained of what it already read and then torn down. + * + * @param server - address of the server to remove + */ + private dropServer(server: IServerInput): void { + const remove = this.findServer(server); + + if (!remove) { + return; + } + const imqToRemove = remove.imq; if (imqToRemove) { @@ -1180,8 +1237,11 @@ export class ClusteredRedisQueue // connection that never answers would then never be torn down at // all, and teardown is the one operation that has to happen // regardless of what the host is doing + // drained first: a reply its reader already took from the + // broker reaches its caller rather than going with the socket imqToRemove - .destroy() + .drain() + .then(() => imqToRemove.destroy()) .catch((err: unknown) => this.verbose(`Error destroying removed server: ${err}`), ); diff --git a/src/RedisQueue.ts b/src/RedisQueue.ts index c6c4dd6..d9618ef 100644 --- a/src/RedisQueue.ts +++ b/src/RedisQueue.ts @@ -263,6 +263,23 @@ export class RedisQueue */ private reader?: IRedisClient; + /** + * The reader's id on the broker, which {@link RedisQueue.drain} unblocks + * it by from another connection. + */ + private readerId?: number; + + /** + * The read loop running on the reader, settled once it stops. + */ + private reading?: Promise; + + /** + * Set while {@link RedisQueue.drain} runs: the read loop hands on what it + * already read and stops instead of reading again. + */ + private draining: boolean = false; + /** * Channel connection associated with this queue instance * Specially designed for client subscriptions to server-emitted events @@ -1353,6 +1370,93 @@ export class RedisQueue return !this.writer || this.writer.status === 'ready'; } + /** + * Whether the broker answers this queue's writer within the given time. + * + * @param timeoutMs - how long to wait for its answer, in milliseconds + * @returns true where a PING came back in time, false otherwise + * + * @remarks + * A connection that only reports itself ready may sit on a socket nobody + * answers; this asks the broker itself. Never rejects. + */ + public async answers(timeoutMs: number): Promise { + const writer = this.writer; + + if (!writer || writer.status !== 'ready' || this.destroyed) { + return false; + } + + let timer: NodeJS.Timeout | undefined; + + try { + return await Promise.race([ + writer.ping().then(reply => reply === 'PONG'), + new Promise(resolve => { + timer = setTimeout(() => resolve(false), timeoutMs); + timer.unref(); + }), + ]); + } catch { + return false; + } finally { + clearTimeout(timer); + } + } + + /** + * Stops reading and hands on every message the reader already took from + * the broker, before a teardown that would drop the reader at once. + * + * @param timeoutMs - how long to wait for the read loop to stop, in + * milliseconds + * + * @remarks + * The reader is released from its blocking read by `CLIENT UNBLOCK` sent + * on the writer. A message the broker already handed it is still in the + * socket, ahead of the release, so it reaches the `message` listeners + * before the loop stops; nothing further is taken. Without this, a reply + * popped but not yet read was lost with the socket, and its caller waited + * out its whole timeout. Never rejects; a host that does not answer is + * given up on after `timeoutMs`. + */ + public async drain( + timeoutMs: number = IMQ_CONNECTION_QUIT_TIMEOUT, + ): Promise { + const reading = this.reading; + + if (!this.reader || !reading || this.destroyed) { + return; + } + + this.draining = true; + + let timer: NodeJS.Timeout | undefined; + + try { + if (this.readerId !== undefined && this.writer) { + await Promise.race([ + this.writer.client('UNBLOCK', this.readerId), + new Promise(resolve => { + timer = setTimeout(resolve, timeoutMs); + timer.unref(); + }), + ]).catch(() => undefined); + clearTimeout(timer); + } + + await Promise.race([ + reading, + new Promise(resolve => { + timer = setTimeout(resolve, timeoutMs); + timer.unref(); + }), + ]); + } finally { + clearTimeout(timer); + } + } + /** * Writer connection associated with this queue instance. Shared by every * queue in the process that targets the same {@link RedisQueue.poolKey}. @@ -1662,6 +1766,12 @@ export class RedisQueue switch (channel) { case 'reader': + // asked before the first blocking read: a blocked reader + // answers nothing else until its read returns + this.readerId = await redis + .client('ID') + .then(id => Number(id)) + .catch(() => undefined); this.read(); break; case 'writer': @@ -2571,7 +2681,7 @@ export class RedisQueue const key = this.key; while (true) { - if (!this.reader) { + if (!this.reader || this.draining) { break; } @@ -2648,7 +2758,12 @@ export class RedisQueue ); while (true) { - if (!this.reader || !this.writer || this.destroyed) { + if ( + !this.reader || + !this.writer || + this.destroyed || + this.draining + ) { break; } @@ -2751,7 +2866,11 @@ export class RedisQueue ? this.readSafe : this.readUnsafe; - process.nextTick(runReader.bind(this)); + this.reading = new Promise(resolve => { + process.nextTick(() => { + runReader.call(this).then(resolve, () => resolve()); + }); + }); return this; } diff --git a/src/UDPClusterManager.ts b/src/UDPClusterManager.ts index a48657a..dafbedf 100644 --- a/src/UDPClusterManager.ts +++ b/src/UDPClusterManager.ts @@ -23,7 +23,11 @@ */ import { join } from 'node:path'; import { Worker } from 'node:worker_threads'; -import { ClusterManager, type ICluster } from './ClusterManager.js'; +import { + ClusterManager, + type ICluster, + type ServerRemoval, +} from './ClusterManager.js'; import { type ILogger, type IServerInput } from './IMessageQueue.js'; /** Shape of a message posted from the UDP worker thread */ @@ -31,6 +35,7 @@ interface WorkerMessage { type?: string; server?: unknown; error?: string; + lapsed?: boolean; } /** @@ -513,9 +518,15 @@ export class UDPClusterManager extends ClusterManager { } const handler = cluster[action] as - | ((server: IServerInput) => unknown) + | ((server: IServerInput, removal?: ServerRemoval) => unknown) | undefined; + if (action === 'remove') { + handler?.(server, { lapsed: message.lapsed === true }); + + return; + } + handler?.(server); }); } diff --git a/src/UDPWorker.ts b/src/UDPWorker.ts index f1a77f1..32fc301 100644 --- a/src/UDPWorker.ts +++ b/src/UDPWorker.ts @@ -187,12 +187,15 @@ export class UDPWorker { * * @param message - the announcement that named it, whether a `down` notice * or the stale `up` whose liveness timer expired + * @param lapsed - true where only its announcements stopped arriving, and + * no `down` notice said it went away */ - private removeServer(message: Message): void { + private removeServer(message: Message, lapsed = false): void { this.servers.delete(UDPWorker.getServerKey(message)); this.messagePort.postMessage({ type: 'cluster:remove', server: UDPWorker.mapMessage(message), + lapsed, }); } @@ -241,7 +244,7 @@ export class UDPWorker { () => setImmediate(() => { if (this.servers.get(key) === stamp) { - this.removeServer(message); + this.removeServer(message, true); } }), effectiveTimeout, diff --git a/test/integration/removedServer.spec.ts b/test/integration/removedServer.spec.ts new file mode 100644 index 0000000..e45385e --- /dev/null +++ b/test/integration/removedServer.spec.ts @@ -0,0 +1,197 @@ +/*! + * IMQ integration tests: what a removed server already delivered + * + * I'm Queue Software Project + * Copyright (C) 2025 imqueue.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * If you want to use this code in a closed source (commercial) project, you can + * purchase a proprietary commercial license. Please contact us at + * to get commercial licensing options. + */ +import assert from 'node:assert/strict'; +import { randomUUID as uuid } from 'node:crypto'; +import { describe, it } from 'node:test'; +import { Redis } from 'ioredis'; +import { ClusteredRedisQueue, RedisQueue } from '../../src/index.js'; + +process.setMaxListeners(100); + +const HOST = process.env.REDIS_HOST || '127.0.0.1'; +const PORT = +(process.env.REDIS_PORT || 6379); + +const quiet = { log() {}, info() {}, warn() {}, error() {} }; + +const brokerReason = async (): Promise => { + const probe = new Redis({ + host: HOST, + port: PORT, + lazyConnect: true, + connectTimeout: 1000, + commandTimeout: 1000, + retryStrategy: null, + }); + + let connectionError: unknown; + probe.on('error', error => { + connectionError = error; + }); + + try { + await probe.connect(); + await probe.ping(); + + return undefined; + } catch (err) { + return `redis at ${HOST}:${PORT} is not reachable: ${String(connectionError || err)}`; + } finally { + probe.disconnect(); + } +}; + +const skip = await brokerReason(); + +/** Holds the event loop, as a gateway busy building a schema does. */ +const busy = (ms: number): void => { + const until = Date.now() + ms; + + while (Date.now() < until) { + // spinning on purpose: nothing else may run meanwhile + } +}; + +const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** A clustered queue on the one broker, started and listening. */ +const joined = async (name: string, got: unknown[]) => { + const queue = new ClusteredRedisQueue(name, { + logger: quiet, + prefix: `test-${uuid()}`, + cluster: [{ host: HOST, port: PORT }], + }); + + queue.on('message', (message: unknown) => got.push(message)); + await queue.start(); + // the reader asks its id, then blocks: give it the moment it takes + await wait(200); + + return queue; +}; + +describe('a server removed from a cluster', { skip }, () => { + it('hands on a reply its reader already took before the broker went', async () => { + const got: unknown[] = []; + const name = `Removed${uuid()}`; + const queue = await joined(name, got); + const sender = new RedisQueue(`Sender${uuid()}`, { + logger: quiet, + prefix: (queue as any).options.prefix, + host: HOST, + port: PORT, + }); + + try { + await sender.start(); + // answered once the broker has pushed it - and with a reader + // blocked on the list, handed it straight to that reader + await sender.send(name, { reply: 'already taken' }); + // the reply is in the reader's socket, unread, while the event + // loop is held; the server is removed before it is read + busy(100); + (queue as any).removeServer({ host: HOST, port: PORT }); + + const deadline = Date.now() + 3000; + + while (got.length === 0 && Date.now() < deadline) { + await wait(25); + } + + assert.deepEqual( + got, + [{ reply: 'already taken' }], + 'a reply the reader had taken was lost with the server', + ); + } finally { + await sender.destroy(); + await queue.destroy(); + } + }); + + it('is kept, and goes on delivering, where only its announcements lapsed', async () => { + const got: unknown[] = []; + const name = `Lapsed${uuid()}`; + const queue = await joined(name, got); + const sender = new RedisQueue(`Sender${uuid()}`, { + logger: quiet, + prefix: (queue as any).options.prefix, + host: HOST, + port: PORT, + }); + + try { + await sender.start(); + (queue as any).removeServer( + { host: HOST, port: PORT }, + { lapsed: true }, + ); + await wait(300); + + assert.equal( + (queue as any).servers.length, + 1, + 'a broker that answers was dropped for a missed heartbeat', + ); + + await sender.send(name, { reply: 'after the lapse' }); + + const deadline = Date.now() + 3000; + + while (got.length === 0 && Date.now() < deadline) { + await wait(25); + } + + assert.deepEqual(got, [{ reply: 'after the lapse' }]); + } finally { + await sender.destroy(); + await queue.destroy(); + } + }); + + it('takes nothing more once drained: what arrives after waits in redis', async () => { + const got: unknown[] = []; + const name = `Drained${uuid()}`; + const queue = await joined(name, got); + const raw = new Redis({ host: HOST, port: PORT }); + + try { + const prefix = (queue as any).options.prefix; + const imq = (queue as any).imqs[0]; + + await imq.drain(1000); + await raw.lpush(`${prefix}:${name}`, JSON.stringify({ later: 1 })); + await wait(200); + + assert.deepEqual(got, [], 'a drained reader took a message'); + assert.equal( + await raw.llen(`${prefix}:${name}`), + 1, + 'the message left redis without reaching anybody', + ); + } finally { + raw.disconnect(); + await queue.destroy(); + } + }); +}); diff --git a/test/mocks/redis.ts b/test/mocks/redis.ts index 5784bb0..c26b1b3 100644 --- a/test/mocks/redis.ts +++ b/test/mocks/redis.ts @@ -39,6 +39,10 @@ export class RedisClientMock extends EventEmitter { private static __queues__: any = {}; private static __clientList: any = {}; private __rt: any; + private __unblock?: () => void; + private static __ids = 0; + private static __byId: Map = new Map(); + public __id: number = ++RedisClientMock.__ids; private static __keys: any = {}; private static __scripts: any = {}; private static readonly __notifyEventsParam = 'notify-keyspace-events'; @@ -57,6 +61,7 @@ export class RedisClientMock extends EventEmitter { constructor(options: any = {}) { super(); this.options = options; + RedisClientMock.__byId.set(this.__id, this); setTimeout(() => { this.emit('ready', this); }); @@ -69,6 +74,10 @@ export class RedisClientMock extends EventEmitter { public end() {} + public ping(): Promise { + return Promise.resolve('PONG'); + } + public quit() { return new Promise(resolve => resolve(undefined)); } @@ -119,10 +128,15 @@ export class RedisClientMock extends EventEmitter { } return new Promise(resolve => { - this.__rt = setTimeout( - () => resolve(this.brpop(key, timeout, cb)), - timeout || 100, - ); + this.__unblock = () => { + clearTimeout(this.__rt); + this.__unblock = undefined; + resolve(null as any); + }; + this.__rt = setTimeout(() => { + this.__unblock = undefined; + resolve(this.brpop(key, timeout, cb)); + }, timeout || 100); }); } else { const result = [key, q.shift()]; @@ -282,7 +296,10 @@ export class RedisClientMock extends EventEmitter { public client(...args: any[]): string | boolean { const self = RedisClientMock; const cmd = args.shift(); - const cb = args.pop(); + const cb = + typeof args[args.length - 1] === 'function' + ? args.pop() + : undefined; const name = args.shift(); if (cmd === 'LIST') { @@ -295,6 +312,15 @@ export class RedisClientMock extends EventEmitter { } else if (cmd === 'SETNAME') { this.__name = name; self.__clientList[name] = true; + } else if (cmd === 'ID') { + return Promise.resolve(this.__id) as any; + } else if (cmd === 'UNBLOCK') { + const blocked = self.__byId.get(Number(name)); + const pending = blocked?.__unblock; + + pending?.(); + + return Promise.resolve(pending ? 1 : 0) as any; } this.cbExecute(cb, null, true); diff --git a/test/unit/ClusteredRedisQueue.spec.ts b/test/unit/ClusteredRedisQueue.spec.ts index 73c43c7..8a50606 100644 --- a/test/unit/ClusteredRedisQueue.spec.ts +++ b/test/unit/ClusteredRedisQueue.spec.ts @@ -1958,3 +1958,81 @@ describe('ClusteredRedisQueue joining-server failure logging', () => { await cq.destroy(); }); }); + +describe('ClusteredRedisQueue - a server whose announcements lapsed', () => { + afterEach(() => { + mock.restoreAll(); + }); + + const settled = () => new Promise(resolve => setTimeout(resolve, 20)); + + it('is kept where it still answers on its connection', async () => { + const cq: any = new ClusteredRedisQueue('TestLapsedKept', { + clusterManagers: [new (ClusterManager as any)()], + }); + + cq.addServer(clusterConfig.cluster[0]); + + const imq = cq.imqs[0]; + const destroy = mock.method(imq, 'destroy'); + + mock.method(imq, 'answers', async () => true); + cq.removeServer(clusterConfig.cluster[0], { lapsed: true }); + await settled(); + + assert.equal(cq.servers.length, 1, 'a broker that answers was dropped'); + assert.equal(cq.imqs.length, 1); + assert.equal(destroy.mock.callCount(), 0); + }); + + it('is removed where it does not answer', async () => { + const cq: any = new ClusteredRedisQueue('TestLapsedGone', { + clusterManagers: [new (ClusterManager as any)()], + }); + + cq.addServer(clusterConfig.cluster[0]); + mock.method(cq.imqs[0], 'answers', async () => false); + cq.removeServer(clusterConfig.cluster[0], { lapsed: true }); + await settled(); + + assert.equal(cq.servers.length, 0); + assert.equal(cq.imqs.length, 0); + }); + + it('is removed at once where it said it went down', () => { + const cq: any = new ClusteredRedisQueue('TestDown', { + clusterManagers: [new (ClusterManager as any)()], + }); + + cq.addServer(clusterConfig.cluster[0]); + + const answers = mock.method(cq.imqs[0], 'answers', async () => true); + + cq.removeServer(clusterConfig.cluster[0]); + + assert.equal(cq.servers.length, 0); + assert.equal(answers.mock.callCount(), 0); + }); + + it('drains what its reader already read before it is torn down', async () => { + const cq: any = new ClusteredRedisQueue('TestDrainFirst', { + clusterManagers: [new (ClusterManager as any)()], + }); + + cq.addServer(clusterConfig.cluster[0]); + + const imq = cq.imqs[0]; + const order: string[] = []; + + mock.method(imq, 'drain', async () => { + order.push('drain'); + }); + mock.method(imq, 'destroy', async () => { + order.push('destroy'); + }); + cq.removeServer(clusterConfig.cluster[0]); + await settled(); + + assert.deepEqual(order, ['drain', 'destroy']); + }); +}); diff --git a/test/unit/RedisQueue.spec.ts b/test/unit/RedisQueue.spec.ts index e5cbf21..4d39375 100644 --- a/test/unit/RedisQueue.spec.ts +++ b/test/unit/RedisQueue.spec.ts @@ -3146,3 +3146,72 @@ describe('RedisQueue publish visibility', () => { assert.equal(matching(cap.warn, /no subscribers/).length, 0); }); }); + +describe('RedisQueue.drain()', () => { + it('hands on a message its reader already took, and reads no more', async () => { + const rq: any = new RedisQueue(`DrainQueue${Date.now()}`, { logger }); + const got: unknown[] = []; + + rq.on('message', (message: unknown) => got.push(message)); + await rq.start(); + await new Promise(resolve => setTimeout(resolve, 20)); + + assert.ok(rq.readerId !== undefined, 'the reader has no id to unblock'); + + await rq.send(rq.name, { reply: 'already on its way' }); + await new Promise(resolve => setTimeout(resolve, 150)); + await rq.drain(1000); + + assert.deepEqual(got, [{ reply: 'already on its way' }]); + assert.equal(rq.draining, true); + + await rq.send(rq.name, { reply: 'after the drain' }); + await new Promise(resolve => setTimeout(resolve, 250)); + + assert.deepEqual( + got, + [{ reply: 'already on its way' }], + 'the reader went on reading after it was drained', + ); + + await rq.destroy(); + }); + + it('releases a reader blocked on an empty queue at once', async () => { + const rq: any = new RedisQueue(`DrainEmpty${Date.now()}`, { logger }); + + await rq.start(); + await new Promise(resolve => setTimeout(resolve, 20)); + + const started = Date.now(); + + await rq.drain(5000); + + assert.ok( + Date.now() - started < 1000, + 'the drain waited out its time instead of releasing the reader', + ); + + await rq.destroy(); + }); +}); + +describe('RedisQueue.answers()', () => { + it('is true where the broker answers a PING', async () => { + const rq: any = new RedisQueue(`Answers${Date.now()}`, { logger }); + + await rq.start(); + assert.equal(await rq.answers(500), true); + await rq.destroy(); + }); + + it('is false where the PING goes unanswered', async () => { + const rq: any = new RedisQueue(`Silent${Date.now()}`, { logger }); + + await rq.start(); + mock.method(rq.writer, 'ping', () => new Promise(() => undefined)); + assert.equal(await rq.answers(50), false); + mock.restoreAll(); + await rq.destroy(); + }); +}); diff --git a/test/unit/UDPClusterManager.spec.ts b/test/unit/UDPClusterManager.spec.ts index 94abfe6..f694da0 100644 --- a/test/unit/UDPClusterManager.spec.ts +++ b/test/unit/UDPClusterManager.spec.ts @@ -521,3 +521,36 @@ describe('UDPClusterManager.respawn() guard', () => { ); }); }); + +describe('UDPClusterManager - why a server is removed', () => { + afterEach(() => { + mock.restoreAll(); + }); + + for (const [lapsed, said] of [ + [true, 'its announcements lapsed'], + [false, 'it went down'], + ] as const) { + it(`tells the cluster ${said}`, async () => { + const removals: unknown[] = []; + const cluster: any = { + add: () => {}, + remove: (_server: unknown, removal: unknown) => + removals.push(removal), + find: () => undefined, + }; + const manager: any = new UDPClusterManager(); + + manager.init(cluster); + getSocket(manager).emit('message', { + type: 'cluster:remove', + server: testMessageDown, + lapsed, + }); + await new Promise(resolve => setTimeout(resolve, 10)); + + assert.deepEqual(removals, [{ lapsed }]); + await manager.destroy(); + }); + } +}); diff --git a/test/unit/UDPWorker.spec.ts b/test/unit/UDPWorker.spec.ts index 69520fa..470c2bd 100644 --- a/test/unit/UDPWorker.spec.ts +++ b/test/unit/UDPWorker.spec.ts @@ -96,6 +96,7 @@ describe('UDPWorker', () => { assert.equal(message.type, 'cluster:remove'); assert.equal(message.server.id, 'id-1'); + assert.equal(message.lapsed, false, 'a down notice read as a lapse'); }); it('should drop malformed datagrams without crashing', () => { @@ -232,6 +233,11 @@ describe('UDPWorker', () => { const types = postedMessages(port).map(message => message.type); assert.deepEqual(types, ['cluster:add', 'cluster:remove']); + assert.equal( + postedMessages(port)[1].lapsed, + true, + 'a lapse read as a down notice', + ); }); it('should keep a server alive while heartbeats keep arriving', async () => {