Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,14 @@ not add these specs to a runner that CI invokes.
this property in any queue change. `ClusteredRedisQueue` balances round-robin
across instances.
- Idle queues must consume no resources — do not introduce background intervals.
- **A server is dropped only when it is gone, and never with what it already
delivered.** A cluster manager says when only a server's announcements lapsed
(`ServerRemoval.lapsed`); `ClusteredRedisQueue` then PINGs it and keeps it if
it answers, since a busy process misses announcements as easily as a dead
broker stops sending them. A server that is removed is drained first:
`RedisQueue.drain()` releases the blocked reader with `CLIENT UNBLOCK` from
the writer, so a reply it already took reaches the listeners before teardown
removes them and drops the socket.

## Using this package correctly (for consumer-facing code an agent writes)

Expand Down
15 changes: 14 additions & 1 deletion src/ClusterManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,17 @@ import {
type IServerInput,
} from './IMessageQueue.js';

/**
* Why a {@link ClusterManager} removes a server, where it can say.
*/
export interface ServerRemoval {
/**
* True where the server's announcements stopped arriving and nothing said
* it went away: a missed heartbeat, not a `down` notice.
*/
lapsed?: boolean;
}

/**
* Membership callbacks a clustered queue hands to a {@link ClusterManager} so
* the manager can add and remove servers as it discovers them.
Expand All @@ -43,8 +54,10 @@ export interface ICluster {
add: (server: IServerInput) => IMessageQueueConnection;
/**
* Removes a server from the cluster. A no-op when the server is unknown.
* With `removal.lapsed`, only the server's announcements stopped arriving,
* and a cluster may keep a server that still answers on its connections.
*/
remove: (server: IServerInput) => void;
remove: (server: IServerInput, removal?: ServerRemoval) => void;
/**
* Looks a server up, matching by `id` when both sides carry one and by host
* and port otherwise. Returns `undefined` when it is not registered.
Expand Down
68 changes: 64 additions & 4 deletions src/ClusteredRedisQueue.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,16 @@
* <support@imqueue.com> to get commercial licensing options.
*/
import { EventEmitter } from 'node:events';
import { type InitializedCluster } from './ClusterManager.js';
import { buildOptions, copyEventEmitter, errorCode } from './helpers/index.js';
import {
type InitializedCluster,
type ServerRemoval,
} from './ClusterManager.js';
import {
buildOptions,
copyEventEmitter,
envInt,
errorCode,
} from './helpers/index.js';
import {
DEFAULT_IMQ_OPTIONS,
type EventMap,
Expand Down Expand Up @@ -55,6 +63,13 @@ const SYNC_RETRY_BASE_DELAY = 1000;
/** Longest delay (ms) between subscription catch-up retries. */
const SYNC_RETRY_MAX_DELAY = 30000;

/**
* Time (ms) a server whose announcements lapsed is given to answer a PING
* before it is removed. Configurable via the IMQ_LAPSED_ANSWER_TIMEOUT
* environment variable, defaults to 2,000.
*/
const LAPSED_ANSWER_TIMEOUT = envInt('IMQ_LAPSED_ANSWER_TIMEOUT', 2000);

/**
* A server registered in a {@link ClusteredRedisQueue}: its address, plus the
* {@link RedisQueue} instance serving that host.
Expand Down Expand Up @@ -1155,7 +1170,10 @@ export class ClusteredRedisQueue
* Because the underlying queues are matched by host and port, all entries
* sharing an address are removed together.
*/
protected removeServer(server: IServerInput): void {
protected removeServer(
server: IServerInput,
removal?: ServerRemoval,
): void {
this.verbose(`Removing the server: ${JSON.stringify(server)}`);

const remove = this.findServer(server);
Expand All @@ -1164,6 +1182,45 @@ export class ClusteredRedisQueue
return;
}

// only its announcements lapsed - a busy process misses them as
// easily as a dead broker stops sending them - so the broker is
// asked first, and kept where it still answers
if (removal?.lapsed && remove.imq) {
const kept = remove.imq;

void kept.answers(LAPSED_ANSWER_TIMEOUT).then(answers => {
if (answers) {
this.verbose(
`Kept the server ${kept.redisKey}: its announcements lapsed, but it answers`,
);

return;
}

if (this.findServer(server)?.imq === kept) {
this.dropServer(server);
}
});

return;
}

this.dropServer(server);
}

/**
* Removes a server known to be gone: out of routing at once, its queue
* drained of what it already read and then torn down.
*
* @param server - address of the server to remove
*/
private dropServer(server: IServerInput): void {
const remove = this.findServer(server);

if (!remove) {
return;
}

const imqToRemove = remove.imq;

if (imqToRemove) {
Expand All @@ -1180,8 +1237,11 @@ export class ClusteredRedisQueue
// connection that never answers would then never be torn down at
// all, and teardown is the one operation that has to happen
// regardless of what the host is doing
// drained first: a reply its reader already took from the
// broker reaches its caller rather than going with the socket
imqToRemove
.destroy()
.drain()
.then(() => imqToRemove.destroy())
.catch((err: unknown) =>
this.verbose(`Error destroying removed server: ${err}`),
);
Expand Down
125 changes: 122 additions & 3 deletions src/RedisQueue.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,23 @@ export class RedisQueue
*/
private reader?: IRedisClient;

/**
* The reader's id on the broker, which {@link RedisQueue.drain} unblocks
* it by from another connection.
*/
private readerId?: number;

/**
* The read loop running on the reader, settled once it stops.
*/
private reading?: Promise<void>;

/**
* Set while {@link RedisQueue.drain} runs: the read loop hands on what it
* already read and stops instead of reading again.
*/
private draining: boolean = false;

/**
* Channel connection associated with this queue instance
* Specially designed for client subscriptions to server-emitted events
Expand Down Expand Up @@ -1353,6 +1370,93 @@ export class RedisQueue
return !this.writer || this.writer.status === 'ready';
}

/**
* Whether the broker answers this queue's writer within the given time.
*
* @param timeoutMs - how long to wait for its answer, in milliseconds
* @returns true where a PING came back in time, false otherwise
*
* @remarks
* A connection that only reports itself ready may sit on a socket nobody
* answers; this asks the broker itself. Never rejects.
*/
public async answers(timeoutMs: number): Promise<boolean> {
const writer = this.writer;

if (!writer || writer.status !== 'ready' || this.destroyed) {
return false;
}

let timer: NodeJS.Timeout | undefined;

try {
return await Promise.race([
writer.ping().then(reply => reply === 'PONG'),
new Promise<boolean>(resolve => {
timer = setTimeout(() => resolve(false), timeoutMs);
timer.unref();
}),
]);
} catch {
return false;
} finally {
clearTimeout(timer);
}
}

/**
* Stops reading and hands on every message the reader already took from
* the broker, before a teardown that would drop the reader at once.
*
* @param timeoutMs - how long to wait for the read loop to stop, in
* milliseconds
*
* @remarks
* The reader is released from its blocking read by `CLIENT UNBLOCK` sent
* on the writer. A message the broker already handed it is still in the
* socket, ahead of the release, so it reaches the `message` listeners
* before the loop stops; nothing further is taken. Without this, a reply
* popped but not yet read was lost with the socket, and its caller waited
* out its whole timeout. Never rejects; a host that does not answer is
* given up on after `timeoutMs`.
*/
public async drain(
timeoutMs: number = IMQ_CONNECTION_QUIT_TIMEOUT,
): Promise<void> {
const reading = this.reading;

if (!this.reader || !reading || this.destroyed) {
return;
}

this.draining = true;

let timer: NodeJS.Timeout | undefined;

try {
if (this.readerId !== undefined && this.writer) {
await Promise.race([
this.writer.client('UNBLOCK', this.readerId),
new Promise(resolve => {
timer = setTimeout(resolve, timeoutMs);
timer.unref();
}),
]).catch(() => undefined);
clearTimeout(timer);
}

await Promise.race([
reading,
new Promise(resolve => {
timer = setTimeout(resolve, timeoutMs);
timer.unref();
}),
]);
} finally {
clearTimeout(timer);
}
}

/**
* Writer connection associated with this queue instance. Shared by every
* queue in the process that targets the same {@link RedisQueue.poolKey}.
Expand Down Expand Up @@ -1662,6 +1766,12 @@ export class RedisQueue

switch (channel) {
case 'reader':
// asked before the first blocking read: a blocked reader
// answers nothing else until its read returns
this.readerId = await redis
.client('ID')
.then(id => Number(id))
.catch(() => undefined);
this.read();
break;
case 'writer':
Expand Down Expand Up @@ -2571,7 +2681,7 @@ export class RedisQueue
const key = this.key;

while (true) {
if (!this.reader) {
if (!this.reader || this.draining) {
break;
}

Expand Down Expand Up @@ -2648,7 +2758,12 @@ export class RedisQueue
);

while (true) {
if (!this.reader || !this.writer || this.destroyed) {
if (
!this.reader ||
!this.writer ||
this.destroyed ||
this.draining
) {
break;
}

Expand Down Expand Up @@ -2751,7 +2866,11 @@ export class RedisQueue
? this.readSafe
: this.readUnsafe;

process.nextTick(runReader.bind(this));
this.reading = new Promise<void>(resolve => {
process.nextTick(() => {
runReader.call(this).then(resolve, () => resolve());
});
});

return this;
}
Expand Down
15 changes: 13 additions & 2 deletions src/UDPClusterManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,19 @@
*/
import { join } from 'node:path';
import { Worker } from 'node:worker_threads';
import { ClusterManager, type ICluster } from './ClusterManager.js';
import {
ClusterManager,
type ICluster,
type ServerRemoval,
} from './ClusterManager.js';
import { type ILogger, type IServerInput } from './IMessageQueue.js';

/** Shape of a message posted from the UDP worker thread */
interface WorkerMessage {
type?: string;
server?: unknown;
error?: string;
lapsed?: boolean;
}

/**
Expand Down Expand Up @@ -513,9 +518,15 @@ export class UDPClusterManager extends ClusterManager {
}

const handler = cluster[action] as
| ((server: IServerInput) => unknown)
| ((server: IServerInput, removal?: ServerRemoval) => unknown)
| undefined;

if (action === 'remove') {
handler?.(server, { lapsed: message.lapsed === true });

return;
}

handler?.(server);
});
}
Expand Down
7 changes: 5 additions & 2 deletions src/UDPWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,15 @@ export class UDPWorker {
*
* @param message - the announcement that named it, whether a `down` notice
* or the stale `up` whose liveness timer expired
* @param lapsed - true where only its announcements stopped arriving, and
* no `down` notice said it went away
*/
private removeServer(message: Message): void {
private removeServer(message: Message, lapsed = false): void {
this.servers.delete(UDPWorker.getServerKey(message));
this.messagePort.postMessage({
type: 'cluster:remove',
server: UDPWorker.mapMessage(message),
lapsed,
});
}

Expand Down Expand Up @@ -241,7 +244,7 @@ export class UDPWorker {
() =>
setImmediate(() => {
if (this.servers.get(key) === stamp) {
this.removeServer(message);
this.removeServer(message, true);
}
}),
effectiveTimeout,
Expand Down
Loading
Loading