From e4041d84de132b8c08e4a9008efcfa0fb92b1ade Mon Sep 17 00:00:00 2001 From: Romain Lanz Date: Sat, 24 Oct 2026 09:27:52 +0000 Subject: [PATCH] fix: bound redis health checks with a timeout RedisCheck could not back a readiness probe. Its ping had no time bound, so a server that accepted the connection but did not answer held the check until the socket failed. While connecting, it waited up to 3 x 1s, which nobody could configure. The #pingAttempts counter was never reset, so a long-lived check stopped waiting after three waits in total. On a closed connection, the check reconnected it as a side effect. RedisMemoryUsageCheck shared the same code. Both checks now go through probeConnection and accept a timeout (default 1s), set with `.timeout(2_000)` or `.timeout('2s')`. The timeout covers the wait for a connecting connection, the ping and the action. A connection that is neither ready nor connecting fails at once with its last error, and the checks never connect or reconnect it. RedisMemoryUsageCheck still pings before it computes the memory usage, custom compute included. On a silent ready connection, each timed out run leaves one PING in the ioredis queue until the socket settles: the timeout bounds the result, not the work already sent. --- src/checks/probe_connection.ts | 81 ++++++++++++++ src/checks/redis_check.ts | 75 +++++-------- src/checks/redis_memory_usage_check.ts | 146 ++++++++++--------------- tests/redis_check.spec.ts | 97 +++++++++++++++- tests/redis_memory_usage_check.spec.ts | 26 ++++- tests_helpers/main.ts | 34 ++++++ 6 files changed, 320 insertions(+), 139 deletions(-) create mode 100644 src/checks/probe_connection.ts diff --git a/src/checks/probe_connection.ts b/src/checks/probe_connection.ts new file mode 100644 index 0000000..87f3fb3 --- /dev/null +++ b/src/checks/probe_connection.ts @@ -0,0 +1,81 @@ +/* + * @adonisjs/redis + * + * (c) AdonisJS + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +import { Result } from '@adonisjs/core/health' +import type { Connection } from '../types.ts' + +/** + * Pings a connection, then runs `action`, for the health checks. It never + * connects or reconnects the connection itself, and each run fails after + * its own `timeout` (in milliseconds), which covers the wait for a + * connecting connection, the ping and the action. + * + * ``` + * ready, connect ──▶ ping ──▶ action + * connecting ──▶ wait for "ready" ──▶ ping ──▶ action + * └── "close" / "end" ──▶ failed (lastError) + * anything else ──▶ failed at once (lastError) + * timeout elapsed ──▶ failed + * ``` + * + * @example + * ```ts + * const result = await probeConnection(connection, 1_000, async () => Result.ok('Redis answered')) + * ``` + */ +export async function probeConnection( + connection: Connection, + timeout: number, + action: () => Promise +): Promise { + if (!connection.isReady() && connection.status !== 'connecting') { + return Result.failed('Unable to connect to the redis server', connection.lastError) + } + + const ioConnection = connection.ioConnection + const statusEvents = ['ready', 'close', 'end'] + let onStatusChange: () => void = () => {} + let timer: NodeJS.Timeout | undefined + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => { + resolve(Result.failed(`Redis did not respond within ${timeout}ms`, connection.lastError)) + }, timeout) + }) + + const probe = async () => { + if (connection.status === 'connecting') { + await new Promise((resolve) => { + onStatusChange = resolve + statusEvents.forEach((event) => ioConnection.once(event, onStatusChange)) + }) + + if (!connection.isReady()) { + return Result.failed('Unable to connect to the redis server', connection.lastError) + } + } + + await connection.ping() + return action() + } + + /** + * On timeout, the ping stays in the ioredis queue until the server + * answers or the socket closes, so each timed out run on a silent + * connection leaves one PING behind. The application's own commands + * pile up on that connection the same way. + */ + try { + return await Promise.race([probe(), timedOut]) + } catch (error) { + return Result.failed(error) + } finally { + clearTimeout(timer) + statusEvents.forEach((event) => ioConnection.off(event, onStatusChange)) + } +} diff --git a/src/checks/redis_check.ts b/src/checks/redis_check.ts index 4592b0b..1f70c68 100644 --- a/src/checks/redis_check.ts +++ b/src/checks/redis_check.ts @@ -7,18 +7,24 @@ * file that was distributed with this source code. */ -import { setTimeout } from 'node:timers/promises' +import stringHelpers from '@adonisjs/core/helpers/string' import { BaseCheck, Result } from '@adonisjs/core/health' import type { HealthCheckResult } from '@adonisjs/core/types/health' +import { probeConnection } from './probe_connection.ts' import type { Connection } from '../types.ts' /** * The RedisCheck pings the redis server to ensure we are * able to connect to it. * + * The check fails at once when the connection is neither ready nor + * connecting (for example while reconnecting), and fails when waiting + * for the connection plus the ping take longer than the timeout. It + * never connects or reconnects the connection itself. + * * @example * ```ts - * const check = new RedisCheck(redis.connection()) + * const check = new RedisCheck(redis.connection()).timeout('2s') * const result = await check.run() * * if (result.isHealthy) { @@ -33,9 +39,9 @@ export class RedisCheck extends BaseCheck { #connection: Connection /** - * Number of times `ping` was deferred, at max we defer it for 3 times + * Time in milliseconds to wait for the connection and the ping */ - #pingAttempts = 0 + #timeout = 1_000 /** * Health check public name @@ -72,40 +78,20 @@ export class RedisCheck extends BaseCheck { } /** - * Internal method to ping the redis server + * Define how long the check waits for a connecting connection and + * the ping, together. Defaults to 1 second. + * + * @param duration - Milliseconds or a duration expression (e.g. '2s') + * + * @example + * ```ts + * check.timeout('2s') + * check.timeout(500) + * ``` */ - async #ping(): Promise { - /** - * When in connecting status, we should wait for maximum 3 seconds with - * (divided into 3 attempts). However, if there was an error, we will - * not wait for 3 seconds. - */ - if (this.#connection.isConnecting() && this.#pingAttempts < 3 && !this.#connection.lastError) { - await setTimeout(1000) - this.#pingAttempts++ - return this.#ping() - } - - /** - * Re-connect when connection is in closed state - */ - if (this.#connection.isClosed()) { - await this.#connection.ioConnection.connect() - return this.#ping() - } - - /** - * If we are not in `connect` or `ready` state, then we should - * report an error. - */ - if (!this.#connection.isReady()) { - return Result.failed( - 'Unable to connect to the redis server', - this.#connection.lastError - ).mergeMetaData(this.#getConnectionMetadata()) - } - - await this.#connection.ping() + timeout(duration: string | number) { + this.#timeout = stringHelpers.milliseconds.parse(duration) + return this } /** @@ -118,17 +104,10 @@ export class RedisCheck extends BaseCheck { * ``` */ async run(): Promise { - try { - const result = await this.#ping() - if (result) { - return result - } + const result = await probeConnection(this.#connection, this.#timeout, async () => + Result.ok('Successfully connected to the redis server') + ) - return Result.ok('Successfully connected to the redis server').mergeMetaData( - this.#getConnectionMetadata() - ) - } catch (error) { - return Result.failed(error).mergeMetaData(this.#getConnectionMetadata()) - } + return result.mergeMetaData(this.#getConnectionMetadata()) } } diff --git a/src/checks/redis_memory_usage_check.ts b/src/checks/redis_memory_usage_check.ts index 6d64570..e8020e8 100644 --- a/src/checks/redis_memory_usage_check.ts +++ b/src/checks/redis_memory_usage_check.ts @@ -7,11 +7,11 @@ * file that was distributed with this source code. */ -import { setTimeout } from 'node:timers/promises' import stringHelpers from '@adonisjs/core/helpers/string' import { BaseCheck, Result } from '@adonisjs/core/health' import type { HealthCheckResult } from '@adonisjs/core/types/health' import * as errors from '../errors.ts' +import { probeConnection } from './probe_connection.ts' import type { Connection } from '../types.ts' @@ -37,9 +37,10 @@ export class RedisMemoryUsageCheck extends BaseCheck { #connection: Connection /** - * Number of times `ping` was deferred, at max we defer it for 3 times + * Time in milliseconds to wait for the connection, the ping and the + * memory info */ - #pingAttempts = 0 + #timeout = 1_000 /** * Method to compute the memory consumption @@ -119,43 +120,6 @@ export class RedisMemoryUsageCheck extends BaseCheck { } } - /** - * Internal method to ping the redis server - */ - async #ping(): Promise { - /** - * When in connecting status, we should wait for maximum 3 seconds with - * (divided into 3 attempts). However, if there was an error, we will - * not wait for 3 seconds. - */ - if (this.#connection.isConnecting() && this.#pingAttempts < 3 && !this.#connection.lastError) { - await setTimeout(1000) - this.#pingAttempts++ - return this.#ping() - } - - /** - * Re-connect when connection is in closed state - */ - if (this.#connection.isClosed()) { - await this.#connection.ioConnection.connect() - return this.#ping() - } - - /** - * If we are not in `connect` or `ready` state, then we should - * report an error. - */ - if (!this.#connection.isReady()) { - return Result.failed( - 'Unable to connect to the redis server', - this.#connection.lastError - ).mergeMetaData(this.#getConnectionMetadata()) - } - - await this.#connection.ping() - } - /** * Define the memory threshold after which a warning * should be created. @@ -222,6 +186,22 @@ export class RedisMemoryUsageCheck extends BaseCheck { return this } + /** + * Define how long the check waits for a connecting connection, the + * ping and the memory info, together. Defaults to 1 second. + * + * @param duration - Milliseconds or a duration expression (e.g. '2s') + * + * @example + * ```ts + * check.timeout('2s') + * ``` + */ + timeout(duration: string | number) { + this.#timeout = stringHelpers.milliseconds.parse(duration) + return this + } + /** * Executes the health check * @@ -234,60 +214,48 @@ export class RedisMemoryUsageCheck extends BaseCheck { * ``` */ async run(): Promise { - try { - const result = await this.#ping() - if (result) { - return result - } - - /** - * Get memory usage when tracking memory - */ - const memoryUsage = await this.#computeFn(this.#connection) + const result = await probeConnection(this.#connection, this.#timeout, () => this.#checkMemory()) + return result.mergeMetaData(this.#getConnectionMetadata()) + } - /** - * Return early when we do not have access to the memory - * usage. - */ - if (!memoryUsage) { - return Result.failed('Check failed. Unable to get redis memory info').mergeMetaData( - this.#getConnectionMetadata() - ) - } + /** + * Computes the memory usage and compares it with the thresholds + */ + async #checkMemory(): Promise { + const memoryUsage = await this.#computeFn(this.#connection) - const memoryUsagePretty = stringHelpers.bytes.format(memoryUsage) - const warnThresholdPretty = stringHelpers.bytes.format(this.#warnThreshold) - const failureThresholdPretty = stringHelpers.bytes.format(this.#failThreshold) + /** + * Return early when we do not have access to the memory + * usage. + */ + if (!memoryUsage) { + return Result.failed('Check failed. Unable to get redis memory info') + } - /** - * Check if we have crossed the failure threshold - */ - if (this.#failThreshold && memoryUsage > this.#failThreshold) { - return Result.failed( - `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${failureThresholdPretty}` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } + const memoryUsagePretty = stringHelpers.bytes.format(memoryUsage) + const warnThresholdPretty = stringHelpers.bytes.format(this.#warnThreshold) + const failureThresholdPretty = stringHelpers.bytes.format(this.#failThreshold) - /** - * Check if we have crossed the warning threshold - */ - if (this.#warnThreshold && memoryUsage > this.#warnThreshold) { - return Result.warning( - `Redis memory usage is ${stringHelpers.bytes.format(memoryUsage)}, which is above the threshold of ${warnThresholdPretty}` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } + /** + * Check if we have crossed the failure threshold + */ + if (this.#failThreshold && memoryUsage > this.#failThreshold) { + return Result.failed( + `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${failureThresholdPretty}` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) + } - return Result.ok( - `Redis memory usage is ${memoryUsagePretty}, which is under the defined thresholds` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } catch (error) { - return Result.failed(error).mergeMetaData(this.#getConnectionMetadata()) + /** + * Check if we have crossed the warning threshold + */ + if (this.#warnThreshold && memoryUsage > this.#warnThreshold) { + return Result.warning( + `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${warnThresholdPretty}` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) } + + return Result.ok( + `Redis memory usage is ${memoryUsagePretty}, which is under the defined thresholds` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) } } diff --git a/tests/redis_check.spec.ts b/tests/redis_check.spec.ts index c3d4c76..7940e3e 100644 --- a/tests/redis_check.spec.ts +++ b/tests/redis_check.spec.ts @@ -7,9 +7,10 @@ * file that was distributed with this source code. */ +import { once } from 'node:events' import { test } from '@japa/runner' -import { pEvent } from '../tests_helpers/main.ts' +import { createSilentServer, pEvent } from '../tests_helpers/main.ts' import { RedisCheck } from '../src/checks/redis_check.ts' import RedisConnection from '../src/connections/redis_connection.ts' import RedisClusterConnection from '../src/connections/redis_cluster_connection.ts' @@ -62,6 +63,100 @@ test.group('Health check | redis connection', () => { }, }) }) + test('fail within the timeout when the server stops answering', async ({ assert, cleanup }) => { + const server = await createSilentServer() + cleanup(() => server.close()) + + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: server.port, + enableReadyCheck: false, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'ready') + server.silence() + + const healthCheck = new RedisCheck(connection).timeout(200) + for (let run = 0; run < 2; run++) { + const startedAt = Date.now() + const result = await healthCheck.run() + + assert.isBelow(Date.now() - startedAt, 400) + assert.containSubset(result, { + message: 'Redis did not respond within 200ms', + status: 'error', + meta: { connection: { name: 'main', status: 'ready' } }, + }) + } + }) + + test('fail at once with the last error while reconnecting', async ({ assert, cleanup }) => { + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: 1, + retryStrategy: () => 5_000, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'reconnecting') + + const healthCheck = new RedisCheck(connection).timeout('2s') + for (let run = 0; run < 2; run++) { + const startedAt = Date.now() + const result = await healthCheck.run() + + assert.isBelow(Date.now() - startedAt, 50) + assert.containSubset(result, { + message: 'Unable to connect to the redis server', + status: 'error', + meta: { connection: { status: 'reconnecting' }, error: { code: 'ECONNREFUSED' } }, + }) + } + }) + + test('wait for a connecting connection on every run of the same check', async ({ + assert, + cleanup, + }) => { + const connection = new RedisConnection('main', { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + }) + cleanup(() => connection.quit()) + await pEvent(connection, 'ready') + + const healthCheck = new RedisCheck(connection) + for (let run = 0; run < 4; run++) { + connection.ioConnection.disconnect() + await once(connection.ioConnection, 'end') + connection.ioConnection.connect() + assert.equal(connection.status, 'connecting') + + const result = await healthCheck.run() + assert.containSubset(result, { + message: 'Successfully connected to the redis server', + status: 'ok', + meta: { connection: { status: 'ready' } }, + }) + } + }) + + test('do not reconnect a closed connection', async ({ assert }) => { + const connection = new RedisConnection('main', { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + }) + await pEvent(connection, 'ready') + await connection.quit() + await pEvent(connection, 'end') + + const result = await new RedisCheck(connection).run() + assert.containSubset(result, { + message: 'Unable to connect to the redis server', + status: 'error', + meta: { connection: { status: 'end' } }, + }) + assert.equal(connection.status, 'end') + }) }) test.group('Health check | cluster connection', () => { diff --git a/tests/redis_memory_usage_check.spec.ts b/tests/redis_memory_usage_check.spec.ts index eceb08d..1444f2d 100644 --- a/tests/redis_memory_usage_check.spec.ts +++ b/tests/redis_memory_usage_check.spec.ts @@ -10,7 +10,7 @@ import { test } from '@japa/runner' import stringHelpers from '@adonisjs/core/helpers/string' -import { pEvent } from '../tests_helpers/main.ts' +import { createSilentServer, pEvent } from '../tests_helpers/main.ts' import RedisConnection from '../src/connections/redis_connection.ts' import { RedisMemoryUsageCheck } from '../src/checks/redis_memory_usage_check.ts' import RedisClusterConnection from '../src/connections/redis_cluster_connection.ts' @@ -164,6 +164,30 @@ test.group('Redis memory usage | redis connection', () => { }, }) }) + + test('fail when the server does not answer the ping, with a custom compute', async ({ + assert, + cleanup, + }) => { + const server = await createSilentServer() + cleanup(() => server.close()) + + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: server.port, + enableReadyCheck: false, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'ready') + server.silence() + + const healthCheck = new RedisMemoryUsageCheck(connection).timeout(200).compute(async () => 1024) + + assert.containSubset(await healthCheck.run(), { + message: 'Redis did not respond within 200ms', + status: 'error', + }) + }) }) test.group('Redis memory usage | cluster connection', () => { diff --git a/tests_helpers/main.ts b/tests_helpers/main.ts index 7345835..b498c81 100644 --- a/tests_helpers/main.ts +++ b/tests_helpers/main.ts @@ -7,7 +7,9 @@ * file that was distributed with this source code. */ +import { once } from 'node:events' import type Emittery from 'emittery' +import { createServer, type AddressInfo, type Socket } from 'node:net' /** * Promisify an event @@ -30,3 +32,35 @@ export function pEvent( emitter.on(event, handler) }) } + +/** + * Starts a fake Redis server that answers "+OK" to every command until + * `silence` is called, then keeps connections open without answering, + * like a half-open TCP connection. + */ +export async function createSilentServer() { + let silent = false + const sockets = new Set() + + const server = createServer((socket) => { + sockets.add(socket) + socket.on('data', (chunk) => { + if (!silent) { + const commands = chunk.toString().match(/^\*/gm)?.length ?? 0 + socket.write('+OK\r\n'.repeat(commands)) + } + }) + }).listen(0, '127.0.0.1') + await once(server, 'listening') + + return { + port: (server.address() as AddressInfo).port, + silence() { + silent = true + }, + close() { + sockets.forEach((socket) => socket.destroy()) + server.close() + }, + } +}