diff --git a/src/checks/probe_connection.ts b/src/checks/probe_connection.ts new file mode 100644 index 0000000..87f3fb3 --- /dev/null +++ b/src/checks/probe_connection.ts @@ -0,0 +1,81 @@ +/* + * @adonisjs/redis + * + * (c) AdonisJS + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +import { Result } from '@adonisjs/core/health' +import type { Connection } from '../types.ts' + +/** + * Pings a connection, then runs `action`, for the health checks. It never + * connects or reconnects the connection itself, and each run fails after + * its own `timeout` (in milliseconds), which covers the wait for a + * connecting connection, the ping and the action. + * + * ``` + * ready, connect ──▶ ping ──▶ action + * connecting ──▶ wait for "ready" ──▶ ping ──▶ action + * └── "close" / "end" ──▶ failed (lastError) + * anything else ──▶ failed at once (lastError) + * timeout elapsed ──▶ failed + * ``` + * + * @example + * ```ts + * const result = await probeConnection(connection, 1_000, async () => Result.ok('Redis answered')) + * ``` + */ +export async function probeConnection( + connection: Connection, + timeout: number, + action: () => Promise +): Promise { + if (!connection.isReady() && connection.status !== 'connecting') { + return Result.failed('Unable to connect to the redis server', connection.lastError) + } + + const ioConnection = connection.ioConnection + const statusEvents = ['ready', 'close', 'end'] + let onStatusChange: () => void = () => {} + let timer: NodeJS.Timeout | undefined + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => { + resolve(Result.failed(`Redis did not respond within ${timeout}ms`, connection.lastError)) + }, timeout) + }) + + const probe = async () => { + if (connection.status === 'connecting') { + await new Promise((resolve) => { + onStatusChange = resolve + statusEvents.forEach((event) => ioConnection.once(event, onStatusChange)) + }) + + if (!connection.isReady()) { + return Result.failed('Unable to connect to the redis server', connection.lastError) + } + } + + await connection.ping() + return action() + } + + /** + * On timeout, the ping stays in the ioredis queue until the server + * answers or the socket closes, so each timed out run on a silent + * connection leaves one PING behind. The application's own commands + * pile up on that connection the same way. + */ + try { + return await Promise.race([probe(), timedOut]) + } catch (error) { + return Result.failed(error) + } finally { + clearTimeout(timer) + statusEvents.forEach((event) => ioConnection.off(event, onStatusChange)) + } +} diff --git a/src/checks/redis_check.ts b/src/checks/redis_check.ts index 4592b0b..1f70c68 100644 --- a/src/checks/redis_check.ts +++ b/src/checks/redis_check.ts @@ -7,18 +7,24 @@ * file that was distributed with this source code. */ -import { setTimeout } from 'node:timers/promises' +import stringHelpers from '@adonisjs/core/helpers/string' import { BaseCheck, Result } from '@adonisjs/core/health' import type { HealthCheckResult } from '@adonisjs/core/types/health' +import { probeConnection } from './probe_connection.ts' import type { Connection } from '../types.ts' /** * The RedisCheck pings the redis server to ensure we are * able to connect to it. * + * The check fails at once when the connection is neither ready nor + * connecting (for example while reconnecting), and fails when waiting + * for the connection plus the ping take longer than the timeout. It + * never connects or reconnects the connection itself. + * * @example * ```ts - * const check = new RedisCheck(redis.connection()) + * const check = new RedisCheck(redis.connection()).timeout('2s') * const result = await check.run() * * if (result.isHealthy) { @@ -33,9 +39,9 @@ export class RedisCheck extends BaseCheck { #connection: Connection /** - * Number of times `ping` was deferred, at max we defer it for 3 times + * Time in milliseconds to wait for the connection and the ping */ - #pingAttempts = 0 + #timeout = 1_000 /** * Health check public name @@ -72,40 +78,20 @@ export class RedisCheck extends BaseCheck { } /** - * Internal method to ping the redis server + * Define how long the check waits for a connecting connection and + * the ping, together. Defaults to 1 second. + * + * @param duration - Milliseconds or a duration expression (e.g. '2s') + * + * @example + * ```ts + * check.timeout('2s') + * check.timeout(500) + * ``` */ - async #ping(): Promise { - /** - * When in connecting status, we should wait for maximum 3 seconds with - * (divided into 3 attempts). However, if there was an error, we will - * not wait for 3 seconds. - */ - if (this.#connection.isConnecting() && this.#pingAttempts < 3 && !this.#connection.lastError) { - await setTimeout(1000) - this.#pingAttempts++ - return this.#ping() - } - - /** - * Re-connect when connection is in closed state - */ - if (this.#connection.isClosed()) { - await this.#connection.ioConnection.connect() - return this.#ping() - } - - /** - * If we are not in `connect` or `ready` state, then we should - * report an error. - */ - if (!this.#connection.isReady()) { - return Result.failed( - 'Unable to connect to the redis server', - this.#connection.lastError - ).mergeMetaData(this.#getConnectionMetadata()) - } - - await this.#connection.ping() + timeout(duration: string | number) { + this.#timeout = stringHelpers.milliseconds.parse(duration) + return this } /** @@ -118,17 +104,10 @@ export class RedisCheck extends BaseCheck { * ``` */ async run(): Promise { - try { - const result = await this.#ping() - if (result) { - return result - } + const result = await probeConnection(this.#connection, this.#timeout, async () => + Result.ok('Successfully connected to the redis server') + ) - return Result.ok('Successfully connected to the redis server').mergeMetaData( - this.#getConnectionMetadata() - ) - } catch (error) { - return Result.failed(error).mergeMetaData(this.#getConnectionMetadata()) - } + return result.mergeMetaData(this.#getConnectionMetadata()) } } diff --git a/src/checks/redis_memory_usage_check.ts b/src/checks/redis_memory_usage_check.ts index 6d64570..e8020e8 100644 --- a/src/checks/redis_memory_usage_check.ts +++ b/src/checks/redis_memory_usage_check.ts @@ -7,11 +7,11 @@ * file that was distributed with this source code. */ -import { setTimeout } from 'node:timers/promises' import stringHelpers from '@adonisjs/core/helpers/string' import { BaseCheck, Result } from '@adonisjs/core/health' import type { HealthCheckResult } from '@adonisjs/core/types/health' import * as errors from '../errors.ts' +import { probeConnection } from './probe_connection.ts' import type { Connection } from '../types.ts' @@ -37,9 +37,10 @@ export class RedisMemoryUsageCheck extends BaseCheck { #connection: Connection /** - * Number of times `ping` was deferred, at max we defer it for 3 times + * Time in milliseconds to wait for the connection, the ping and the + * memory info */ - #pingAttempts = 0 + #timeout = 1_000 /** * Method to compute the memory consumption @@ -119,43 +120,6 @@ export class RedisMemoryUsageCheck extends BaseCheck { } } - /** - * Internal method to ping the redis server - */ - async #ping(): Promise { - /** - * When in connecting status, we should wait for maximum 3 seconds with - * (divided into 3 attempts). However, if there was an error, we will - * not wait for 3 seconds. - */ - if (this.#connection.isConnecting() && this.#pingAttempts < 3 && !this.#connection.lastError) { - await setTimeout(1000) - this.#pingAttempts++ - return this.#ping() - } - - /** - * Re-connect when connection is in closed state - */ - if (this.#connection.isClosed()) { - await this.#connection.ioConnection.connect() - return this.#ping() - } - - /** - * If we are not in `connect` or `ready` state, then we should - * report an error. - */ - if (!this.#connection.isReady()) { - return Result.failed( - 'Unable to connect to the redis server', - this.#connection.lastError - ).mergeMetaData(this.#getConnectionMetadata()) - } - - await this.#connection.ping() - } - /** * Define the memory threshold after which a warning * should be created. @@ -222,6 +186,22 @@ export class RedisMemoryUsageCheck extends BaseCheck { return this } + /** + * Define how long the check waits for a connecting connection, the + * ping and the memory info, together. Defaults to 1 second. + * + * @param duration - Milliseconds or a duration expression (e.g. '2s') + * + * @example + * ```ts + * check.timeout('2s') + * ``` + */ + timeout(duration: string | number) { + this.#timeout = stringHelpers.milliseconds.parse(duration) + return this + } + /** * Executes the health check * @@ -234,60 +214,48 @@ export class RedisMemoryUsageCheck extends BaseCheck { * ``` */ async run(): Promise { - try { - const result = await this.#ping() - if (result) { - return result - } - - /** - * Get memory usage when tracking memory - */ - const memoryUsage = await this.#computeFn(this.#connection) + const result = await probeConnection(this.#connection, this.#timeout, () => this.#checkMemory()) + return result.mergeMetaData(this.#getConnectionMetadata()) + } - /** - * Return early when we do not have access to the memory - * usage. - */ - if (!memoryUsage) { - return Result.failed('Check failed. Unable to get redis memory info').mergeMetaData( - this.#getConnectionMetadata() - ) - } + /** + * Computes the memory usage and compares it with the thresholds + */ + async #checkMemory(): Promise { + const memoryUsage = await this.#computeFn(this.#connection) - const memoryUsagePretty = stringHelpers.bytes.format(memoryUsage) - const warnThresholdPretty = stringHelpers.bytes.format(this.#warnThreshold) - const failureThresholdPretty = stringHelpers.bytes.format(this.#failThreshold) + /** + * Return early when we do not have access to the memory + * usage. + */ + if (!memoryUsage) { + return Result.failed('Check failed. Unable to get redis memory info') + } - /** - * Check if we have crossed the failure threshold - */ - if (this.#failThreshold && memoryUsage > this.#failThreshold) { - return Result.failed( - `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${failureThresholdPretty}` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } + const memoryUsagePretty = stringHelpers.bytes.format(memoryUsage) + const warnThresholdPretty = stringHelpers.bytes.format(this.#warnThreshold) + const failureThresholdPretty = stringHelpers.bytes.format(this.#failThreshold) - /** - * Check if we have crossed the warning threshold - */ - if (this.#warnThreshold && memoryUsage > this.#warnThreshold) { - return Result.warning( - `Redis memory usage is ${stringHelpers.bytes.format(memoryUsage)}, which is above the threshold of ${warnThresholdPretty}` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } + /** + * Check if we have crossed the failure threshold + */ + if (this.#failThreshold && memoryUsage > this.#failThreshold) { + return Result.failed( + `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${failureThresholdPretty}` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) + } - return Result.ok( - `Redis memory usage is ${memoryUsagePretty}, which is under the defined thresholds` - ) - .mergeMetaData(this.#getConnectionMetadata()) - .mergeMetaData(this.#getMemoryMetadata(memoryUsage)) - } catch (error) { - return Result.failed(error).mergeMetaData(this.#getConnectionMetadata()) + /** + * Check if we have crossed the warning threshold + */ + if (this.#warnThreshold && memoryUsage > this.#warnThreshold) { + return Result.warning( + `Redis memory usage is ${memoryUsagePretty}, which is above the threshold of ${warnThresholdPretty}` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) } + + return Result.ok( + `Redis memory usage is ${memoryUsagePretty}, which is under the defined thresholds` + ).mergeMetaData(this.#getMemoryMetadata(memoryUsage)) } } diff --git a/tests/redis_check.spec.ts b/tests/redis_check.spec.ts index c3d4c76..7940e3e 100644 --- a/tests/redis_check.spec.ts +++ b/tests/redis_check.spec.ts @@ -7,9 +7,10 @@ * file that was distributed with this source code. */ +import { once } from 'node:events' import { test } from '@japa/runner' -import { pEvent } from '../tests_helpers/main.ts' +import { createSilentServer, pEvent } from '../tests_helpers/main.ts' import { RedisCheck } from '../src/checks/redis_check.ts' import RedisConnection from '../src/connections/redis_connection.ts' import RedisClusterConnection from '../src/connections/redis_cluster_connection.ts' @@ -62,6 +63,100 @@ test.group('Health check | redis connection', () => { }, }) }) + test('fail within the timeout when the server stops answering', async ({ assert, cleanup }) => { + const server = await createSilentServer() + cleanup(() => server.close()) + + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: server.port, + enableReadyCheck: false, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'ready') + server.silence() + + const healthCheck = new RedisCheck(connection).timeout(200) + for (let run = 0; run < 2; run++) { + const startedAt = Date.now() + const result = await healthCheck.run() + + assert.isBelow(Date.now() - startedAt, 400) + assert.containSubset(result, { + message: 'Redis did not respond within 200ms', + status: 'error', + meta: { connection: { name: 'main', status: 'ready' } }, + }) + } + }) + + test('fail at once with the last error while reconnecting', async ({ assert, cleanup }) => { + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: 1, + retryStrategy: () => 5_000, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'reconnecting') + + const healthCheck = new RedisCheck(connection).timeout('2s') + for (let run = 0; run < 2; run++) { + const startedAt = Date.now() + const result = await healthCheck.run() + + assert.isBelow(Date.now() - startedAt, 50) + assert.containSubset(result, { + message: 'Unable to connect to the redis server', + status: 'error', + meta: { connection: { status: 'reconnecting' }, error: { code: 'ECONNREFUSED' } }, + }) + } + }) + + test('wait for a connecting connection on every run of the same check', async ({ + assert, + cleanup, + }) => { + const connection = new RedisConnection('main', { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + }) + cleanup(() => connection.quit()) + await pEvent(connection, 'ready') + + const healthCheck = new RedisCheck(connection) + for (let run = 0; run < 4; run++) { + connection.ioConnection.disconnect() + await once(connection.ioConnection, 'end') + connection.ioConnection.connect() + assert.equal(connection.status, 'connecting') + + const result = await healthCheck.run() + assert.containSubset(result, { + message: 'Successfully connected to the redis server', + status: 'ok', + meta: { connection: { status: 'ready' } }, + }) + } + }) + + test('do not reconnect a closed connection', async ({ assert }) => { + const connection = new RedisConnection('main', { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + }) + await pEvent(connection, 'ready') + await connection.quit() + await pEvent(connection, 'end') + + const result = await new RedisCheck(connection).run() + assert.containSubset(result, { + message: 'Unable to connect to the redis server', + status: 'error', + meta: { connection: { status: 'end' } }, + }) + assert.equal(connection.status, 'end') + }) }) test.group('Health check | cluster connection', () => { diff --git a/tests/redis_memory_usage_check.spec.ts b/tests/redis_memory_usage_check.spec.ts index eceb08d..1444f2d 100644 --- a/tests/redis_memory_usage_check.spec.ts +++ b/tests/redis_memory_usage_check.spec.ts @@ -10,7 +10,7 @@ import { test } from '@japa/runner' import stringHelpers from '@adonisjs/core/helpers/string' -import { pEvent } from '../tests_helpers/main.ts' +import { createSilentServer, pEvent } from '../tests_helpers/main.ts' import RedisConnection from '../src/connections/redis_connection.ts' import { RedisMemoryUsageCheck } from '../src/checks/redis_memory_usage_check.ts' import RedisClusterConnection from '../src/connections/redis_cluster_connection.ts' @@ -164,6 +164,30 @@ test.group('Redis memory usage | redis connection', () => { }, }) }) + + test('fail when the server does not answer the ping, with a custom compute', async ({ + assert, + cleanup, + }) => { + const server = await createSilentServer() + cleanup(() => server.close()) + + const connection = new RedisConnection('main', { + host: '127.0.0.1', + port: server.port, + enableReadyCheck: false, + }) + cleanup(() => connection.disconnect()) + await pEvent(connection, 'ready') + server.silence() + + const healthCheck = new RedisMemoryUsageCheck(connection).timeout(200).compute(async () => 1024) + + assert.containSubset(await healthCheck.run(), { + message: 'Redis did not respond within 200ms', + status: 'error', + }) + }) }) test.group('Redis memory usage | cluster connection', () => { diff --git a/tests_helpers/main.ts b/tests_helpers/main.ts index 7345835..b498c81 100644 --- a/tests_helpers/main.ts +++ b/tests_helpers/main.ts @@ -7,7 +7,9 @@ * file that was distributed with this source code. */ +import { once } from 'node:events' import type Emittery from 'emittery' +import { createServer, type AddressInfo, type Socket } from 'node:net' /** * Promisify an event @@ -30,3 +32,35 @@ export function pEvent( emitter.on(event, handler) }) } + +/** + * Starts a fake Redis server that answers "+OK" to every command until + * `silence` is called, then keeps connections open without answering, + * like a half-open TCP connection. + */ +export async function createSilentServer() { + let silent = false + const sockets = new Set() + + const server = createServer((socket) => { + sockets.add(socket) + socket.on('data', (chunk) => { + if (!silent) { + const commands = chunk.toString().match(/^\*/gm)?.length ?? 0 + socket.write('+OK\r\n'.repeat(commands)) + } + }) + }).listen(0, '127.0.0.1') + await once(server, 'listening') + + return { + port: (server.address() as AddressInfo).port, + silence() { + silent = true + }, + close() { + sockets.forEach((socket) => socket.destroy()) + server.close() + }, + } +}