diff --git a/packages/pg-pool/README.md b/packages/pg-pool/README.md index 3ff657fe0..91adcd770 100644 --- a/packages/pg-pool/README.md +++ b/packages/pg-pool/README.md @@ -356,6 +356,28 @@ maxUses = rebalanceWindowSeconds * totalRequestsPerSecond / numAppInstances / po 7200 = 1800 * 1000 / 10 / 25 ``` +## Retry on timeout + +It is possible to configure the pool to retry connections that timeout. +This is useful in cases where you have DNS issues or other transient network problems that cause the pool to fail to +connect to the database. + +This can be done by setting the `connectionTimeoutMillis` option, plus passing the `retryOnTimeout`, `maxRetries`, and +`retryDelay` options to the pool constructor. + +```js +const Pool = require('pg-pool') +const pool = new Pool({ + connectionTimeoutMillis: 1000, // timeout for acquiring a connection + retryOnTimeout: true, // retry on timeout + maxRetries: 5, // maximum number of retries + retryDelay: 1_000, // delay between retries in milliseconds +}) +``` +The connection will be retried up to `maxRetries` times if it times out, with a delay of `retryDelay` milliseconds between each retry. +The existing error (connection timeout) will still be thrown if the maximum number of retries is reached. + + ## tests To run tests clone the repo, `npm i` in the working dir, and then run `npm test` diff --git a/packages/pg-pool/index.js b/packages/pg-pool/index.js index b3d9ada96..864e01db9 100644 --- a/packages/pg-pool/index.js +++ b/packages/pg-pool/index.js @@ -91,6 +91,9 @@ class Pool extends EventEmitter { this.options.maxUses = this.options.maxUses || Infinity this.options.allowExitOnIdle = this.options.allowExitOnIdle || false this.options.maxLifetimeSeconds = this.options.maxLifetimeSeconds || 0 + this.options.retryOnTimeout = this.options.retryOnTimeout || false + this.options.maxRetries = this.options.maxRetries || 3 + this.options.retryDelay = this.options.retryDelay || 3_000 this.log = this.options.log || function () {} this.Client = this.options.Client || Client || require('pg').Client this.Promise = this.options.Promise || global.Promise @@ -229,7 +232,7 @@ class Pool extends EventEmitter { return result } - newClient(pendingItem) { + newClient(pendingItem, retryAttempt = 0) { const client = new this.Client(this.options) this._clients.push(client) const idleListener = makeIdleListener(this, client) @@ -258,6 +261,18 @@ class Pool extends EventEmitter { this.log('client failed to connect', err) // remove the dead client from our list of clients this._clients = this._clients.filter((c) => c !== client) + + // Retry on timeout if enabled + if (timeoutHit && this.options.retryOnTimeout && retryAttempt < this.options.maxRetries) { + this.log(`Connection timeout, retry ${retryAttempt + 1}/${this.options.maxRetries}`) + + // delay the retry to avoid tight loops + setTimeout(() => { + this.newClient(pendingItem, retryAttempt + 1) + }, this.options.retryDelay) + + return + } if (timeoutHit) { err = new Error('Connection terminated due to connection timeout', { cause: err }) } diff --git a/packages/pg-pool/test/retry-on-timeout.js b/packages/pg-pool/test/retry-on-timeout.js new file mode 100644 index 000000000..6055d1bea --- /dev/null +++ b/packages/pg-pool/test/retry-on-timeout.js @@ -0,0 +1,124 @@ +'use strict' + +const assert = require('assert') +const Pool = require('../') +const expect = require('expect.js') + +let connectionAttempts = 0 + +// Test Client that simulates always timeout +class TimeoutClient { + constructor() {} + + connect(cb) { + connectionAttempts++ + setTimeout(() => { + const err = new Error('timeout') + err.code = 'ETIMEDOUT' + cb(err) + }, 10) + } + + end() { + // No-op for event handling + } + + on(event, listener) { + // No-op for event handling + } + + removeListener(event, listener) { + // No-op for event handling + } +} + +// Test Client that simulates two timeout then connects successfully +class TimeoutTwiceClient extends TimeoutClient { + constructor() { + super() + this.timeout = true + this.ended = false + } + + connect(cb) { + if (connectionAttempts++ > 1) { + this.timeout = false + } + if (this.timeout) { + setTimeout(() => { + const err = new Error('timeout') + err.code = 'ETIMEDOUT' + cb(err) + }, 10) + } else { + cb() + } + } + + end() { + this.ended = true + } +} + +describe('retry on timeout', () => { + beforeEach(() => { + connectionAttempts = 0 + }) + + it('should retry when client connection times out', function (done) { + const pool = new Pool({ + Client: TimeoutTwiceClient, + max: 1, + connectionTimeoutMillis: 5, + retryOnTimeout: true, + maxRetries: 3, + retryDelay: 15, + }) + + pool.connect((err, client, release) => { + expect(err).to.be(undefined) + expect(client).to.be.an(TimeoutTwiceClient) + assert.equal(connectionAttempts, 3, 'should have tried 3 times') + release() + done() + }) + }) + + it('should fail after max retries', function (done) { + const pool = new Pool({ + Client: TimeoutClient, + max: 1, + connectionTimeoutMillis: 5, + retryOnTimeout: true, + maxRetries: 3, + retryDelay: 15, + }) + + pool.connect((err, client, release) => { + expect(err).to.be.an(Error) + expect(err.message).to.equal('Connection terminated due to connection timeout') + expect(client).to.be(undefined) + assert.equal(connectionAttempts, 4, 'should have tried 4 times (first attempt + 3 retries)') + release() + done() + }) + }) + + it('should not retry when retryOnTimeout is false', function (done) { + const pool = new Pool({ + Client: TimeoutClient, + max: 1, + connectionTimeoutMillis: 5, + retryOnTimeout: false, + }) + + pool.connect((err, client, release) => { + expect(err).to.be.an(Error) + expect(err.message).to.equal('Connection terminated due to connection timeout') + expect(client).to.be(undefined) + assert.equal(connectionAttempts, 1, 'should have tried only once without retries') + release() + done() + }) + }) +})