jprdonnelly/docs
1
0
Fork 0
mirror of synced 2025-12-23 11:54:18 -05:00
Code Issues Projects Releases Wiki Activity

Redis hardening: retry strategy (#18565)

Browse Source 
* Add an error handler to ensure the Redis server connection is forcibly closed
* Explain unknown errors are usually server idle timeouts
* Add an automatic retry_strategy
* Explicitly set connect_timeout to the default value of 1 hour
This commit is contained in:
James M. Greene
2021-04-05 10:49:46 -05:00
committed by GitHub
parent aeda8ec4bc
commit 920daecd4d
1 changed files with 35 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
lib/redis/create-client.js
View File

@@ -6,10 +6,13 @@ const { REDIS_MIN_DB, REDIS_MAX_DB } = process.env
const redisMinDb = REDIS_MIN_DB || 0
const redisMaxDb = REDIS_MAX_DB || 15
// Maximum delay between reconnection attempts after backoff
const maxReconnectDelay = 5000
function formatRedisError (error) {
const errorCode = error ? error.code : null
const errorName = error ? error.constructor.name : 'Error'
const errorMsg = error ? error.toString() : 'unknown error'
const errorName = error ? error.constructor.name : 'Server disconnection'
const errorMsg = error ? error.toString() : 'unknown (commonly a server idle timeout)'
const preamble = errorName + (errorCode ? ` with code "${errorCode}"` : '')
return preamble + ': ' + errorMsg
}
@@ -51,6 +54,35 @@ module.exports = function createClient (options = {}) {
// This is also critical to preventing a backend pile-up!
enable_offline_queue: false,
// This timeout value will be applied to both the initial connection
// and any auto-reconnect attempts (if the `retry_strategy` option is
// provided). If not using the `retry_strategy` option, this value can be
// set to a very low number. If using the `retry_strategy` option to allow
// more than one reconnection attempt, this value must be set to a higher
// number. Defaults to 1 hour if not configured!
connect_timeout: 60 * 60 * 1000, // 60 minutes
// Be aware that this retry (NOT just reconnection) strategy appears to
// be a major point of confusion (and possibly legitimate issues) between
// reconnecting and retrying failed commands.
retry_strategy:
function ({
attempt,
error,
total_retry_time: totalRetryTime,
times_connected: timesConnected
}) {
let delayPerAttempt = 100
// If the server appears to be unavailable, slow down faster
if (error && error.code === 'ECONNREFUSED') {
delayPerAttempt *= 5
}
// Reconnect after delay
return Math.min(attempt * delayPerAttempt, maxReconnectDelay)
},
// Expand whatever other options and overrides were provided
...options
})
@@ -72,6 +104,7 @@ module.exports = function createClient (options = {}) {
// Add event listeners for basic logging
client.on('connect', () => { console.log(logPrefix, 'Connection opened') })
client.on('ready', () => { console.log(logPrefix, 'Ready to receive commands') })
client.on('end', () => { console.log(logPrefix, 'Connection closed') })
client.on(
'reconnecting',
({
@@ -93,7 +126,6 @@ module.exports = function createClient (options = {}) {
)
}
)
client.on('end', () => { console.log(logPrefix, 'Connection closed') })
client.on('warning', (msg) => { console.warn(logPrefix, 'Warning:', msg) })
client.on('error', (error) => { console.error(logPrefix, formatRedisError(error)) })