mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #16 from overleaf/jpa-rewrite
[misc] rewrite in es and promisify
This commit is contained in:
commit
d6492cc564
16 changed files with 5422 additions and 302 deletions
11
libraries/redis-wrapper/.editorconfig
Normal file
11
libraries/redis-wrapper/.editorconfig
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
charset = utf-8
|
||||||
|
end_of_line = lf
|
||||||
|
indent_size = 2
|
||||||
|
indent_style = space
|
||||||
|
insert_final_newline = true
|
||||||
|
max_line_length = 79
|
||||||
|
tab_width = 4
|
||||||
|
trim_trailing_whitespace = true
|
23
libraries/redis-wrapper/.eslintrc.json
Normal file
23
libraries/redis-wrapper/.eslintrc.json
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"extends": [
|
||||||
|
"standard",
|
||||||
|
"plugin:prettier/recommended",
|
||||||
|
"plugin:mocha/recommended",
|
||||||
|
"plugin:chai-expect/recommended",
|
||||||
|
"plugin:chai-friendly/recommended"
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"node": true
|
||||||
|
},
|
||||||
|
"parserOptions": {
|
||||||
|
"ecmaVersion": 2018
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"files": ["test/**/*.js"],
|
||||||
|
"env": {
|
||||||
|
"mocha": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
4
libraries/redis-wrapper/.prettierrc.json
Normal file
4
libraries/redis-wrapper/.prettierrc.json
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
{
|
||||||
|
"semi": false,
|
||||||
|
"singleQuote": true
|
||||||
|
}
|
15
libraries/redis-wrapper/Errors.js
Normal file
15
libraries/redis-wrapper/Errors.js
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
const OError = require('@overleaf/o-error')
|
||||||
|
|
||||||
|
class RedisError extends OError {}
|
||||||
|
class RedisHealthCheckFailed extends RedisError {}
|
||||||
|
class RedisHealthCheckTimedOut extends RedisHealthCheckFailed {}
|
||||||
|
class RedisHealthCheckWriteError extends RedisHealthCheckFailed {}
|
||||||
|
class RedisHealthCheckVerifyError extends RedisHealthCheckFailed {}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
RedisError,
|
||||||
|
RedisHealthCheckFailed,
|
||||||
|
RedisHealthCheckTimedOut,
|
||||||
|
RedisHealthCheckWriteError,
|
||||||
|
RedisHealthCheckVerifyError,
|
||||||
|
}
|
|
@ -1,35 +0,0 @@
|
||||||
module.exports = (grunt) ->
|
|
||||||
|
|
||||||
# Project configuration.
|
|
||||||
grunt.initConfig
|
|
||||||
coffee:
|
|
||||||
server:
|
|
||||||
expand: true,
|
|
||||||
flatten: false,
|
|
||||||
cwd: 'app/coffee',
|
|
||||||
src: ['**/*.coffee'],
|
|
||||||
dest: 'app/js/',
|
|
||||||
ext: '.js'
|
|
||||||
|
|
||||||
server_tests:
|
|
||||||
expand: true,
|
|
||||||
flatten: false,
|
|
||||||
cwd: 'test/unit/coffee',
|
|
||||||
src: ['**/*.coffee'],
|
|
||||||
dest: 'test/unit/js/',
|
|
||||||
ext: '.js'
|
|
||||||
|
|
||||||
mochaTest:
|
|
||||||
unit:
|
|
||||||
options:
|
|
||||||
reporter: process.env.MOCHA_RUNNER || "spec"
|
|
||||||
grep: grunt.option("grep")
|
|
||||||
require: 'coffee-script/register'
|
|
||||||
src: ['test.coffee']
|
|
||||||
|
|
||||||
|
|
||||||
grunt.loadNpmTasks 'grunt-contrib-coffee'
|
|
||||||
grunt.loadNpmTasks 'grunt-mocha-test'
|
|
||||||
|
|
||||||
grunt.registerTask 'test:unit', ['mochaTest:unit']
|
|
||||||
|
|
|
@ -1,13 +0,0 @@
|
||||||
# execute this script with a redis container running to test the health check
|
|
||||||
# starting and stopping redis with this script running is a good test
|
|
||||||
|
|
||||||
redis = require "./index.coffee"
|
|
||||||
|
|
||||||
rclient = redis.createClient({host:"localhost",port:"6379"})
|
|
||||||
setInterval () ->
|
|
||||||
rclient.healthCheck (err) ->
|
|
||||||
if err?
|
|
||||||
console.log "HEALTH CHECK FAILED", err
|
|
||||||
else
|
|
||||||
console.log "HEALTH CHECK OK"
|
|
||||||
, 1000
|
|
|
@ -1,111 +0,0 @@
|
||||||
_ = require("underscore")
|
|
||||||
async = require "async"
|
|
||||||
os = require('os')
|
|
||||||
crypto = require('crypto')
|
|
||||||
|
|
||||||
# generate unique values for health check
|
|
||||||
HOST = os.hostname()
|
|
||||||
PID = process.pid
|
|
||||||
RND = crypto.randomBytes(4).toString('hex')
|
|
||||||
COUNT = 0
|
|
||||||
|
|
||||||
module.exports = RedisSharelatex =
|
|
||||||
createClient: (opts = {port: 6379, host: "localhost"})->
|
|
||||||
if !opts.retry_max_delay?
|
|
||||||
opts.retry_max_delay = 5000 # ms
|
|
||||||
|
|
||||||
if opts.endpoints?
|
|
||||||
standardOpts = _.clone(opts)
|
|
||||||
delete standardOpts.endpoints
|
|
||||||
delete standardOpts.masterName
|
|
||||||
client = require("redis-sentinel").createClient opts.endpoints, opts.masterName, standardOpts
|
|
||||||
client.healthCheck = RedisSharelatex.singleInstanceHealthCheckBuilder(client)
|
|
||||||
else if opts.cluster?
|
|
||||||
Redis = require("ioredis")
|
|
||||||
standardOpts = _.clone(opts)
|
|
||||||
delete standardOpts.cluster
|
|
||||||
delete standardOpts.key_schema
|
|
||||||
client = new Redis.Cluster(opts.cluster, standardOpts)
|
|
||||||
client.healthCheck = RedisSharelatex.clusterHealthCheckBuilder(client)
|
|
||||||
RedisSharelatex._monkeyPatchIoredisExec(client)
|
|
||||||
else
|
|
||||||
standardOpts = _.clone(opts)
|
|
||||||
ioredis = require("ioredis")
|
|
||||||
client = new ioredis(standardOpts)
|
|
||||||
RedisSharelatex._monkeyPatchIoredisExec(client)
|
|
||||||
client.healthCheck = RedisSharelatex.singleInstanceHealthCheckBuilder(client)
|
|
||||||
return client
|
|
||||||
|
|
||||||
HEARTBEAT_TIMEOUT: 2000
|
|
||||||
singleInstanceHealthCheckBuilder: (client) ->
|
|
||||||
healthCheck = (callback) ->
|
|
||||||
RedisSharelatex._checkClient(client, callback)
|
|
||||||
return healthCheck
|
|
||||||
|
|
||||||
clusterHealthCheckBuilder: (client) ->
|
|
||||||
healthCheck = (callback) ->
|
|
||||||
jobs = client.nodes("all").map (node) =>
|
|
||||||
(cb) => RedisSharelatex._checkClient(node, cb)
|
|
||||||
async.parallel jobs, callback
|
|
||||||
|
|
||||||
return healthCheck
|
|
||||||
|
|
||||||
_checkClient: (client, callback) ->
|
|
||||||
callback = _.once(callback)
|
|
||||||
# check the redis connection by storing and retrieving a unique key/value pair
|
|
||||||
uniqueToken = "host=#{HOST}:pid=#{PID}:random=#{RND}:time=#{Date.now()}:count=#{COUNT++}"
|
|
||||||
timer = setTimeout () ->
|
|
||||||
error = new Error("redis client health check timed out #{client?.options?.host}")
|
|
||||||
console.error {
|
|
||||||
err: error,
|
|
||||||
key: client.options?.key # only present for cluster
|
|
||||||
clientOptions: client.options
|
|
||||||
uniqueToken: uniqueToken
|
|
||||||
}, "client timed out"
|
|
||||||
callback(error)
|
|
||||||
, RedisSharelatex.HEARTBEAT_TIMEOUT
|
|
||||||
healthCheckKey = "_redis-wrapper:healthCheckKey:{#{uniqueToken}}"
|
|
||||||
healthCheckValue = "_redis-wrapper:healthCheckValue:{#{uniqueToken}}"
|
|
||||||
# set the unique key/value pair
|
|
||||||
multi = client.multi()
|
|
||||||
multi.set healthCheckKey, healthCheckValue, "EX", 60
|
|
||||||
multi.exec (err, reply) ->
|
|
||||||
if err?
|
|
||||||
clearTimeout timer
|
|
||||||
return callback(err)
|
|
||||||
# check that we can retrieve the unique key/value pair
|
|
||||||
multi = client.multi()
|
|
||||||
multi.get healthCheckKey
|
|
||||||
multi.del healthCheckKey
|
|
||||||
multi.exec (err, reply) ->
|
|
||||||
clearTimeout timer
|
|
||||||
return callback(err) if err?
|
|
||||||
return callback(new Error("bad response from redis health check")) if reply?[0] isnt healthCheckValue or reply?[1] isnt 1
|
|
||||||
return callback()
|
|
||||||
|
|
||||||
_monkeyPatchIoredisExec: (client) ->
|
|
||||||
_multi = client.multi
|
|
||||||
client.multi = (args...) ->
|
|
||||||
multi = _multi.call(client, args...)
|
|
||||||
_exec = multi.exec
|
|
||||||
multi.exec = (callback = () ->) ->
|
|
||||||
_exec.call multi, (error, result) ->
|
|
||||||
# ioredis exec returns an results like:
|
|
||||||
# [ [null, 42], [null, "foo"] ]
|
|
||||||
# where the first entries in each 2-tuple are
|
|
||||||
# presumably errors for each individual command,
|
|
||||||
# and the second entry is the result. We need to transform
|
|
||||||
# this into the same result as the old redis driver:
|
|
||||||
# [ 42, "foo" ]
|
|
||||||
filtered_result = []
|
|
||||||
for entry in result or []
|
|
||||||
if entry[0]?
|
|
||||||
return callback(entry[0])
|
|
||||||
else
|
|
||||||
filtered_result.push entry[1]
|
|
||||||
callback error, filtered_result
|
|
||||||
return multi
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,2 +1,173 @@
|
||||||
require("coffee-script").register();
|
const crypto = require('crypto')
|
||||||
module.exports = require("./index.coffee");
|
const os = require('os')
|
||||||
|
const { promisify } = require('util')
|
||||||
|
|
||||||
|
const Redis = require('ioredis')
|
||||||
|
|
||||||
|
const {
|
||||||
|
RedisHealthCheckTimedOut,
|
||||||
|
RedisHealthCheckWriteError,
|
||||||
|
RedisHealthCheckVerifyError,
|
||||||
|
} = require('./Errors')
|
||||||
|
|
||||||
|
const HEARTBEAT_TIMEOUT = 2000
|
||||||
|
|
||||||
|
// generate unique values for health check
|
||||||
|
const HOST = os.hostname()
|
||||||
|
const PID = process.pid
|
||||||
|
const RND = crypto.randomBytes(4).toString('hex')
|
||||||
|
let COUNT = 0
|
||||||
|
|
||||||
|
function createClient(opts) {
|
||||||
|
const standardOpts = Object.assign({}, opts)
|
||||||
|
delete standardOpts.key_schema
|
||||||
|
|
||||||
|
if (standardOpts.retry_max_delay == null) {
|
||||||
|
standardOpts.retry_max_delay = 5000 // ms
|
||||||
|
}
|
||||||
|
|
||||||
|
if (standardOpts.endpoints) {
|
||||||
|
throw new Error(
|
||||||
|
'@overleaf/redis-wrapper: redis-sentinel is no longer supported'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
let client
|
||||||
|
if (standardOpts.cluster) {
|
||||||
|
delete standardOpts.cluster
|
||||||
|
client = new Redis.Cluster(opts.cluster, standardOpts)
|
||||||
|
} else {
|
||||||
|
client = new Redis(standardOpts)
|
||||||
|
}
|
||||||
|
monkeyPatchIoRedisExec(client)
|
||||||
|
client.healthCheck = (callback) => {
|
||||||
|
if (callback) {
|
||||||
|
// callback based invocation
|
||||||
|
healthCheck(client).then(callback).catch(callback)
|
||||||
|
} else {
|
||||||
|
// Promise based invocation
|
||||||
|
return healthCheck(client)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
async function healthCheck(client) {
|
||||||
|
// check the redis connection by storing and retrieving a unique key/value pair
|
||||||
|
const uniqueToken = `host=${HOST}:pid=${PID}:random=${RND}:time=${Date.now()}:count=${COUNT++}`
|
||||||
|
|
||||||
|
// o-error context
|
||||||
|
const context = {
|
||||||
|
uniqueToken,
|
||||||
|
stage: 'add context for a timeout',
|
||||||
|
}
|
||||||
|
|
||||||
|
await runWithTimeout({
|
||||||
|
runner: runCheck(client, uniqueToken, context),
|
||||||
|
timeout: HEARTBEAT_TIMEOUT,
|
||||||
|
context,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCheck(client, uniqueToken, context) {
|
||||||
|
const healthCheckKey = `_redis-wrapper:healthCheckKey:{${uniqueToken}}`
|
||||||
|
const healthCheckValue = `_redis-wrapper:healthCheckValue:{${uniqueToken}}`
|
||||||
|
|
||||||
|
// set the unique key/value pair
|
||||||
|
context.stage = 'write'
|
||||||
|
const writeAck = await client
|
||||||
|
.set(healthCheckKey, healthCheckValue, 'EX', 60)
|
||||||
|
.catch((err) => {
|
||||||
|
throw new RedisHealthCheckWriteError('write errored', context, err)
|
||||||
|
})
|
||||||
|
if (writeAck !== 'OK') {
|
||||||
|
context.writeAck = writeAck
|
||||||
|
throw new RedisHealthCheckWriteError('write failed', context)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check that we can retrieve the unique key/value pair
|
||||||
|
context.stage = 'verify'
|
||||||
|
const [roundTrippedHealthCheckValue, deleteAck] = await client
|
||||||
|
.multi()
|
||||||
|
.get(healthCheckKey)
|
||||||
|
.del(healthCheckKey)
|
||||||
|
.exec()
|
||||||
|
.catch((err) => {
|
||||||
|
throw new RedisHealthCheckVerifyError(
|
||||||
|
'read/delete errored',
|
||||||
|
context,
|
||||||
|
err
|
||||||
|
)
|
||||||
|
})
|
||||||
|
if (roundTrippedHealthCheckValue !== healthCheckValue) {
|
||||||
|
context.roundTrippedHealthCheckValue = roundTrippedHealthCheckValue
|
||||||
|
throw new RedisHealthCheckVerifyError('read failed', context)
|
||||||
|
}
|
||||||
|
if (deleteAck !== 1) {
|
||||||
|
context.deleteAck = deleteAck
|
||||||
|
throw new RedisHealthCheckVerifyError('delete failed', context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function unwrapMultiResult(result, callback) {
|
||||||
|
// ioredis exec returns a results like:
|
||||||
|
// [ [null, 42], [null, "foo"] ]
|
||||||
|
// where the first entries in each 2-tuple are
|
||||||
|
// presumably errors for each individual command,
|
||||||
|
// and the second entry is the result. We need to transform
|
||||||
|
// this into the same result as the old redis driver:
|
||||||
|
// [ 42, "foo" ]
|
||||||
|
//
|
||||||
|
// Basically reverse:
|
||||||
|
// https://github.com/luin/ioredis/blob/v4.17.3/lib/utils/index.ts#L75-L92
|
||||||
|
const filteredResult = []
|
||||||
|
for (const [err, value] of result || []) {
|
||||||
|
if (err) {
|
||||||
|
return callback(err)
|
||||||
|
} else {
|
||||||
|
filteredResult.push(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
callback(null, filteredResult)
|
||||||
|
}
|
||||||
|
const unwrapMultiResultPromisified = promisify(unwrapMultiResult)
|
||||||
|
|
||||||
|
function monkeyPatchIoRedisExec(client) {
|
||||||
|
const _multi = client.multi
|
||||||
|
client.multi = function () {
|
||||||
|
const multi = _multi.apply(client, arguments)
|
||||||
|
const _exec = multi.exec
|
||||||
|
multi.exec = (callback) => {
|
||||||
|
if (callback) {
|
||||||
|
// callback based invocation
|
||||||
|
_exec.call(multi, (error, result) => {
|
||||||
|
// The command can fail all-together due to syntax errors
|
||||||
|
if (error) return callback(error)
|
||||||
|
unwrapMultiResult(result, callback)
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
// Promise based invocation
|
||||||
|
return _exec.call(multi).then(unwrapMultiResultPromisified)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return multi
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runWithTimeout({ runner, timeout, context }) {
|
||||||
|
let healthCheckDeadline
|
||||||
|
await Promise.race([
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
|
healthCheckDeadline = setTimeout(() => {
|
||||||
|
// attach the timeout when hitting the timeout only
|
||||||
|
context.timeout = timeout
|
||||||
|
reject(new RedisHealthCheckTimedOut('timeout', context))
|
||||||
|
}, timeout)
|
||||||
|
}),
|
||||||
|
runner.finally(() => clearTimeout(healthCheckDeadline)),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createClient,
|
||||||
|
}
|
||||||
|
|
4872
libraries/redis-wrapper/package-lock.json
generated
Normal file
4872
libraries/redis-wrapper/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
|
@ -1,24 +1,46 @@
|
||||||
{
|
{
|
||||||
"name": "redis-sharelatex",
|
"name": "@overleaf/redis-wrapper",
|
||||||
"version": "1.0.13",
|
"version": "2.0.0",
|
||||||
"description": "Redis wrapper for node which will either use cluster, sentinal, or single instance redis",
|
"description": "Redis wrapper for node which will either use cluster or single instance redis",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"author": "ShareLaTeX",
|
"files": [
|
||||||
|
"index.js",
|
||||||
|
"Errors.js"
|
||||||
|
],
|
||||||
|
"author": "Overleaf (https://www.overleaf.com)",
|
||||||
|
"repository": "github:overleaf/redis-wrapper",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
|
"scripts": {
|
||||||
|
"lint": "eslint --max-warnings 0 .",
|
||||||
|
"format": "prettier-eslint $PWD'/**/*.js' --list-different",
|
||||||
|
"format:fix": "prettier-eslint $PWD'/**/*.js' --write",
|
||||||
|
"test": "mocha --recursive test/unit/src/"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@overleaf/o-error": "^3.1.0"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"async": "^2.5.0",
|
"ioredis": "~4.17.3"
|
||||||
"coffee-script": "1.8.0",
|
|
||||||
"ioredis": "~4.17.3",
|
|
||||||
"redis-sentinel": "0.1.1",
|
|
||||||
"underscore": "1.7.0"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"chai": "1.9.1",
|
"@overleaf/o-error": "^3.1.0",
|
||||||
"grunt": "0.4.5",
|
"chai": "^4.2.0",
|
||||||
"grunt-contrib-coffee": "0.11.1",
|
"eslint": "^6.8.0",
|
||||||
"grunt-mocha-test": "0.12.0",
|
"eslint-config-prettier": "^6.10.1",
|
||||||
"mocha": "1.21.4",
|
"eslint-config-standard": "^14.1.1",
|
||||||
"sandboxed-module": "1.0.1",
|
"eslint-plugin-chai-expect": "^2.1.0",
|
||||||
"sinon": "1.10.3"
|
"eslint-plugin-chai-friendly": "^0.5.0",
|
||||||
|
"eslint-plugin-import": "^2.20.2",
|
||||||
|
"eslint-plugin-mocha": "^6.3.0",
|
||||||
|
"eslint-plugin-node": "^11.1.0",
|
||||||
|
"eslint-plugin-prettier": "^3.1.3",
|
||||||
|
"eslint-plugin-promise": "^4.2.1",
|
||||||
|
"eslint-plugin-standard": "^4.0.1",
|
||||||
|
"logger-sharelatex": "^2.2.0",
|
||||||
|
"mocha": "^8.2.1",
|
||||||
|
"prettier": "^2.0.2",
|
||||||
|
"prettier-eslint-cli": "^5.0.0",
|
||||||
|
"sandboxed-module": "^2.0.4",
|
||||||
|
"sinon": "^9.2.1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,125 +0,0 @@
|
||||||
should = require('chai').should()
|
|
||||||
SandboxedModule = require('sandboxed-module')
|
|
||||||
assert = require('assert')
|
|
||||||
path = require('path')
|
|
||||||
sinon = require('sinon')
|
|
||||||
modulePath = path.join __dirname, "./index.coffee"
|
|
||||||
expect = require("chai").expect
|
|
||||||
|
|
||||||
describe "index", ->
|
|
||||||
|
|
||||||
beforeEach ->
|
|
||||||
|
|
||||||
@settings = {}
|
|
||||||
@sentinelClient =
|
|
||||||
set: ->
|
|
||||||
on: ->
|
|
||||||
@normalRedisClient =
|
|
||||||
get: ->
|
|
||||||
on: ->
|
|
||||||
@ioredisConstructor = ioredisConstructor = sinon.stub()
|
|
||||||
|
|
||||||
@sentinel =
|
|
||||||
createClient: sinon.stub().returns(@sentinelClient)
|
|
||||||
@normalRedis =
|
|
||||||
createClient: sinon.stub().returns(@normalRedisClient)
|
|
||||||
@ioredis = class IoRedis
|
|
||||||
constructor: ioredisConstructor
|
|
||||||
on: sinon.stub()
|
|
||||||
@ioredis.Cluster = class Cluster
|
|
||||||
constructor: (@config, @options) ->
|
|
||||||
on: sinon.stub()
|
|
||||||
@redis = SandboxedModule.require modulePath, requires:
|
|
||||||
"redis-sentinel":@sentinel
|
|
||||||
"redis":@normalRedis
|
|
||||||
"ioredis": @ioredis
|
|
||||||
@auth_pass = "1234 pass"
|
|
||||||
@endpoints = [
|
|
||||||
{host: '127.0.0.1', port: 26379},
|
|
||||||
{host: '127.0.0.1', port: 26380}
|
|
||||||
]
|
|
||||||
|
|
||||||
describe "sentinel", ->
|
|
||||||
beforeEach ->
|
|
||||||
@masterName = "my master"
|
|
||||||
@sentinelOptions =
|
|
||||||
endpoints:@endpoints
|
|
||||||
masterName:@masterName
|
|
||||||
auth_pass:@auth_pass
|
|
||||||
|
|
||||||
it "should use sentinal if the first argument in an array", ->
|
|
||||||
client = @redis.createClient @sentinelOptions
|
|
||||||
@sentinel.createClient.called.should.equal true
|
|
||||||
@normalRedis.createClient.called.should.equal false
|
|
||||||
client.should.equal @sentinelClient
|
|
||||||
|
|
||||||
it "should pass the options correctly though", ->
|
|
||||||
client = @redis.createClient @sentinelOptions
|
|
||||||
@sentinel.createClient.calledWith(@endpoints, @masterName, {auth_pass:@auth_pass, retry_max_delay: 5000}).should.equal true
|
|
||||||
client.should.equal @sentinelClient
|
|
||||||
|
|
||||||
describe "single node redis", ->
|
|
||||||
beforeEach ->
|
|
||||||
@standardOpts =
|
|
||||||
auth_pass: @auth_pass
|
|
||||||
port: 1234
|
|
||||||
host: "redis.mysite.env"
|
|
||||||
|
|
||||||
it "should use the ioredis driver in single-instance mode if a non array is passed", ->
|
|
||||||
client = @redis.createClient @standardOpts
|
|
||||||
@sentinel.createClient.called.should.equal false
|
|
||||||
@normalRedis.createClient.called.should.equal false
|
|
||||||
assert.equal(client.constructor, @ioredis)
|
|
||||||
|
|
||||||
it "should call createClient for the ioredis driver in single-instance mode if a non array is passed", ->
|
|
||||||
client = @redis.createClient @standardOpts
|
|
||||||
@ioredisConstructor.calledWith(@standardOpts).should.equal true
|
|
||||||
|
|
||||||
describe "cluster", ->
|
|
||||||
beforeEach ->
|
|
||||||
@cluster = [{"mock": "cluster"}, { "mock": "cluster2"}]
|
|
||||||
@extraOptions = {keepAlive:100}
|
|
||||||
@settings =
|
|
||||||
cluster: @cluster
|
|
||||||
redisOptions: @extraOptions
|
|
||||||
key_schema: {foo: (x) -> "#{x}"}
|
|
||||||
|
|
||||||
it "should pass the options correctly though with no options", ->
|
|
||||||
client = @redis.createClient cluster: @cluster
|
|
||||||
assert(client instanceof @ioredis.Cluster)
|
|
||||||
client.config.should.deep.equal @cluster
|
|
||||||
|
|
||||||
it "should not pass the key_schema through to the driver", ->
|
|
||||||
client = @redis.createClient cluster: @cluster, key_schema: "foobar"
|
|
||||||
assert(client instanceof @ioredis.Cluster)
|
|
||||||
client.config.should.deep.equal @cluster
|
|
||||||
expect(client.options).to.deep.equal {retry_max_delay: 5000}
|
|
||||||
|
|
||||||
it "should pass the options correctly though with additional options", ->
|
|
||||||
client = @redis.createClient @settings
|
|
||||||
assert(client instanceof @ioredis.Cluster)
|
|
||||||
client.config.should.deep.equal @cluster
|
|
||||||
# need to use expect here because of _.clone in sandbox
|
|
||||||
expect(client.options).to.deep.equal {redisOptions: @extraOptions, retry_max_delay: 5000}
|
|
||||||
|
|
||||||
describe "monkey patch ioredis exec", ->
|
|
||||||
beforeEach ->
|
|
||||||
@callback = sinon.stub()
|
|
||||||
@results = []
|
|
||||||
@multiOrig = { exec: sinon.stub().yields(null, @results)}
|
|
||||||
@client = { multi: sinon.stub().returns(@multiOrig) }
|
|
||||||
@redis._monkeyPatchIoredisExec(@client)
|
|
||||||
@multi = @client.multi()
|
|
||||||
|
|
||||||
it "should return the old redis format for an array", ->
|
|
||||||
@results[0] = [null, 42]
|
|
||||||
@results[1] = [null, "foo"]
|
|
||||||
@multi.exec @callback
|
|
||||||
@callback.calledWith(null, [42, "foo"]).should.equal true
|
|
||||||
|
|
||||||
it "should return the old redis format when there is an error", ->
|
|
||||||
@results[0] = [null, 42]
|
|
||||||
@results[1] = ["error", "foo"]
|
|
||||||
@multi.exec @callback
|
|
||||||
@callback.calledWith("error").should.equal true
|
|
||||||
|
|
4
libraries/redis-wrapper/test/scripts/cluster/clear-dbs.sh
Executable file
4
libraries/redis-wrapper/test/scripts/cluster/clear-dbs.sh
Executable file
|
@ -0,0 +1,4 @@
|
||||||
|
while true; do
|
||||||
|
seq 0 8 \
|
||||||
|
| xargs -I% redis-cli -p 700% FLUSHALL > /dev/null
|
||||||
|
done
|
26
libraries/redis-wrapper/test/scripts/cluster/cluster.js
Normal file
26
libraries/redis-wrapper/test/scripts/cluster/cluster.js
Normal file
|
@ -0,0 +1,26 @@
|
||||||
|
/*
|
||||||
|
execute this script with a redis cluster running to test the health check.
|
||||||
|
starting and stopping shards with this script running is a good test.
|
||||||
|
|
||||||
|
to create a new cluster, use $ ./create-redis-cluster.sh
|
||||||
|
to run a chaos monkey, use $ ./clear-dbs.sh
|
||||||
|
*/
|
||||||
|
|
||||||
|
const redis = require('../../../')
|
||||||
|
const logger = require('logger-sharelatex')
|
||||||
|
|
||||||
|
const rclient = redis.createClient({
|
||||||
|
cluster: Array.from({ length: 9 }).map((value, index) => {
|
||||||
|
return { host: '127.0.0.1', port: 7000 + index }
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
rclient.healthCheck((err) => {
|
||||||
|
if (err) {
|
||||||
|
logger.error({ err }, 'HEALTH CHECK FAILED')
|
||||||
|
} else {
|
||||||
|
logger.log('HEALTH CHECK OK')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}, 1000)
|
73
libraries/redis-wrapper/test/scripts/cluster/create-cluster.sh
Executable file
73
libraries/redis-wrapper/test/scripts/cluster/create-cluster.sh
Executable file
|
@ -0,0 +1,73 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# USAGE: $0 [NUMBER_OF_NODES, default: 9] [DATA_DIR, default: a new temp dir]
|
||||||
|
#
|
||||||
|
# ports are assigned from 7000 on
|
||||||
|
#
|
||||||
|
# NOTE: the cluster setup requires redis 5+
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
COUNT=${1:-9}
|
||||||
|
DATA=$2
|
||||||
|
|
||||||
|
if [[ -z "$DATA" ]]; then
|
||||||
|
IS_TEMP=1
|
||||||
|
TEMP=`mktemp -d`
|
||||||
|
DATA="$TEMP"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HAS_DATA=
|
||||||
|
if [[ -e "$DATA/7000/node.conf" ]]; then
|
||||||
|
HAS_DATA=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PIDs=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
# ensure that we delete the temp dir, no matter how the kill cmd exists
|
||||||
|
set +e
|
||||||
|
# invoke kill with at least one PID
|
||||||
|
echo "$PIDs" | xargs -r kill
|
||||||
|
if [[ ! -z "$IS_TEMP" ]]; then
|
||||||
|
rm -rf "$TEMP"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup exit
|
||||||
|
|
||||||
|
for NUM in `seq "$COUNT"`; do
|
||||||
|
PORT=`expr 6999 + "$NUM"`
|
||||||
|
CWD="$DATA/$PORT"
|
||||||
|
mkdir -p "$CWD"
|
||||||
|
pushd "$CWD"
|
||||||
|
redis-server \
|
||||||
|
--appendonly no \
|
||||||
|
--cluster-enabled yes \
|
||||||
|
--cluster-config-file node.conf \
|
||||||
|
--port "$PORT" \
|
||||||
|
--save "" \
|
||||||
|
> /dev/null \
|
||||||
|
&
|
||||||
|
PIDs="$PIDs $!"
|
||||||
|
popd
|
||||||
|
done
|
||||||
|
|
||||||
|
# initial nodes
|
||||||
|
if [[ -z "$HAS_DATA" ]]; then
|
||||||
|
# confirm the setup
|
||||||
|
echo yes \
|
||||||
|
| redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002
|
||||||
|
fi
|
||||||
|
|
||||||
|
# scale up as requested
|
||||||
|
for NUM in `seq 4 "$COUNT"`; do
|
||||||
|
PORT=`expr 6999 + "$NUM"`
|
||||||
|
GUARD="$DATA/$PORT/.joined"
|
||||||
|
if [[ ! -e "$GUARD" ]]; then
|
||||||
|
redis-cli --cluster add-node "127.0.0.1:$PORT" 127.0.0.1:7000 --cluster-slave
|
||||||
|
touch "$GUARD"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "CLUSTER IS READY" >&2
|
||||||
|
wait
|
17
libraries/redis-wrapper/test/scripts/standalone.js
Normal file
17
libraries/redis-wrapper/test/scripts/standalone.js
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
// execute this script with a redis container running to test the health check
|
||||||
|
// starting and stopping redis with this script running is a good test
|
||||||
|
|
||||||
|
const redis = require('../../')
|
||||||
|
const logger = require('logger-sharelatex')
|
||||||
|
|
||||||
|
const rclient = redis.createClient({ host: 'localhost', port: '6379' })
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
rclient.healthCheck((err) => {
|
||||||
|
if (err) {
|
||||||
|
logger.error({ err }, 'HEALTH CHECK FAILED')
|
||||||
|
} else {
|
||||||
|
logger.log('HEALTH CHECK OK')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}, 1000)
|
166
libraries/redis-wrapper/test/unit/src/test.js
Normal file
166
libraries/redis-wrapper/test/unit/src/test.js
Normal file
|
@ -0,0 +1,166 @@
|
||||||
|
/*
|
||||||
|
* decaffeinate suggestions:
|
||||||
|
* DS102: Remove unnecessary code created because of implicit returns
|
||||||
|
* DS206: Consider reworking classes to avoid initClass
|
||||||
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||||
|
*/
|
||||||
|
const should = require('chai').should()
|
||||||
|
const SandboxedModule = require('sandboxed-module')
|
||||||
|
const assert = require('assert')
|
||||||
|
const path = require('path')
|
||||||
|
const sinon = require('sinon')
|
||||||
|
const modulePath = path.join(__dirname, './../../../index.js')
|
||||||
|
const { expect } = require('chai')
|
||||||
|
|
||||||
|
describe('index', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
let Cluster, IoRedis, ioredisConstructor
|
||||||
|
this.settings = {}
|
||||||
|
this.ioredisConstructor = ioredisConstructor = sinon.stub()
|
||||||
|
|
||||||
|
this.ioredis = IoRedis = (function () {
|
||||||
|
let createIoRedis
|
||||||
|
IoRedis = class IoRedis {
|
||||||
|
static initClass() {
|
||||||
|
this.prototype.on = sinon.stub()
|
||||||
|
createIoRedis = ioredisConstructor
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
return createIoRedis.apply(this, arguments)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
IoRedis.initClass()
|
||||||
|
return IoRedis
|
||||||
|
})()
|
||||||
|
this.ioredis.Cluster = Cluster = (function () {
|
||||||
|
Cluster = class Cluster {
|
||||||
|
static initClass() {
|
||||||
|
this.prototype.on = sinon.stub()
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(config, options) {
|
||||||
|
this.config = config
|
||||||
|
this.options = options
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Cluster.initClass()
|
||||||
|
return Cluster
|
||||||
|
})()
|
||||||
|
this.redis = SandboxedModule.require(modulePath, {
|
||||||
|
requires: {
|
||||||
|
ioredis: this.ioredis,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return (this.auth_pass = '1234 pass')
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('redis-sentinel', function () {
|
||||||
|
it('should throw an error when creating a client', function () {
|
||||||
|
const redisSentinelOptions = {
|
||||||
|
endpoints: ['127.0.0.1:1234', '127.0.0.1:2345', '127.0.0.1:3456'],
|
||||||
|
}
|
||||||
|
const createNewClient = () => {
|
||||||
|
this.redis.createClient(redisSentinelOptions)
|
||||||
|
}
|
||||||
|
expect(createNewClient).to.throw(
|
||||||
|
'@overleaf/redis-wrapper: redis-sentinel is no longer supported'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('single node redis', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
return (this.standardOpts = {
|
||||||
|
auth_pass: this.auth_pass,
|
||||||
|
port: 1234,
|
||||||
|
host: 'redis.mysite.env',
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should work without opts', function () {
|
||||||
|
this.redis.createClient()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should use the ioredis driver in single-instance mode if a non array is passed', function () {
|
||||||
|
const client = this.redis.createClient(this.standardOpts)
|
||||||
|
return assert.equal(client.constructor, this.ioredis)
|
||||||
|
})
|
||||||
|
|
||||||
|
return it('should call createClient for the ioredis driver in single-instance mode if a non array is passed', function () {
|
||||||
|
const client = this.redis.createClient(this.standardOpts)
|
||||||
|
return this.ioredisConstructor
|
||||||
|
.calledWith(sinon.match(this.standardOpts))
|
||||||
|
.should.equal(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('cluster', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
this.cluster = [{ mock: 'cluster' }, { mock: 'cluster2' }]
|
||||||
|
this.extraOptions = { keepAlive: 100 }
|
||||||
|
return (this.settings = {
|
||||||
|
cluster: this.cluster,
|
||||||
|
redisOptions: this.extraOptions,
|
||||||
|
key_schema: {
|
||||||
|
foo(x) {
|
||||||
|
return `${x}`
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should pass the options correctly though with no options', function () {
|
||||||
|
const client = this.redis.createClient({ cluster: this.cluster })
|
||||||
|
assert(client instanceof this.ioredis.Cluster)
|
||||||
|
return client.config.should.deep.equal(this.cluster)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not pass the key_schema through to the driver', function () {
|
||||||
|
const client = this.redis.createClient({
|
||||||
|
cluster: this.cluster,
|
||||||
|
key_schema: 'foobar',
|
||||||
|
})
|
||||||
|
assert(client instanceof this.ioredis.Cluster)
|
||||||
|
client.config.should.deep.equal(this.cluster)
|
||||||
|
return expect(client.options).to.deep.equal({ retry_max_delay: 5000 })
|
||||||
|
})
|
||||||
|
|
||||||
|
return it('should pass the options correctly though with additional options', function () {
|
||||||
|
const client = this.redis.createClient(this.settings)
|
||||||
|
assert(client instanceof this.ioredis.Cluster)
|
||||||
|
client.config.should.deep.equal(this.cluster)
|
||||||
|
// need to use expect here because of _.clone in sandbox
|
||||||
|
return expect(client.options).to.deep.equal({
|
||||||
|
redisOptions: this.extraOptions,
|
||||||
|
retry_max_delay: 5000,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
return describe('monkey patch ioredis exec', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
this.callback = sinon.stub()
|
||||||
|
this.results = []
|
||||||
|
this.multiOrig = { exec: sinon.stub().yields(null, this.results) }
|
||||||
|
this.client = { multi: sinon.stub().returns(this.multiOrig) }
|
||||||
|
this.ioredisConstructor.returns(this.client)
|
||||||
|
this.redis.createClient(this.client)
|
||||||
|
return (this.multi = this.client.multi())
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return the old redis format for an array', function () {
|
||||||
|
this.results[0] = [null, 42]
|
||||||
|
this.results[1] = [null, 'foo']
|
||||||
|
this.multi.exec(this.callback)
|
||||||
|
return this.callback.calledWith(null, [42, 'foo']).should.equal(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
return it('should return the old redis format when there is an error', function () {
|
||||||
|
this.results[0] = [null, 42]
|
||||||
|
this.results[1] = ['error', 'foo']
|
||||||
|
this.multi.exec(this.callback)
|
||||||
|
return this.callback.calledWith('error').should.equal(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
Loading…
Reference in a new issue