Merge pull request #493 from sharelatex/ja-redis-cluster-refactor

Move all redis end points to be cluster compatible
This commit is contained in:
James Allen 2017-05-10 14:38:37 +01:00 committed by GitHub
commit 274d6f4ac6
22 changed files with 91 additions and 212 deletions

View file

@ -1,7 +1,7 @@
Settings = require "settings-sharelatex"
request = require('request')
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("clsi_cookie")
Cookie = require('cookie')
logger = require "logger-sharelatex"

View file

@ -1,6 +1,6 @@
Settings = require('settings-sharelatex')
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("clsi_recently_compiled")
DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler"
Project = require("../../models/Project").Project
ProjectRootDocManager = require "../Project/ProjectRootDocManager"

View file

@ -5,24 +5,10 @@ _ = require 'underscore'
async = require 'async'
logger = require('logger-sharelatex')
metrics = require('metrics-sharelatex')
redis = require("redis-sharelatex")
rclient = redis.createClient(settings.redis.web)
Project = require("../../models/Project").Project
ProjectLocator = require('../../Features/Project/ProjectLocator')
module.exports = DocumentUpdaterHandler =
queueChange : (project_id, doc_id, change, callback = ()->)->
jsonChange = JSON.stringify change
doc_key = keys.combineProjectIdAndDocId(project_id, doc_id)
multi = rclient.multi()
multi.rpush keys.pendingUpdates(doc_id:doc_id), jsonChange
multi.sadd keys.docsWithPendingUpdates, doc_key
multi.rpush "pending-updates-list", doc_key
multi.exec (error) ->
return callback(error) if error?
callback()
flushProjectToMongo: (project_id, callback = (error) ->)->
logger.log project_id:project_id, "flushing project from document updater"
timer = new metrics.Timer("flushing.mongo.project")

View file

@ -1,14 +1,10 @@
Settings = require 'settings-sharelatex'
redis = require("redis-sharelatex")
rclientPub = redis.createClient(Settings.redis.web)
rclientSub = redis.createClient(Settings.redis.web)
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("realtime")
module.exports = EditorRealTimeController =
rclientPub: rclientPub
rclientSub: rclientSub
emitToRoom: (room_id, message, payload...) ->
@rclientPub.publish "editor-events", JSON.stringify
rclient.publish "editor-events", JSON.stringify
room_id: room_id
message: message
payload: payload

View file

@ -1,8 +1,8 @@
Mocha = require "mocha"
Base = require("mocha/lib/reporters/base")
redis = require("redis-sharelatex")
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("health_check")
settings = require("settings-sharelatex")
redisCheck = redis.activeHealthCheckRedis(settings.redis.web)
logger = require "logger-sharelatex"
domain = require "domain"
@ -31,10 +31,12 @@ module.exports = HealthCheckController =
delete require.cache[path]
checkRedis: (req, res, next)->
if redisCheck.isAlive()
res.sendStatus 200
else
res.sendStatus 500
rclient.healthCheck (error) ->
if error?
logger.err {err: error}, "failed redis health check"
res.sendStatus 500
else
res.sendStatus 200
Reporter = (res) ->
(runner) ->

View file

@ -1,6 +1,6 @@
Settings = require('settings-sharelatex')
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("one_time_token")
crypto = require("crypto")
logger = require("logger-sharelatex")

View file

@ -3,7 +3,6 @@ logger = require("logger-sharelatex")
Async = require('async')
_ = require('underscore')
UserSessionsRedis = require('./UserSessionsRedis')
rclient = UserSessionsRedis.client()
module.exports = UserSessionsManager =

View file

@ -1,21 +1,9 @@
Settings = require 'settings-sharelatex'
redis = require 'redis-sharelatex'
ioredis = require 'ioredis'
logger = require 'logger-sharelatex'
redisSessionsSettings = Settings.redis.websessions or Settings.redis.web
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("websessions")
module.exports = Redis =
client: () ->
if redisSessionsSettings?.cluster?
logger.log {}, "using redis cluster for web sessions"
rclient = new ioredis.Cluster(redisSessionsSettings.cluster)
else
rclient = redis.createClient(redisSessionsSettings)
return rclient
sessionSetKey: (user) ->
if redisSessionsSettings?.cluster?
return "UserSessions:{#{user._id}}"
else
return "UserSessions:#{user._id}"
return "UserSessions:{#{user._id}}"

View file

@ -1,7 +1,7 @@
metrics = require('metrics-sharelatex')
Settings = require('settings-sharelatex')
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
RedisWrapper = require("./RedisWrapper")
rclient = RedisWrapper.client("lock")
logger = require "logger-sharelatex"
module.exports = LockManager =

View file

@ -1,28 +1,14 @@
Settings = require 'settings-sharelatex'
redis = require 'redis-sharelatex'
ioredis = require 'ioredis'
logger = require 'logger-sharelatex'
# A per-feature interface to Redis,
# looks up the feature in `settings.redis`
# and returns an appropriate client.
# Necessary because we don't want to migrate web over
# to redis-cluster all at once.
# TODO: consider merging into `redis-sharelatex`
module.exports = Redis =
# feature = 'websessions' | 'ratelimiter' | ...
client: (feature) ->
redisFeatureSettings = Settings.redis[feature] or Settings.redis.web
if redisFeatureSettings?.cluster?
logger.log {feature}, "creating redis-cluster client"
rclient = new ioredis.Cluster(redisFeatureSettings.cluster)
rclient.__is_redis_cluster = true
else
logger.log {feature}, "creating redis client"
rclient = redis.createClient(redisFeatureSettings)
rclient = redis.createClient(redisFeatureSettings)
return rclient

View file

@ -740,12 +740,6 @@
"from": "double-ended-queue@>=2.1.0-0 <3.0.0",
"resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz"
},
"dtrace-provider": {
"version": "0.2.8",
"from": "dtrace-provider@0.2.8",
"resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz",
"optional": true
},
"each-series": {
"version": "1.0.0",
"from": "each-series@>=1.0.0 <2.0.0",
@ -2742,18 +2736,18 @@
}
},
"redis-sharelatex": {
"version": "0.0.9",
"from": "redis-sharelatex@0.0.9",
"resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-0.0.9.tgz",
"version": "1.0.2",
"from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2",
"resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#143b7eb192675f36d835080e534a4ac4899f918a",
"dependencies": {
"ansi-regex": {
"version": "0.2.1",
"from": "ansi-regex@^0.2.0",
"from": "ansi-regex@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz"
},
"ansi-styles": {
"version": "1.1.0",
"from": "ansi-styles@^1.1.0",
"from": "ansi-styles@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.1.0.tgz"
},
"assertion-error": {
@ -2761,6 +2755,11 @@
"from": "assertion-error@1.0.0",
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz"
},
"async": {
"version": "2.4.0",
"from": "async@>=2.4.0 <3.0.0",
"resolved": "https://registry.npmjs.org/async/-/async-2.4.0.tgz"
},
"chai": {
"version": "1.9.1",
"from": "chai@1.9.1",
@ -2768,7 +2767,7 @@
},
"chalk": {
"version": "0.5.1",
"from": "chalk@~0.5.0",
"from": "chalk@>=0.5.0 <0.6.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz"
},
"coffee-script": {
@ -2793,7 +2792,7 @@
"dependencies": {
"mkdirp": {
"version": "0.5.1",
"from": "mkdirp@^0.5.0",
"from": "mkdirp@>=0.5.0 <0.6.0",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
}
}
@ -2815,8 +2814,13 @@
"dependencies": {
"coffee-script": {
"version": "1.7.1",
"from": "coffee-script@~1.7.0",
"from": "coffee-script@>=1.7.0 <1.8.0",
"resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.7.1.tgz"
},
"lodash": {
"version": "2.4.2",
"from": "lodash@>=2.4.1 <2.5.0",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz"
}
}
},
@ -2827,7 +2831,7 @@
},
"has-ansi": {
"version": "0.1.0",
"from": "has-ansi@^0.1.0",
"from": "has-ansi@>=0.1.0 <0.2.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-0.1.0.tgz"
},
"jade": {
@ -2852,15 +2856,10 @@
"from": "jsonfile@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz"
},
"lodash": {
"version": "2.4.2",
"from": "lodash@~2.4.1",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz"
},
"minimatch": {
"version": "3.0.3",
"version": "3.0.4",
"from": "minimatch@>=3.0.2 <4.0.0",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz"
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz"
},
"mkdirp": {
"version": "0.3.5",
@ -2884,7 +2883,7 @@
},
"minimatch": {
"version": "0.2.14",
"from": "minimatch@~0.2.11",
"from": "minimatch@>=0.2.11 <0.3.0",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz"
}
}
@ -2904,11 +2903,6 @@
"from": "rimraf@>=2.2.8 <3.0.0",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.1.tgz"
},
"samsam": {
"version": "1.1.3",
"from": "samsam@>=1.1.0 <1.2.0",
"resolved": "https://registry.npmjs.org/samsam/-/samsam-1.1.3.tgz"
},
"sandboxed-module": {
"version": "1.0.1",
"from": "sandboxed-module@1.0.1",
@ -2921,12 +2915,12 @@
},
"strip-ansi": {
"version": "0.3.0",
"from": "strip-ansi@^0.3.0",
"from": "strip-ansi@>=0.3.0 <0.4.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz"
},
"supports-color": {
"version": "0.2.0",
"from": "supports-color@^0.2.0",
"from": "supports-color@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz"
},
"underscore": {
@ -3036,6 +3030,11 @@
"resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz",
"optional": true
},
"samsam": {
"version": "1.1.2",
"from": "samsam@1.1.2",
"resolved": "https://registry.npmjs.org/samsam/-/samsam-1.1.2.tgz"
},
"sanitizer": {
"version": "0.1.1",
"from": "sanitizer@0.1.1",

View file

@ -49,7 +49,7 @@
"passport-ldapauth": "^0.6.0",
"passport-local": "^1.0.0",
"redis": "0.10.1",
"redis-sharelatex": "0.0.9",
"redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2",
"request": "^2.69.0",
"requests": "^0.1.7",
"rimraf": "2.2.6",

View file

@ -34,8 +34,8 @@ describe "ClsiCookieManager", ->
ttl:Math.random()
key: "coooookie"
@requires =
"redis-sharelatex" :
createClient: =>
"../../infrastructure/RedisWrapper":
client: =>
@redis
"settings-sharelatex": @settings
"request": @request

View file

@ -15,8 +15,8 @@ describe "CompileManager", ->
@CompileManager = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings =
redis: web: {host: "localhost", port: 42}
"redis-sharelatex":
createClient: () => @rclient = { auth: () -> }
"../../infrastructure/RedisWrapper":
client: () => @rclient = { auth: () -> }
"../DocumentUpdater/DocumentUpdaterHandler": @DocumentUpdaterHandler = {}
"../Project/ProjectRootDocManager": @ProjectRootDocManager = {}
"../../models/Project": Project: @Project = {}

View file

@ -20,10 +20,8 @@ describe 'DocumentUpdaterHandler', ->
@request = {}
@projectEntityHandler = {}
@rclient = {auth:->}
@settings =
apis : documentupdater: url : "http://something.com"
redis:{web:{}}
@handler = SandboxedModule.require modulePath, requires:
'request': defaults:=> return @request
'settings-sharelatex':@settings
@ -31,52 +29,10 @@ describe 'DocumentUpdaterHandler', ->
'../Project/ProjectEntityHandler':@projectEntityHandler
"../../models/Project": Project: @Project={}
'../../Features/Project/ProjectLocator':{}
'redis-sharelatex' : createClient: () => @rclient
"metrics-sharelatex":
Timer:->
done:->
describe 'queueChange', ->
beforeEach ->
@change = {
"action":"removeText",
"range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}},
"text":"e"
}
@rclient.multi = sinon.stub().returns @rclient
@rclient.exec = sinon.stub().callsArg(0)
@rclient.rpush = sinon.stub()
@rclient.sadd = sinon.stub()
@callback = sinon.stub()
describe "successfully", ->
beforeEach ->
@handler.queueChange(@project_id, @doc_id, @change, @callback)
it "should push the change", ->
@rclient.rpush
.calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change))
.should.equal true
it "should notify the doc updater of the change via the pending-updates-list queue", ->
@rclient.rpush
.calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}")
.should.equal true
it "should push the doc id into the pending updates set", ->
@rclient.sadd
.calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}")
.should.equal true
describe "with error connecting to redis during exec", ->
beforeEach ->
@rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong"))
@handler.queueChange(@project_id, @doc_id, @change, @callback)
it "should return an error", ->
@callback.calledWithExactly(sinon.match(Error)).should.equal true
describe 'flushProjectToMongo', ->
beforeEach ->
@callback = sinon.stub()

View file

@ -5,16 +5,13 @@ modulePath = require('path').join __dirname, '../../../../app/js/Features/Editor
describe "EditorRealTimeController", ->
beforeEach ->
@rclient =
publish: sinon.stub()
@EditorRealTimeController = SandboxedModule.require modulePath, requires:
"redis-sharelatex":
createClient: () ->
auth:->
"../../infrastructure/RedisWrapper":
client: () => @rclient
"../../infrastructure/Server" : io: @io = {}
"settings-sharelatex":{redis:{}}
@EditorRealTimeController.rclientPub = publish: sinon.stub()
@EditorRealTimeController.rclientSub =
subscribe: sinon.stub()
on: sinon.stub()
@room_id = "room-id"
@message = "message-to-editor"
@ -25,7 +22,7 @@ describe "EditorRealTimeController", ->
@EditorRealTimeController.emitToRoom(@room_id, @message, @payload...)
it "should publish the message to redis", ->
@EditorRealTimeController.rclientPub.publish
@rclient.publish
.calledWith("editor-events", JSON.stringify(
room_id: @room_id,
message: @message

View file

@ -23,8 +23,8 @@ describe "OneTimeTokenHandler", ->
exec:sinon.stub()
self = @
@OneTimeTokenHandler = SandboxedModule.require modulePath, requires:
"redis-sharelatex" :
createClient: =>
"../../infrastructure/RedisWrapper" :
client: =>
auth:->
multi: -> return self.redisMulti

View file

@ -7,7 +7,7 @@ doc_id = 5678
blockingKey = "Blocking:#{doc_id}"
SandboxedModule = require('sandboxed-module')
describe 'Lock Manager - checking the lock', ()->
describe 'LockManager - checking the lock', ()->
existsStub = sinon.stub()
setStub = sinon.stub()
@ -17,8 +17,8 @@ describe 'Lock Manager - checking the lock', ()->
mocks =
"logger-sharelatex": log:->
"redis-sharelatex":
createClient : ()->
"./RedisWrapper":
client: ()->
auth:->
multi: ->
exists: existsStub

View file

@ -12,8 +12,8 @@ describe 'LockManager - releasing the lock', ()->
mocks =
"logger-sharelatex": log:->
"redis-sharelatex":
createClient : ()->
"./RedisWrapper":
client: ()->
auth:->
del:deleteStub

View file

@ -9,8 +9,8 @@ describe 'LockManager - getting the lock', ->
beforeEach ->
@LockManager = SandboxedModule.require modulePath, requires:
"logger-sharelatex": log:->
"redis-sharelatex":
createClient : () =>
"./RedisWrapper":
client: ()->
auth:->
"settings-sharelatex":{redis:{}}
"metrics-sharelatex": inc:->

View file

@ -9,8 +9,8 @@ describe 'LockManager - trying the lock', ->
beforeEach ->
@LockManager = SandboxedModule.require modulePath, requires:
"logger-sharelatex": log:->
"redis-sharelatex":
createClient : () =>
"./RedisWrapper":
client: () =>
auth:->
set: @set = sinon.stub()
"settings-sharelatex":{redis:{}}

View file

@ -9,58 +9,28 @@ SandboxedModule = require('sandboxed-module')
describe 'RedisWrapper', ->
beforeEach ->
@featureName = 'somefeature'
@settings =
redis:
web:
port:"1234"
host:"somewhere"
password: "password"
somefeature: {}
@normalRedisInstance =
thisIsANormalRedisInstance: true
n: 1
@clusterRedisInstance =
thisIsAClusterRedisInstance: true
n: 2
@settings = { redis: {} }
@redis =
createClient: sinon.stub().returns(@normalRedisInstance)
@ioredis =
Cluster: sinon.stub().returns(@clusterRedisInstance)
@logger = {log: sinon.stub()}
createClient: sinon.stub()
@RedisWrapper = SandboxedModule.require modulePath, requires:
'logger-sharelatex': @logger
'settings-sharelatex': @settings
'redis-sharelatex': @redis
'ioredis': @ioredis
describe 'client', ->
it "should use the feature settings if present", ->
@settings.redis =
my_feature:
port:"23456"
host:"otherhost"
password: "banana"
@RedisWrapper.client("my_feature")
@redis.createClient.calledWith(@settings.redis.my_feature).should.equal true
beforeEach ->
@call = () =>
@RedisWrapper.client(@featureName)
describe 'when feature uses cluster', ->
beforeEach ->
@settings.redis.somefeature =
cluster: [1, 2, 3]
it 'should return a cluster client', ->
client = @call()
expect(client).to.equal @clusterRedisInstance
expect(client.__is_redis_cluster).to.equal true
describe 'when feature uses normal redis', ->
beforeEach ->
@settings.redis.somefeature =
port:"1234"
host:"somewhere"
password: "password"
it 'should return a regular redis client', ->
client = @call()
expect(client).to.equal @normalRedisInstance
expect(client.__is_redis_cluster).to.equal undefined
it "should use the web settings if feature not present", ->
@settings.redis =
web:
port:"43"
host:"otherhost"
password: "banana"
@RedisWrapper.client("my_feature")
@redis.createClient.calledWith(@settings.redis.web).should.equal true