diff --git a/services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee b/services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee index c237f212c0..c19c5e02ea 100644 --- a/services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee @@ -1,7 +1,7 @@ Settings = require "settings-sharelatex" request = require('request') -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("clsi_cookie") Cookie = require('cookie') logger = require "logger-sharelatex" diff --git a/services/web/app/coffee/Features/Compile/CompileManager.coffee b/services/web/app/coffee/Features/Compile/CompileManager.coffee index 3bfbf8df7d..ad8a2459ec 100755 --- a/services/web/app/coffee/Features/Compile/CompileManager.coffee +++ b/services/web/app/coffee/Features/Compile/CompileManager.coffee @@ -1,6 +1,6 @@ Settings = require('settings-sharelatex') -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("clsi_recently_compiled") DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler" Project = require("../../models/Project").Project ProjectRootDocManager = require "../Project/ProjectRootDocManager" diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index 595fe07971..805f063808 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -5,24 +5,10 @@ _ = require 'underscore' async = require 'async' logger = require('logger-sharelatex') metrics = require('metrics-sharelatex') -redis = require("redis-sharelatex") -rclient = redis.createClient(settings.redis.web) Project = require("../../models/Project").Project ProjectLocator = require('../../Features/Project/ProjectLocator') module.exports = DocumentUpdaterHandler = - - queueChange : (project_id, doc_id, change, callback = ()->)-> - jsonChange = JSON.stringify change - doc_key = keys.combineProjectIdAndDocId(project_id, doc_id) - multi = rclient.multi() - multi.rpush keys.pendingUpdates(doc_id:doc_id), jsonChange - multi.sadd keys.docsWithPendingUpdates, doc_key - multi.rpush "pending-updates-list", doc_key - multi.exec (error) -> - return callback(error) if error? - callback() - flushProjectToMongo: (project_id, callback = (error) ->)-> logger.log project_id:project_id, "flushing project from document updater" timer = new metrics.Timer("flushing.mongo.project") diff --git a/services/web/app/coffee/Features/Editor/EditorRealTimeController.coffee b/services/web/app/coffee/Features/Editor/EditorRealTimeController.coffee index bd4f6af8e8..5062533cbb 100644 --- a/services/web/app/coffee/Features/Editor/EditorRealTimeController.coffee +++ b/services/web/app/coffee/Features/Editor/EditorRealTimeController.coffee @@ -1,14 +1,10 @@ Settings = require 'settings-sharelatex' -redis = require("redis-sharelatex") -rclientPub = redis.createClient(Settings.redis.web) -rclientSub = redis.createClient(Settings.redis.web) +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("realtime") module.exports = EditorRealTimeController = - rclientPub: rclientPub - rclientSub: rclientSub - emitToRoom: (room_id, message, payload...) -> - @rclientPub.publish "editor-events", JSON.stringify + rclient.publish "editor-events", JSON.stringify room_id: room_id message: message payload: payload diff --git a/services/web/app/coffee/Features/HealthCheck/HealthCheckController.coffee b/services/web/app/coffee/Features/HealthCheck/HealthCheckController.coffee index 51acdbd2f0..544933b5fc 100644 --- a/services/web/app/coffee/Features/HealthCheck/HealthCheckController.coffee +++ b/services/web/app/coffee/Features/HealthCheck/HealthCheckController.coffee @@ -1,8 +1,8 @@ Mocha = require "mocha" Base = require("mocha/lib/reporters/base") -redis = require("redis-sharelatex") +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("health_check") settings = require("settings-sharelatex") -redisCheck = redis.activeHealthCheckRedis(settings.redis.web) logger = require "logger-sharelatex" domain = require "domain" @@ -31,10 +31,12 @@ module.exports = HealthCheckController = delete require.cache[path] checkRedis: (req, res, next)-> - if redisCheck.isAlive() - res.sendStatus 200 - else - res.sendStatus 500 + rclient.healthCheck (error) -> + if error? + logger.err {err: error}, "failed redis health check" + res.sendStatus 500 + else + res.sendStatus 200 Reporter = (res) -> (runner) -> diff --git a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee index b84e1c9b33..3824703efb 100644 --- a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee +++ b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee @@ -1,6 +1,6 @@ Settings = require('settings-sharelatex') -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("one_time_token") crypto = require("crypto") logger = require("logger-sharelatex") diff --git a/services/web/app/coffee/Features/User/UserSessionsManager.coffee b/services/web/app/coffee/Features/User/UserSessionsManager.coffee index 2cd3b17e7f..273c7fe4a9 100644 --- a/services/web/app/coffee/Features/User/UserSessionsManager.coffee +++ b/services/web/app/coffee/Features/User/UserSessionsManager.coffee @@ -3,7 +3,6 @@ logger = require("logger-sharelatex") Async = require('async') _ = require('underscore') UserSessionsRedis = require('./UserSessionsRedis') - rclient = UserSessionsRedis.client() module.exports = UserSessionsManager = diff --git a/services/web/app/coffee/Features/User/UserSessionsRedis.coffee b/services/web/app/coffee/Features/User/UserSessionsRedis.coffee index 89ab7ed192..0c460b4604 100644 --- a/services/web/app/coffee/Features/User/UserSessionsRedis.coffee +++ b/services/web/app/coffee/Features/User/UserSessionsRedis.coffee @@ -1,21 +1,9 @@ -Settings = require 'settings-sharelatex' -redis = require 'redis-sharelatex' -ioredis = require 'ioredis' -logger = require 'logger-sharelatex' - -redisSessionsSettings = Settings.redis.websessions or Settings.redis.web +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("websessions") module.exports = Redis = client: () -> - if redisSessionsSettings?.cluster? - logger.log {}, "using redis cluster for web sessions" - rclient = new ioredis.Cluster(redisSessionsSettings.cluster) - else - rclient = redis.createClient(redisSessionsSettings) return rclient sessionSetKey: (user) -> - if redisSessionsSettings?.cluster? - return "UserSessions:{#{user._id}}" - else - return "UserSessions:#{user._id}" + return "UserSessions:{#{user._id}}" diff --git a/services/web/app/coffee/infrastructure/LockManager.coffee b/services/web/app/coffee/infrastructure/LockManager.coffee index 3e40f9d9dc..370e4c09c9 100644 --- a/services/web/app/coffee/infrastructure/LockManager.coffee +++ b/services/web/app/coffee/infrastructure/LockManager.coffee @@ -1,7 +1,7 @@ metrics = require('metrics-sharelatex') Settings = require('settings-sharelatex') -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) +RedisWrapper = require("./RedisWrapper") +rclient = RedisWrapper.client("lock") logger = require "logger-sharelatex" module.exports = LockManager = diff --git a/services/web/app/coffee/infrastructure/RedisWrapper.coffee b/services/web/app/coffee/infrastructure/RedisWrapper.coffee index 5d8b5836b5..523785427e 100644 --- a/services/web/app/coffee/infrastructure/RedisWrapper.coffee +++ b/services/web/app/coffee/infrastructure/RedisWrapper.coffee @@ -1,28 +1,14 @@ Settings = require 'settings-sharelatex' redis = require 'redis-sharelatex' -ioredis = require 'ioredis' -logger = require 'logger-sharelatex' - # A per-feature interface to Redis, # looks up the feature in `settings.redis` # and returns an appropriate client. # Necessary because we don't want to migrate web over # to redis-cluster all at once. - -# TODO: consider merging into `redis-sharelatex` - - module.exports = Redis = - # feature = 'websessions' | 'ratelimiter' | ... client: (feature) -> redisFeatureSettings = Settings.redis[feature] or Settings.redis.web - if redisFeatureSettings?.cluster? - logger.log {feature}, "creating redis-cluster client" - rclient = new ioredis.Cluster(redisFeatureSettings.cluster) - rclient.__is_redis_cluster = true - else - logger.log {feature}, "creating redis client" - rclient = redis.createClient(redisFeatureSettings) + rclient = redis.createClient(redisFeatureSettings) return rclient diff --git a/services/web/test/UnitTests/coffee/Compile/ClsiCookieManagerTests.coffee b/services/web/test/UnitTests/coffee/Compile/ClsiCookieManagerTests.coffee index 01d4ad0002..c54f405bb9 100644 --- a/services/web/test/UnitTests/coffee/Compile/ClsiCookieManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Compile/ClsiCookieManagerTests.coffee @@ -34,8 +34,8 @@ describe "ClsiCookieManager", -> ttl:Math.random() key: "coooookie" @requires = - "redis-sharelatex" : - createClient: => + "../../infrastructure/RedisWrapper": + client: => @redis "settings-sharelatex": @settings "request": @request diff --git a/services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee b/services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee index de56443707..3964acea41 100644 --- a/services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee @@ -15,8 +15,8 @@ describe "CompileManager", -> @CompileManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = redis: web: {host: "localhost", port: 42} - "redis-sharelatex": - createClient: () => @rclient = { auth: () -> } + "../../infrastructure/RedisWrapper": + client: () => @rclient = { auth: () -> } "../DocumentUpdater/DocumentUpdaterHandler": @DocumentUpdaterHandler = {} "../Project/ProjectRootDocManager": @ProjectRootDocManager = {} "../../models/Project": Project: @Project = {} diff --git a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee index f5519ffa68..ee92d00ae3 100644 --- a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee @@ -20,10 +20,8 @@ describe 'DocumentUpdaterHandler', -> @request = {} @projectEntityHandler = {} - @rclient = {auth:->} @settings = apis : documentupdater: url : "http://something.com" - redis:{web:{}} @handler = SandboxedModule.require modulePath, requires: 'request': defaults:=> return @request 'settings-sharelatex':@settings @@ -31,52 +29,10 @@ describe 'DocumentUpdaterHandler', -> '../Project/ProjectEntityHandler':@projectEntityHandler "../../models/Project": Project: @Project={} '../../Features/Project/ProjectLocator':{} - 'redis-sharelatex' : createClient: () => @rclient "metrics-sharelatex": Timer:-> done:-> - describe 'queueChange', -> - beforeEach -> - @change = { - "action":"removeText", - "range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}}, - "text":"e" - } - @rclient.multi = sinon.stub().returns @rclient - @rclient.exec = sinon.stub().callsArg(0) - @rclient.rpush = sinon.stub() - @rclient.sadd = sinon.stub() - @callback = sinon.stub() - - describe "successfully", -> - beforeEach -> - @handler.queueChange(@project_id, @doc_id, @change, @callback) - - it "should push the change", -> - @rclient.rpush - .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change)) - .should.equal true - - it "should notify the doc updater of the change via the pending-updates-list queue", -> - @rclient.rpush - .calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}") - .should.equal true - - it "should push the doc id into the pending updates set", -> - @rclient.sadd - .calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}") - .should.equal true - - describe "with error connecting to redis during exec", -> - beforeEach -> - @rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong")) - @handler.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true - - describe 'flushProjectToMongo', -> beforeEach -> @callback = sinon.stub() diff --git a/services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee b/services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee index bbad9b1ae5..a06bd032dc 100644 --- a/services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee @@ -5,16 +5,13 @@ modulePath = require('path').join __dirname, '../../../../app/js/Features/Editor describe "EditorRealTimeController", -> beforeEach -> + @rclient = + publish: sinon.stub() @EditorRealTimeController = SandboxedModule.require modulePath, requires: - "redis-sharelatex": - createClient: () -> - auth:-> + "../../infrastructure/RedisWrapper": + client: () => @rclient "../../infrastructure/Server" : io: @io = {} "settings-sharelatex":{redis:{}} - @EditorRealTimeController.rclientPub = publish: sinon.stub() - @EditorRealTimeController.rclientSub = - subscribe: sinon.stub() - on: sinon.stub() @room_id = "room-id" @message = "message-to-editor" @@ -25,7 +22,7 @@ describe "EditorRealTimeController", -> @EditorRealTimeController.emitToRoom(@room_id, @message, @payload...) it "should publish the message to redis", -> - @EditorRealTimeController.rclientPub.publish + @rclient.publish .calledWith("editor-events", JSON.stringify( room_id: @room_id, message: @message diff --git a/services/web/test/UnitTests/coffee/Security/OneTimeTokenHandlerTests.coffee b/services/web/test/UnitTests/coffee/Security/OneTimeTokenHandlerTests.coffee index a6305db27c..046acaa720 100644 --- a/services/web/test/UnitTests/coffee/Security/OneTimeTokenHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/Security/OneTimeTokenHandlerTests.coffee @@ -23,8 +23,8 @@ describe "OneTimeTokenHandler", -> exec:sinon.stub() self = @ @OneTimeTokenHandler = SandboxedModule.require modulePath, requires: - "redis-sharelatex" : - createClient: => + "../../infrastructure/RedisWrapper" : + client: => auth:-> multi: -> return self.redisMulti diff --git a/services/web/test/UnitTests/coffee/infrastructure/LockManager/CheckingTheLock.coffee b/services/web/test/UnitTests/coffee/infrastructure/LockManager/CheckingTheLock.coffee index 3f56c346ac..8926382f8b 100644 --- a/services/web/test/UnitTests/coffee/infrastructure/LockManager/CheckingTheLock.coffee +++ b/services/web/test/UnitTests/coffee/infrastructure/LockManager/CheckingTheLock.coffee @@ -7,7 +7,7 @@ doc_id = 5678 blockingKey = "Blocking:#{doc_id}" SandboxedModule = require('sandboxed-module') -describe 'Lock Manager - checking the lock', ()-> +describe 'LockManager - checking the lock', ()-> existsStub = sinon.stub() setStub = sinon.stub() @@ -17,8 +17,8 @@ describe 'Lock Manager - checking the lock', ()-> mocks = "logger-sharelatex": log:-> - "redis-sharelatex": - createClient : ()-> + "./RedisWrapper": + client: ()-> auth:-> multi: -> exists: existsStub diff --git a/services/web/test/UnitTests/coffee/infrastructure/LockManager/ReleasingTheLock.coffee b/services/web/test/UnitTests/coffee/infrastructure/LockManager/ReleasingTheLock.coffee index 8eb0f4d75c..fa99b87739 100644 --- a/services/web/test/UnitTests/coffee/infrastructure/LockManager/ReleasingTheLock.coffee +++ b/services/web/test/UnitTests/coffee/infrastructure/LockManager/ReleasingTheLock.coffee @@ -12,8 +12,8 @@ describe 'LockManager - releasing the lock', ()-> mocks = "logger-sharelatex": log:-> - "redis-sharelatex": - createClient : ()-> + "./RedisWrapper": + client: ()-> auth:-> del:deleteStub diff --git a/services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee b/services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee index ec0d0d4950..16ae0a8b8c 100644 --- a/services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee +++ b/services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee @@ -9,8 +9,8 @@ describe 'LockManager - getting the lock', -> beforeEach -> @LockManager = SandboxedModule.require modulePath, requires: "logger-sharelatex": log:-> - "redis-sharelatex": - createClient : () => + "./RedisWrapper": + client: ()-> auth:-> "settings-sharelatex":{redis:{}} "metrics-sharelatex": inc:-> diff --git a/services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee b/services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee index 98f624c70b..a227ac60bd 100644 --- a/services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee +++ b/services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee @@ -9,8 +9,8 @@ describe 'LockManager - trying the lock', -> beforeEach -> @LockManager = SandboxedModule.require modulePath, requires: "logger-sharelatex": log:-> - "redis-sharelatex": - createClient : () => + "./RedisWrapper": + client: () => auth:-> set: @set = sinon.stub() "settings-sharelatex":{redis:{}} diff --git a/services/web/test/UnitTests/coffee/infrastructure/RedisWrapperTests.coffee b/services/web/test/UnitTests/coffee/infrastructure/RedisWrapperTests.coffee index 83ea202dcd..4837711f56 100644 --- a/services/web/test/UnitTests/coffee/infrastructure/RedisWrapperTests.coffee +++ b/services/web/test/UnitTests/coffee/infrastructure/RedisWrapperTests.coffee @@ -9,58 +9,28 @@ SandboxedModule = require('sandboxed-module') describe 'RedisWrapper', -> beforeEach -> - @featureName = 'somefeature' - @settings = - redis: - web: - port:"1234" - host:"somewhere" - password: "password" - somefeature: {} - @normalRedisInstance = - thisIsANormalRedisInstance: true - n: 1 - @clusterRedisInstance = - thisIsAClusterRedisInstance: true - n: 2 + @settings = { redis: {} } @redis = - createClient: sinon.stub().returns(@normalRedisInstance) - @ioredis = - Cluster: sinon.stub().returns(@clusterRedisInstance) - @logger = {log: sinon.stub()} - + createClient: sinon.stub() @RedisWrapper = SandboxedModule.require modulePath, requires: - 'logger-sharelatex': @logger 'settings-sharelatex': @settings 'redis-sharelatex': @redis - 'ioredis': @ioredis describe 'client', -> + it "should use the feature settings if present", -> + @settings.redis = + my_feature: + port:"23456" + host:"otherhost" + password: "banana" + @RedisWrapper.client("my_feature") + @redis.createClient.calledWith(@settings.redis.my_feature).should.equal true - beforeEach -> - @call = () => - @RedisWrapper.client(@featureName) - - describe 'when feature uses cluster', -> - - beforeEach -> - @settings.redis.somefeature = - cluster: [1, 2, 3] - - it 'should return a cluster client', -> - client = @call() - expect(client).to.equal @clusterRedisInstance - expect(client.__is_redis_cluster).to.equal true - - describe 'when feature uses normal redis', -> - - beforeEach -> - @settings.redis.somefeature = - port:"1234" - host:"somewhere" - password: "password" - - it 'should return a regular redis client', -> - client = @call() - expect(client).to.equal @normalRedisInstance - expect(client.__is_redis_cluster).to.equal undefined + it "should use the web settings if feature not present", -> + @settings.redis = + web: + port:"43" + host:"otherhost" + password: "banana" + @RedisWrapper.client("my_feature") + @redis.createClient.calledWith(@settings.redis.web).should.equal true