github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #12 from sharelatex/ja-redis-cluster-refactor

Browse source 
Use new redis-sharelatex with support for cluster
This commit is contained in:
James Allen 2017-05-10 15:30:41 +01:00 committed by GitHub
commit b7a07ea1b7
11 changed files with 56 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
services/real-time/app.coffee
View file

@ -51,12 +51,14 @@ io.configure ->
app.get "/status", (req, res, next) ->
res.send "real-time-sharelatex is alive"
redisCheck = redis.activeHealthCheckRedis(Settings.redis.web)
rclient = require("redis-sharelatex").createClient(Settings.redis.realtime)
app.get "/health_check/redis", (req, res, next) ->
if redisCheck.isAlive()
res.send 200
rclient.healthCheck (error) ->
if error?
logger.err {err: error}, "failed redis health check"
res.sendStatus 500
else
res.send 500
res.sendStatus 200
Router = require "./app/js/Router"
Router.configure(app, io, sessionSockets)

36
services/real-time/app/coffee/ConnectedUsersManager.coffee
View file

@ -2,8 +2,8 @@ async = require("async")
Settings = require('settings-sharelatex')
logger = require("logger-sharelatex")
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
rclient = redis.createClient(Settings.redis.realtime)
Keys = Settings.redis.realtime.key_schema
ONE_HOUR_IN_S = 60 * 60
ONE_DAY_IN_S = ONE_HOUR_IN_S * 24
@ -11,10 +11,6 @@ FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4
USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4
buildProjectSetKey = (project_id)-> return "clients_in_project:#{project_id}"
buildUserKey = (project_id, client_id)-> return "connected_user:#{project_id}:#{client_id}"
module.exports =
# Use the same method for when a user connects, and when a user sends a cursor
@ -25,18 +21,18 @@ module.exports =
multi = rclient.multi()
multi.sadd buildProjectSetKey(project_id), client_id
multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S
multi.sadd Keys.clientsInProject({project_id}), client_id
multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S
multi.hset buildUserKey(project_id, client_id), "last_updated_at", Date.now()
multi.hset buildUserKey(project_id, client_id), "user_id", user._id
multi.hset buildUserKey(project_id, client_id), "first_name", user.first_name or ""
multi.hset buildUserKey(project_id, client_id), "last_name", user.last_name or ""
multi.hset buildUserKey(project_id, client_id), "email", user.email or ""
multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()
multi.hset Keys.connectedUser({project_id, client_id}), "user_id", user._id
multi.hset Keys.connectedUser({project_id, client_id}), "first_name", user.first_name or ""
multi.hset Keys.connectedUser({project_id, client_id}), "last_name", user.last_name or ""
multi.hset Keys.connectedUser({project_id, client_id}), "email", user.email or ""
if cursorData?
multi.hset buildUserKey(project_id, client_id), "cursorData", JSON.stringify(cursorData)
multi.expire buildUserKey(project_id, client_id), USER_TIMEOUT_IN_S
multi.hset Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData)
multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S
multi.exec (err)->
if err?
@ -46,14 +42,14 @@ module.exports =
markUserAsDisconnected: (project_id, client_id, callback)->
logger.log project_id:project_id, client_id:client_id, "marking user as disconnected"
multi = rclient.multi()
multi.srem buildProjectSetKey(project_id), client_id
multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S
multi.del buildUserKey(project_id, client_id)
multi.srem Keys.clientsInProject({project_id}), client_id
multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S
multi.del Keys.connectedUser({project_id, client_id})
multi.exec callback
_getConnectedUser: (project_id, client_id, callback)->
rclient.hgetall buildUserKey(project_id, client_id), (err, result)->
rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)->
if !result?
result =
connected : false
@ -71,7 +67,7 @@ module.exports =
getConnectedUsers: (project_id, callback)->
self = @
rclient.smembers buildProjectSetKey(project_id), (err, results)->
rclient.smembers Keys.clientsInProject({project_id}), (err, results)->
return callback(err) if err?
jobs = results.map (client_id)->
(cb)->

2
services/real-time/app/coffee/DocumentUpdaterController.coffee
View file

@ -1,7 +1,7 @@
logger = require "logger-sharelatex"
settings = require 'settings-sharelatex'
redis = require("redis-sharelatex")
rclient = redis.createClient(settings.redis.web)
rclient = redis.createClient(settings.redis.documentupdater)
SafeJsonParse = require "./SafeJsonParse"
MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb

6
services/real-time/app/coffee/DocumentUpdaterManager.coffee
View file

@ -3,8 +3,8 @@ logger = require "logger-sharelatex"
settings = require "settings-sharelatex"
metrics = require("metrics-sharelatex")
redis = require("redis-sharelatex")
rclient = redis.createClient(settings.redis.web)
rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater)
Keys = settings.redis.documentupdater.key_schema
module.exports = DocumentUpdaterManager =
getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) ->
@ -56,7 +56,7 @@ module.exports = DocumentUpdaterManager =
jsonChange = JSON.stringify change
doc_key = "#{project_id}:#{doc_id}"
multi = rclient.multi()
multi.rpush "PendingUpdates:#{doc_id}", jsonChange
multi.rpush Keys.pendingUpdates({doc_id}), jsonChange
multi.sadd "DocsWithPendingUpdates", doc_key
multi.rpush "pending-updates-list", doc_key
multi.exec (error) ->

4
services/real-time/app/coffee/WebsocketLoadBalancer.coffee
View file

@ -2,8 +2,8 @@ Settings = require 'settings-sharelatex'
logger = require 'logger-sharelatex'
redis = require("redis-sharelatex")
SafeJsonParse = require "./SafeJsonParse"
rclientPub = redis.createClient(Settings.redis.web)
rclientSub = redis.createClient(Settings.redis.web)
rclientPub = redis.createClient(Settings.redis.realtime)
rclientSub = redis.createClient(Settings.redis.realtime)
module.exports = WebsocketLoadBalancer =
rclientPub: rclientPub

12
services/real-time/config/settings.defaults.coffee
View file

@ -1,9 +1,19 @@
module.exports =
redis:
web:
realtime:
host: "localhost"
port: "6379"
password: ""
key_schema:
clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}"
connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}"
documentupdater:
host: "localhost"
port: "6379"
password: ""
key_schema:
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
internal:
realTime:

2
services/real-time/package.json
View file

@ -18,7 +18,7 @@
"ioredis": "^2.4.0",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0",
"redis-sharelatex": "0.0.9",
"redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2",
"request": "~2.34.0",
"session.socket.io": "^0.1.6",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",

5
services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee
View file

@ -15,7 +15,10 @@ describe "ConnectedUsersManager", ->
@settings =
redis:
web:{}
realtime:
key_schema:
clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}"
connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}"
@rClient =
auth:->
setex:sinon.stub()

9
services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee
View file

@ -13,10 +13,13 @@ describe "DocumentUpdaterController", ->
@EditorUpdatesController = SandboxedModule.require modulePath, requires:
"logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() }
"settings-sharelatex": @settings =
redis: web: {}
redis:
documentupdater:
key_schema:
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
"redis-sharelatex" :
createClient: ()=>
@rclient = {auth:->}
createClient: () =>
@rclient = {}
"./SafeJsonParse": @SafeJsonParse =
parse: (data, cb) => cb null, JSON.parse(data)

4
services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee
View file

@ -12,7 +12,9 @@ describe 'DocumentUpdaterManager', ->
@version = 42
@settings =
apis: documentupdater: url: "http://doc-updater.example.com"
redis: web: {}
redis: documentupdater:
key_schema:
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
@rclient = {auth:->}
@DocumentUpdaterManager = SandboxedModule.require modulePath, requires:

4
services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee
View file

@ -5,10 +5,10 @@ modulePath = require('path').join __dirname, '../../../app/js/WebsocketLoadBalan
describe "WebsocketLoadBalancer", ->
beforeEach ->
@rclient = {}
@WebsocketLoadBalancer = SandboxedModule.require modulePath, requires:
"redis-sharelatex":
createClient: () ->
auth:->
createClient: () => @rclient
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
"./SafeJsonParse": @SafeJsonParse =
parse: (data, cb) => cb null, JSON.parse(data)