github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Create script that will ensure data is migrate between redis backends

Browse source
This commit is contained in:
James Allen 2016-06-29 12:57:56 +01:00
parent dbe03e2757
commit fde334acc4
4 changed files with 122 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/document-updater/app/coffee/RedisBackend.coffee
View file

@ -61,6 +61,7 @@ class MultiClient
jobs = @clients.map (client) ->
(cb) ->
client.rclient.exec (error, result) ->
console.log "EXEC [#{client.driver}]"
if client.driver == "ioredis"
# ioredis returns an results like:
# [ [null, 42], [null, "foo"] ]
@ -126,6 +127,7 @@ for command, key_pos of COMMANDS
MultiClient.prototype[command] = (args...) ->
for client in @clients
console.log "COMMAND [#{client.driver}]", command, args
key_builder = args[key_pos]
key = key_builder(client.key_schema)
args_with_key = args.slice(0)

25
services/document-updater/app/coffee/RedisManager.coffee
View file

@ -119,3 +119,28 @@ module.exports = RedisManager =
getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) ->
rclient.smembers keys.docsInProject(project_id: project_id), callback
getAndSetDoc: (doc_id, callback = (error, project_id) ->) ->
multi = rclient.multi()
multi.get keys.docLines(doc_id:doc_id)
multi.get keys.docVersion(doc_id:doc_id)
multi.lrange keys.docOps(doc_id:doc_id), 0, -1
multi.get keys.projectKey(doc_id:doc_id)
multi.exec (error, results = []) ->
return callback(error) if error?
[lines, version, ops, project_id] = results
multi = rclient.multi()
multi.set keys.docLines(doc_id:doc_id), lines
multi.set keys.docVersion(doc_id:doc_id), version
multi.del keys.docOps(doc_id:doc_id)
if ops.length > 0
multi.rpush keys.docOps(doc_id:doc_id), ops...
multi.set keys.projectKey(doc_id:doc_id), project_id
multi.exec (error) ->
return callback(error) if error?
return callback null, project_id
getAndSetProject: (project_id, callback = (error) ->) ->
rclient.smembers keys.docsInProject(project_id: project_id), (error, doc_ids) ->
return callback(error) if error?
rclient.sadd keys.docsInProject(project_id: project_id), doc_ids..., callback

29
services/document-updater/migrate.coffee Normal file
View file

@ -0,0 +1,29 @@
RedisManager = require "./app/coffee/RedisManager"
UpdateManager = require "./app/coffee/UpdateManager"
LockManager = require "./app/coffee/LockManager"
handleErrorInsideLock = (doc_id, lockValue, original_error, callback = (error) ->) ->
LockManager.releaseLock doc_id, lockValue, (lock_error) ->
callback(original_error)
migrateDoc = (doc_id, callback = (error) ->) ->
LockManager.getLock doc_id, (error, lockValue) ->
return callback(error) if error?
RedisManager.getAndSetDoc doc_id, (error, project_id) ->
return handleErrorInsideLock(doc_id, lockValue, error, callback) if error?
RedisManager.getAndSetProject project_id, (error) ->
return handleErrorInsideLock(doc_id, lockValue, error, callback) if error?
LockManager.releaseLock doc_id, lockValue, (error) ->
return callback(error) if error?
UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id, callback
doc_id = process.argv[2]
if !doc_id?
console.log "Usage: coffee migrate.coffee DOC_ID"
process.exit(1)
migrateDoc doc_id, (error) ->
throw error if error?
setTimeout () ->
process.exit(0)
, 200

67
services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee
View file

@ -229,4 +229,69 @@ describe "RedisManager", ->
it "should remove the doc_id from the project set", ->
@rclient.srem
.calledWith("DocsIn:#{@project_id}", @doc_id)
.should.equal true
.should.equal true
describe "getAndSetDoc", ->
beforeEach ->
@rclient.get = sinon.stub()
@rclient.lrange = sinon.stub()
@rclient.del = sinon.stub()
@rclient.set = sinon.stub()
@rclient.rpush = sinon.stub()
@rclient.exec = sinon.stub()
@rclient.exec.yields(null, [
@lines = '["mock","lines"]',
@version = 42,
@doc_ops = ["mock", "doc", "ops"],
@project_id = "mock-project-id"
])
@RedisManager.getAndSetDoc @doc_id, @callback
it "should get the original values", ->
@rclient.get
.calledWith("doclines:#{@doc_id}")
.should.equal true
@rclient.get
.calledWith("DocVersion:#{@doc_id}")
.should.equal true
@rclient.get
.calledWith("ProjectId:#{@doc_id}")
.should.equal true
@rclient.lrange
.calledWith("DocOps:#{@doc_id}", 0, -1)
.should.equal true
it "should set the doclines again", ->
@rclient.set
.calledWith("doclines:#{@doc_id}", @lines)
.should.equal true
it "should set the DocVersion again", ->
@rclient.set
.calledWith("DocVersion:#{@doc_id}", @version)
.should.equal true
it "should set the project id again", ->
@rclient.set
.calledWith("ProjectId:#{@doc_id}", @project_id)
.should.equal true
it "should set the doc ops again", ->
@rclient.del
.calledWith("DocOps:#{@doc_id}")
.should.equal true
@rclient.rpush
.calledWith("DocOps:#{@doc_id}", @doc_ops...)
.should.equal true
describe "getAndSetProject", ->
beforeEach ->
@rclient.smembers = sinon.stub()
@rclient.sadd = sinon.stub()
@rclient.smembers.withArgs("DocsIn:#{@project_id}").yields(null, @doc_ids = ["mock-doc-1", "mock-doc-2"])
@RedisManager.getAndSetProject @project_id, @callback
it "should set the doc ids again", ->
@rclient.sadd
.calledWith("DocsIn:#{@project_id}", @doc_ids...)
.should.equal true