mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Create script that will ensure data is migrate between redis backends
This commit is contained in:
parent
dbe03e2757
commit
fde334acc4
4 changed files with 122 additions and 1 deletions
|
@ -61,6 +61,7 @@ class MultiClient
|
|||
jobs = @clients.map (client) ->
|
||||
(cb) ->
|
||||
client.rclient.exec (error, result) ->
|
||||
console.log "EXEC [#{client.driver}]"
|
||||
if client.driver == "ioredis"
|
||||
# ioredis returns an results like:
|
||||
# [ [null, 42], [null, "foo"] ]
|
||||
|
@ -126,6 +127,7 @@ for command, key_pos of COMMANDS
|
|||
|
||||
MultiClient.prototype[command] = (args...) ->
|
||||
for client in @clients
|
||||
console.log "COMMAND [#{client.driver}]", command, args
|
||||
key_builder = args[key_pos]
|
||||
key = key_builder(client.key_schema)
|
||||
args_with_key = args.slice(0)
|
||||
|
|
|
@ -119,3 +119,28 @@ module.exports = RedisManager =
|
|||
|
||||
getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) ->
|
||||
rclient.smembers keys.docsInProject(project_id: project_id), callback
|
||||
|
||||
getAndSetDoc: (doc_id, callback = (error, project_id) ->) ->
|
||||
multi = rclient.multi()
|
||||
multi.get keys.docLines(doc_id:doc_id)
|
||||
multi.get keys.docVersion(doc_id:doc_id)
|
||||
multi.lrange keys.docOps(doc_id:doc_id), 0, -1
|
||||
multi.get keys.projectKey(doc_id:doc_id)
|
||||
multi.exec (error, results = []) ->
|
||||
return callback(error) if error?
|
||||
[lines, version, ops, project_id] = results
|
||||
multi = rclient.multi()
|
||||
multi.set keys.docLines(doc_id:doc_id), lines
|
||||
multi.set keys.docVersion(doc_id:doc_id), version
|
||||
multi.del keys.docOps(doc_id:doc_id)
|
||||
if ops.length > 0
|
||||
multi.rpush keys.docOps(doc_id:doc_id), ops...
|
||||
multi.set keys.projectKey(doc_id:doc_id), project_id
|
||||
multi.exec (error) ->
|
||||
return callback(error) if error?
|
||||
return callback null, project_id
|
||||
|
||||
getAndSetProject: (project_id, callback = (error) ->) ->
|
||||
rclient.smembers keys.docsInProject(project_id: project_id), (error, doc_ids) ->
|
||||
return callback(error) if error?
|
||||
rclient.sadd keys.docsInProject(project_id: project_id), doc_ids..., callback
|
||||
|
|
29
services/document-updater/migrate.coffee
Normal file
29
services/document-updater/migrate.coffee
Normal file
|
@ -0,0 +1,29 @@
|
|||
RedisManager = require "./app/coffee/RedisManager"
|
||||
UpdateManager = require "./app/coffee/UpdateManager"
|
||||
LockManager = require "./app/coffee/LockManager"
|
||||
|
||||
handleErrorInsideLock = (doc_id, lockValue, original_error, callback = (error) ->) ->
|
||||
LockManager.releaseLock doc_id, lockValue, (lock_error) ->
|
||||
callback(original_error)
|
||||
|
||||
migrateDoc = (doc_id, callback = (error) ->) ->
|
||||
LockManager.getLock doc_id, (error, lockValue) ->
|
||||
return callback(error) if error?
|
||||
RedisManager.getAndSetDoc doc_id, (error, project_id) ->
|
||||
return handleErrorInsideLock(doc_id, lockValue, error, callback) if error?
|
||||
RedisManager.getAndSetProject project_id, (error) ->
|
||||
return handleErrorInsideLock(doc_id, lockValue, error, callback) if error?
|
||||
LockManager.releaseLock doc_id, lockValue, (error) ->
|
||||
return callback(error) if error?
|
||||
UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id, callback
|
||||
|
||||
doc_id = process.argv[2]
|
||||
if !doc_id?
|
||||
console.log "Usage: coffee migrate.coffee DOC_ID"
|
||||
process.exit(1)
|
||||
|
||||
migrateDoc doc_id, (error) ->
|
||||
throw error if error?
|
||||
setTimeout () ->
|
||||
process.exit(0)
|
||||
, 200
|
|
@ -229,4 +229,69 @@ describe "RedisManager", ->
|
|||
it "should remove the doc_id from the project set", ->
|
||||
@rclient.srem
|
||||
.calledWith("DocsIn:#{@project_id}", @doc_id)
|
||||
.should.equal true
|
||||
.should.equal true
|
||||
|
||||
describe "getAndSetDoc", ->
|
||||
beforeEach ->
|
||||
@rclient.get = sinon.stub()
|
||||
@rclient.lrange = sinon.stub()
|
||||
@rclient.del = sinon.stub()
|
||||
@rclient.set = sinon.stub()
|
||||
@rclient.rpush = sinon.stub()
|
||||
@rclient.exec = sinon.stub()
|
||||
@rclient.exec.yields(null, [
|
||||
@lines = '["mock","lines"]',
|
||||
@version = 42,
|
||||
@doc_ops = ["mock", "doc", "ops"],
|
||||
@project_id = "mock-project-id"
|
||||
])
|
||||
@RedisManager.getAndSetDoc @doc_id, @callback
|
||||
|
||||
it "should get the original values", ->
|
||||
@rclient.get
|
||||
.calledWith("doclines:#{@doc_id}")
|
||||
.should.equal true
|
||||
@rclient.get
|
||||
.calledWith("DocVersion:#{@doc_id}")
|
||||
.should.equal true
|
||||
@rclient.get
|
||||
.calledWith("ProjectId:#{@doc_id}")
|
||||
.should.equal true
|
||||
@rclient.lrange
|
||||
.calledWith("DocOps:#{@doc_id}", 0, -1)
|
||||
.should.equal true
|
||||
|
||||
it "should set the doclines again", ->
|
||||
@rclient.set
|
||||
.calledWith("doclines:#{@doc_id}", @lines)
|
||||
.should.equal true
|
||||
|
||||
it "should set the DocVersion again", ->
|
||||
@rclient.set
|
||||
.calledWith("DocVersion:#{@doc_id}", @version)
|
||||
.should.equal true
|
||||
|
||||
it "should set the project id again", ->
|
||||
@rclient.set
|
||||
.calledWith("ProjectId:#{@doc_id}", @project_id)
|
||||
.should.equal true
|
||||
|
||||
it "should set the doc ops again", ->
|
||||
@rclient.del
|
||||
.calledWith("DocOps:#{@doc_id}")
|
||||
.should.equal true
|
||||
@rclient.rpush
|
||||
.calledWith("DocOps:#{@doc_id}", @doc_ops...)
|
||||
.should.equal true
|
||||
|
||||
describe "getAndSetProject", ->
|
||||
beforeEach ->
|
||||
@rclient.smembers = sinon.stub()
|
||||
@rclient.sadd = sinon.stub()
|
||||
@rclient.smembers.withArgs("DocsIn:#{@project_id}").yields(null, @doc_ids = ["mock-doc-1", "mock-doc-2"])
|
||||
@RedisManager.getAndSetProject @project_id, @callback
|
||||
|
||||
it "should set the doc ids again", ->
|
||||
@rclient.sadd
|
||||
.calledWith("DocsIn:#{@project_id}", @doc_ids...)
|
||||
.should.equal true
|
||||
|
|
Loading…
Reference in a new issue