github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

cache projectHistoryId with doc in Redis

Browse source
This commit is contained in:
Hayden Faulds 2018-04-11 11:03:20 +01:00
parent dffc0a42c3
commit fb1852a593
9 changed files with 80 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
services/document-updater/app/coffee/DocumentManager.coffee
View file

@ -13,39 +13,39 @@ async = require "async"
MAX_UNFLUSHED_AGE = 300 * 1000 # 5 mins, document should be flushed to mongo this time after a change MAX_UNFLUSHED_AGE = 300 * 1000 # 5 mins, document should be flushed to mongo this time after a change
module.exports = DocumentManager = module.exports = DocumentManager =
getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname, unflushedTime, alreadyLoaded) ->) -> getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname, projectHistoryId, unflushedTime, alreadyLoaded) ->) ->
timer = new Metrics.Timer("docManager.getDoc") timer = new Metrics.Timer("docManager.getDoc")
callback = (args...) -> callback = (args...) ->
timer.done() timer.done()
_callback(args...) _callback(args...)
RedisManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, unflushedTime) -> RedisManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, projectHistoryId, unflushedTime) ->
return callback(error) if error? return callback(error) if error?
if !lines? or !version? if !lines? or !version?
logger.log {project_id, doc_id}, "doc not in redis so getting from persistence API" logger.log {project_id, doc_id}, "doc not in redis so getting from persistence API"
PersistenceManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname) -> PersistenceManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, projectHistoryId) ->
return callback(error) if error? return callback(error) if error?
logger.log {project_id, doc_id, lines, version, pathname}, "got doc from persistence API" logger.log {project_id, doc_id, lines, version, pathname, projectHistoryId}, "got doc from persistence API"
RedisManager.putDocInMemory project_id, doc_id, lines, version, ranges, pathname, (error) -> RedisManager.putDocInMemory project_id, doc_id, lines, version, ranges, pathname, projectHistoryId, (error) ->
return callback(error) if error? return callback(error) if error?
callback null, lines, version, ranges, pathname, null, false callback null, lines, version, ranges, pathname, projectHistoryId, null, false
else else
callback null, lines, version, ranges, pathname, unflushedTime, true callback null, lines, version, ranges, pathname, projectHistoryId, unflushedTime, true
getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, ops, ranges, pathname) ->) -> getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, ops, ranges, pathname, projectHistoryId) ->) ->
timer = new Metrics.Timer("docManager.getDocAndRecentOps") timer = new Metrics.Timer("docManager.getDocAndRecentOps")
callback = (args...) -> callback = (args...) ->
timer.done() timer.done()
_callback(args...) _callback(args...)
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname) -> DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, projectHistoryId) ->
return callback(error) if error? return callback(error) if error?
if fromVersion == -1 if fromVersion == -1
callback null, lines, version, [], ranges, pathname callback null, lines, version, [], ranges, pathname, projectHistoryId
else else
RedisManager.getPreviousDocOps doc_id, fromVersion, version, (error, ops) -> RedisManager.getPreviousDocOps doc_id, fromVersion, version, (error, ops) ->
return callback(error) if error? return callback(error) if error?
callback null, lines, version, ops, ranges, pathname callback null, lines, version, ops, ranges, pathname, projectHistoryId
setDoc: (project_id, doc_id, newLines, source, user_id, undoing, _callback = (error) ->) -> setDoc: (project_id, doc_id, newLines, source, user_id, undoing, _callback = (error) ->) ->
timer = new Metrics.Timer("docManager.setDoc") timer = new Metrics.Timer("docManager.setDoc")
@ -57,7 +57,7 @@ module.exports = DocumentManager =
return callback(new Error("No lines were provided to setDoc")) return callback(new Error("No lines were provided to setDoc"))
UpdateManager = require "./UpdateManager" UpdateManager = require "./UpdateManager"
DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, ranges, pathname, unflushedTime, alreadyLoaded) -> DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, ranges, pathname, projectHistoryId, unflushedTime, alreadyLoaded) ->
return callback(error) if error? return callback(error) if error?
if oldLines? and oldLines.length > 0 and oldLines[0].text? if oldLines? and oldLines.length > 0 and oldLines[0].text?
@ -170,7 +170,7 @@ module.exports = DocumentManager =
RedisManager.renameDoc project_id, doc_id, user_id, update, callback RedisManager.renameDoc project_id, doc_id, user_id, update, callback
getDocAndFlushIfOld: (project_id, doc_id, callback = (error, doc) ->) -> getDocAndFlushIfOld: (project_id, doc_id, callback = (error, doc) ->) ->
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, unflushedTime, alreadyLoaded) -> DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, projectHistoryId, unflushedTime, alreadyLoaded) ->
return callback(error) if error? return callback(error) if error?
# if doc was already loaded see if it needs to be flushed # if doc was already loaded see if it needs to be flushed
if alreadyLoaded and unflushedTime? and (Date.now() - unflushedTime) > MAX_UNFLUSHED_AGE if alreadyLoaded and unflushedTime? and (Date.now() - unflushedTime) > MAX_UNFLUSHED_AGE
@ -195,7 +195,7 @@ module.exports = DocumentManager =
UpdateManager = require "./UpdateManager" UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.getDoc, project_id, doc_id, callback UpdateManager.lockUpdatesAndDo DocumentManager.getDoc, project_id, doc_id, callback
getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version, ops, ranges, pathname) ->) -> getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version, ops, ranges, pathname, projectHistoryId) ->) ->
UpdateManager = require "./UpdateManager" UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.getDocAndRecentOps, project_id, doc_id, fromVersion, callback UpdateManager.lockUpdatesAndDo DocumentManager.getDocAndRecentOps, project_id, doc_id, fromVersion, callback

4
services/document-updater/app/coffee/PersistenceManager.coffee
View file

@ -13,7 +13,7 @@ request = (require("requestretry")).defaults({
MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds
module.exports = PersistenceManager = module.exports = PersistenceManager =
getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname) ->) -> getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname, projectHistoryId) ->) ->
timer = new Metrics.Timer("persistenceManager.getDoc") timer = new Metrics.Timer("persistenceManager.getDoc")
callback = (args...) -> callback = (args...) ->
timer.done() timer.done()
@ -44,7 +44,7 @@ module.exports = PersistenceManager =
return callback(new Error("web API response had no valid doc version")) return callback(new Error("web API response had no valid doc version"))
if !body.pathname? if !body.pathname?
return callback(new Error("web API response had no valid doc pathname")) return callback(new Error("web API response had no valid doc pathname"))
return callback null, body.lines, body.version, body.ranges, body.pathname return callback null, body.lines, body.version, body.ranges, body.pathname, body.projectHistoryId
else if res.statusCode == 404 else if res.statusCode == 404
return callback(new Errors.NotFoundError("doc not not found: #{url}")) return callback(new Errors.NotFoundError("doc not not found: #{url}"))
else else

15
services/document-updater/app/coffee/RedisManager.coffee
View file

@ -36,7 +36,7 @@ historyKeys = Settings.redis.history.key_schema
module.exports = RedisManager = module.exports = RedisManager =
rclient: rclient rclient: rclient
putDocInMemory : (project_id, doc_id, docLines, version, ranges, pathname, _callback)-> putDocInMemory : (project_id, doc_id, docLines, version, ranges, pathname, projectHistoryId, _callback)->
timer = new metrics.Timer("redis.put-doc") timer = new metrics.Timer("redis.put-doc")
callback = (error) -> callback = (error) ->
timer.done() timer.done()
@ -47,7 +47,7 @@ module.exports = RedisManager =
logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message
return callback(error) return callback(error)
docHash = RedisManager._computeHash(docLines) docHash = RedisManager._computeHash(docLines)
logger.log {project_id, doc_id, version, docHash, pathname}, "putting doc in redis" logger.log {project_id, doc_id, version, docHash, pathname, projectHistoryId}, "putting doc in redis"
RedisManager._serializeRanges ranges, (error, ranges) -> RedisManager._serializeRanges ranges, (error, ranges) ->
if error? if error?
logger.error {err: error, doc_id, project_id}, error.message logger.error {err: error, doc_id, project_id}, error.message
@ -62,6 +62,7 @@ module.exports = RedisManager =
else else
multi.del keys.ranges(doc_id:doc_id) multi.del keys.ranges(doc_id:doc_id)
multi.set keys.pathname(doc_id:doc_id), pathname multi.set keys.pathname(doc_id:doc_id), pathname
multi.set keys.projectHistoryId(doc_id:doc_id), projectHistoryId
multi.exec (error, result) -> multi.exec (error, result) ->
return callback(error) if error? return callback(error) if error?
# check the hash computed on the redis server # check the hash computed on the redis server
@ -88,6 +89,7 @@ module.exports = RedisManager =
multi.del keys.docHash(doc_id:doc_id) multi.del keys.docHash(doc_id:doc_id)
multi.del keys.ranges(doc_id:doc_id) multi.del keys.ranges(doc_id:doc_id)
multi.del keys.pathname(doc_id:doc_id) multi.del keys.pathname(doc_id:doc_id)
multi.del keys.projectHistoryId(doc_id:doc_id)
multi.del keys.unflushedTime(doc_id:doc_id) multi.del keys.unflushedTime(doc_id:doc_id)
multi.exec (error) -> multi.exec (error) ->
return callback(error) if error? return callback(error) if error?
@ -108,7 +110,7 @@ module.exports = RedisManager =
clearProjectState: (project_id, callback = (error) ->) -> clearProjectState: (project_id, callback = (error) ->) ->
rclient.del keys.projectState(project_id:project_id), callback rclient.del keys.projectState(project_id:project_id), callback
getDoc : (project_id, doc_id, callback = (error, lines, version, ranges, pathname, unflushedTime) ->)-> getDoc : (project_id, doc_id, callback = (error, lines, version, ranges, pathname, projectHistoryId, unflushedTime) ->)->
timer = new metrics.Timer("redis.get-doc") timer = new metrics.Timer("redis.get-doc")
multi = rclient.multi() multi = rclient.multi()
multi.get keys.docLines(doc_id:doc_id) multi.get keys.docLines(doc_id:doc_id)
@ -117,8 +119,9 @@ module.exports = RedisManager =
multi.get keys.projectKey(doc_id:doc_id) multi.get keys.projectKey(doc_id:doc_id)
multi.get keys.ranges(doc_id:doc_id) multi.get keys.ranges(doc_id:doc_id)
multi.get keys.pathname(doc_id:doc_id) multi.get keys.pathname(doc_id:doc_id)
multi.get keys.projectHistoryId(doc_id:doc_id)
multi.get keys.unflushedTime(doc_id:doc_id) multi.get keys.unflushedTime(doc_id:doc_id)
multi.exec (error, [docLines, version, storedHash, doc_project_id, ranges, pathname, unflushedTime])-> multi.exec (error, [docLines, version, storedHash, doc_project_id, ranges, pathname, projectHistoryId, unflushedTime])->
timeSpan = timer.done() timeSpan = timer.done()
return callback(error) if error? return callback(error) if error?
# check if request took too long and bail out. only do this for # check if request took too long and bail out. only do this for
@ -147,14 +150,14 @@ module.exports = RedisManager =
# doc is not in redis, bail out # doc is not in redis, bail out
if !docLines? if !docLines?
return callback null, docLines, version, ranges, pathname, unflushedTime return callback null, docLines, version, ranges, pathname, projectHistoryId, unflushedTime
# doc should be in project set, check if missing (workaround for missing docs from putDoc) # doc should be in project set, check if missing (workaround for missing docs from putDoc)
rclient.sadd keys.docsInProject(project_id:project_id), doc_id, (error, result) -> rclient.sadd keys.docsInProject(project_id:project_id), doc_id, (error, result) ->
return callback(error) if error? return callback(error) if error?
if result isnt 0 # doc should already be in set if result isnt 0 # doc should already be in set
logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, "doc missing from docsInProject set" logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, "doc missing from docsInProject set"
callback null, docLines, version, ranges, pathname, unflushedTime callback null, docLines, version, ranges, pathname, projectHistoryId, unflushedTime
getDocVersion: (doc_id, callback = (error, version) ->) -> getDocVersion: (doc_id, callback = (error, version) ->) ->
rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> rclient.get keys.docVersion(doc_id: doc_id), (error, version) ->

7
services/document-updater/app/coffee/UpdateManager.coffee
View file

@ -71,7 +71,7 @@ module.exports = UpdateManager =
profile = new Profiler("applyUpdate", {project_id, doc_id}) profile = new Profiler("applyUpdate", {project_id, doc_id})
UpdateManager._sanitizeUpdate update UpdateManager._sanitizeUpdate update
profile.log("sanitizeUpdate") profile.log("sanitizeUpdate")
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname) -> DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, projectHistoryId) ->
profile.log("getDoc") profile.log("getDoc")
return callback(error) if error? return callback(error) if error?
if !lines? or !version? if !lines? or !version?
@ -80,7 +80,7 @@ module.exports = UpdateManager =
profile.log("sharejs.applyUpdate") profile.log("sharejs.applyUpdate")
return callback(error) if error? return callback(error) if error?
RangesManager.applyUpdate project_id, doc_id, ranges, appliedOps, updatedDocLines, (error, new_ranges) -> RangesManager.applyUpdate project_id, doc_id, ranges, appliedOps, updatedDocLines, (error, new_ranges) ->
UpdateManager._addProjectHistoryMetadataToOps(appliedOps, pathname, lines) UpdateManager._addProjectHistoryMetadataToOps(appliedOps, pathname, projectHistoryId, lines)
profile.log("RangesManager.applyUpdate") profile.log("RangesManager.applyUpdate")
return callback(error) if error? return callback(error) if error?
RedisManager.updateDocument project_id, doc_id, updatedDocLines, version, appliedOps, new_ranges, (error, doc_ops_length, project_ops_length) -> RedisManager.updateDocument project_id, doc_id, updatedDocLines, version, appliedOps, new_ranges, (error, doc_ops_length, project_ops_length) ->
@ -130,12 +130,13 @@ module.exports = UpdateManager =
op.i = op.i.replace(/[\uD800-\uDFFF]/g, "\uFFFD") op.i = op.i.replace(/[\uD800-\uDFFF]/g, "\uFFFD")
return update return update
_addProjectHistoryMetadataToOps: (updates, pathname, lines) -> _addProjectHistoryMetadataToOps: (updates, pathname, projectHistoryId, lines) ->
doc_length = _.reduce lines, doc_length = _.reduce lines,
(chars, line) -> chars + line.length, (chars, line) -> chars + line.length,
0 0
doc_length += lines.length - 1 # count newline characters doc_length += lines.length - 1 # count newline characters
updates.forEach (update) -> updates.forEach (update) ->
update.projectHistoryId = projectHistoryId
update.meta ||= {} update.meta ||= {}
update.meta.pathname = pathname update.meta.pathname = pathname
update.meta.doc_length = doc_length update.meta.doc_length = doc_length

1
services/document-updater/config/settings.defaults.coffee
View file

@ -46,6 +46,7 @@ module.exports =
docsInProject: ({project_id}) -> "DocsIn:#{project_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
ranges: ({doc_id}) -> "Ranges:#{doc_id}" ranges: ({doc_id}) -> "Ranges:#{doc_id}"
pathname: ({doc_id}) -> "Pathname:#{doc_id}" pathname: ({doc_id}) -> "Pathname:#{doc_id}"
projectHistoryId: ({doc_id}) -> "ProjectHistoryId:#{doc_id}"
projectState: ({project_id}) -> "ProjectState:#{project_id}" projectState: ({project_id}) -> "ProjectState:#{project_id}"
unflushedTime: ({doc_id}) -> "UnflushedTime:#{doc_id}" unflushedTime: ({doc_id}) -> "UnflushedTime:#{doc_id}"
# cluster: [{ # cluster: [{

22
services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee
View file

@ -111,7 +111,7 @@ describe "DocumentManager", ->
describe "getDocAndRecentOps", -> describe "getDocAndRecentOps", ->
describe "with a previous version specified", -> describe "with a previous version specified", ->
beforeEach -> beforeEach ->
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname) @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, @projectHistoryId)
@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
@DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback
@ -126,14 +126,14 @@ describe "DocumentManager", ->
.should.equal true .should.equal true
it "should call the callback with the doc info", -> it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, @ops, @ranges, @pathname).should.equal true @callback.calledWith(null, @lines, @version, @ops, @ranges, @pathname, @projectHistoryId).should.equal true
it "should time the execution", -> it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true @Metrics.Timer::done.called.should.equal true
describe "with no previous version specified", -> describe "with no previous version specified", ->
beforeEach -> beforeEach ->
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname) @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, @projectHistoryId)
@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
@DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback
@ -146,7 +146,7 @@ describe "DocumentManager", ->
@RedisManager.getPreviousDocOps.called.should.equal false @RedisManager.getPreviousDocOps.called.should.equal false
it "should call the callback with the doc info", -> it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, [], @ranges, @pathname).should.equal true @callback.calledWith(null, @lines, @version, [], @ranges, @pathname, @projectHistoryId).should.equal true
it "should time the execution", -> it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true @Metrics.Timer::done.called.should.equal true
@ -154,7 +154,7 @@ describe "DocumentManager", ->
describe "getDoc", -> describe "getDoc", ->
describe "when the doc exists in Redis", -> describe "when the doc exists in Redis", ->
beforeEach -> beforeEach ->
@RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, @unflushedTime) @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, @projectHistoryId, @unflushedTime)
@DocumentManager.getDoc @project_id, @doc_id, @callback @DocumentManager.getDoc @project_id, @doc_id, @callback
it "should get the doc from Redis", -> it "should get the doc from Redis", ->
@ -163,7 +163,7 @@ describe "DocumentManager", ->
.should.equal true .should.equal true
it "should call the callback with the doc info", -> it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, @ranges, @pathname, @unflushedTime, true).should.equal true @callback.calledWith(null, @lines, @version, @ranges, @pathname, @projectHistoryId, @unflushedTime, true).should.equal true
it "should time the execution", -> it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true @Metrics.Timer::done.called.should.equal true
@ -171,7 +171,7 @@ describe "DocumentManager", ->
describe "when the doc does not exist in Redis", -> describe "when the doc does not exist in Redis", ->
beforeEach -> beforeEach ->
@RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null, null) @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null, null)
@PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname) @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, @projectHistoryId)
@RedisManager.putDocInMemory = sinon.stub().yields() @RedisManager.putDocInMemory = sinon.stub().yields()
@DocumentManager.getDoc @project_id, @doc_id, @callback @DocumentManager.getDoc @project_id, @doc_id, @callback
@ -187,11 +187,11 @@ describe "DocumentManager", ->
it "should set the doc in Redis", -> it "should set the doc in Redis", ->
@RedisManager.putDocInMemory @RedisManager.putDocInMemory
.calledWith(@project_id, @doc_id, @lines, @version, @ranges, @pathname) .calledWith(@project_id, @doc_id, @lines, @version, @ranges, @pathname, @projectHistoryId)
.should.equal true .should.equal true
it "should call the callback with the doc info", -> it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, @ranges, @pathname, null, false).should.equal true @callback.calledWith(null, @lines, @version, @ranges, @pathname, @projectHistoryId, null, false).should.equal true
it "should time the execution", -> it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true @Metrics.Timer::done.called.should.equal true
@ -202,7 +202,7 @@ describe "DocumentManager", ->
@beforeLines = ["before", "lines"] @beforeLines = ["before", "lines"]
@afterLines = ["after", "lines"] @afterLines = ["after", "lines"]
@ops = [{ i: "foo", p: 4 }, { d: "bar", p: 42 }] @ops = [{ i: "foo", p: 4 }, { d: "bar", p: 42 }]
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @ranges, @pathname, @unflushedTime, true) @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @ranges, @pathname, @projectHistoryId, @unflushedTime, true)
@DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops)
@UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null) @UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null)
@DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2)
@ -402,7 +402,7 @@ describe "DocumentManager", ->
describe "when the doc is in Redis", -> describe "when the doc is in Redis", ->
describe "and has changes to be flushed", -> describe "and has changes to be flushed", ->
beforeEach -> beforeEach ->
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname, Date.now() - 1e9, true) @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @projectHistoryId, @pathname, Date.now() - 1e9, true)
@DocumentManager.getDocAndFlushIfOld @project_id, @doc_id, @callback @DocumentManager.getDocAndFlushIfOld @project_id, @doc_id, @callback
it "should get the doc", -> it "should get the doc", ->

6
services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee
View file

@ -17,6 +17,7 @@ describe "PersistenceManager", ->
done: sinon.stub() done: sinon.stub()
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()} "logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@project_id = "project-id-123" @project_id = "project-id-123"
@projectHistoryId = "history-id-123"
@doc_id = "doc-id-123" @doc_id = "doc-id-123"
@lines = ["one", "two", "three"] @lines = ["one", "two", "three"]
@version = 42 @version = 42
@ -36,6 +37,7 @@ describe "PersistenceManager", ->
version: @version, version: @version,
ranges: @ranges ranges: @ranges
pathname: @pathname, pathname: @pathname,
projectHistoryId: @projectHistoryId
} }
describe "with a successful response from the web api", -> describe "with a successful response from the web api", ->
@ -60,7 +62,9 @@ describe "PersistenceManager", ->
.should.equal true .should.equal true
it "should call the callback with the doc lines, version and ranges", -> it "should call the callback with the doc lines, version and ranges", ->
@callback.calledWith(null, @lines, @version, @ranges, @pathname).should.equal true @callback
.calledWith(null, @lines, @version, @ranges, @pathname, @projectHistoryId)
.should.equal true
it "should time the execution", -> it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true @Metrics.Timer::done.called.should.equal true

36
services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee
View file

@ -33,6 +33,7 @@ describe "RedisManager", ->
docsInProject: ({project_id}) -> "DocsIn:#{project_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
ranges: ({doc_id}) -> "Ranges:#{doc_id}" ranges: ({doc_id}) -> "Ranges:#{doc_id}"
pathname: ({doc_id}) -> "Pathname:#{doc_id}" pathname: ({doc_id}) -> "Pathname:#{doc_id}"
projectHistoryId: ({doc_id}) -> "ProjectHistoryId:#{doc_id}"
projectState: ({project_id}) -> "ProjectState:#{project_id}" projectState: ({project_id}) -> "ProjectState:#{project_id}"
unflushedTime: ({doc_id}) -> "UnflushedTime:#{doc_id}" unflushedTime: ({doc_id}) -> "UnflushedTime:#{doc_id}"
history: history:
@ -72,7 +73,7 @@ describe "RedisManager", ->
@unflushed_time = 12345 @unflushed_time = 12345
@pathname = '/a/b/c.tex' @pathname = '/a/b/c.tex'
@multi.get = sinon.stub() @multi.get = sinon.stub()
@multi.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @hash, @project_id, @json_ranges, @pathname, @unflushed_time]) @multi.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @hash, @project_id, @json_ranges, @pathname, @projectHistoryId, @unflushed_time])
@rclient.sadd = sinon.stub().yields(null, 0) @rclient.sadd = sinon.stub().yields(null, 0)
describe "successfully", -> describe "successfully", ->
@ -109,6 +110,11 @@ describe "RedisManager", ->
.calledWith("Pathname:#{@doc_id}") .calledWith("Pathname:#{@doc_id}")
.should.equal true .should.equal true
it "should get the projectHistoryId", ->
@multi.get
.calledWith("ProjectHistoryId:#{@doc_id}")
.should.equal true
it "should check if the document is in the DocsIn set", -> it "should check if the document is in the DocsIn set", ->
@rclient.sadd @rclient.sadd
.calledWith("DocsIn:#{@project_id}") .calledWith("DocsIn:#{@project_id}")
@ -116,7 +122,7 @@ describe "RedisManager", ->
it 'should return the document', -> it 'should return the document', ->
@callback @callback
.calledWithExactly(null, @lines, @version, @ranges, @pathname, @unflushed_time) .calledWithExactly(null, @lines, @version, @ranges, @pathname, @projectHistoryId, @unflushed_time)
.should.equal true .should.equal true
it 'should not log any errors', -> it 'should not log any errors', ->
@ -125,7 +131,7 @@ describe "RedisManager", ->
describe "when the document is not present", -> describe "when the document is not present", ->
beforeEach -> beforeEach ->
@multi.exec = sinon.stub().callsArgWith(0, null, [null, null, null, null, null, null, null]) @multi.exec = sinon.stub().callsArgWith(0, null, [null, null, null, null, null, null, null, null])
@rclient.sadd = sinon.stub().yields() @rclient.sadd = sinon.stub().yields()
@RedisManager.getDoc @project_id, @doc_id, @callback @RedisManager.getDoc @project_id, @doc_id, @callback
@ -136,7 +142,7 @@ describe "RedisManager", ->
it 'should return an empty result', -> it 'should return an empty result', ->
@callback @callback
.calledWithExactly(null, null, 0, {}, null, null) .calledWithExactly(null, null, 0, {}, null, null, null)
.should.equal true .should.equal true
it 'should not log any errors', -> it 'should not log any errors', ->
@ -154,7 +160,7 @@ describe "RedisManager", ->
it 'should return the document', -> it 'should return the document', ->
@callback @callback
.calledWithExactly(null, @lines, @version, @ranges, @pathname, @unflushed_time) .calledWithExactly(null, @lines, @version, @ranges, @pathname, @projectHistoryId, @unflushed_time)
.should.equal true .should.equal true
describe "with a corrupted document", -> describe "with a corrupted document", ->
@ -532,7 +538,7 @@ describe "RedisManager", ->
describe "with non-empty ranges", -> describe "with non-empty ranges", ->
beforeEach (done) -> beforeEach (done) ->
@RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, done @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @projectHistoryId, done
it "should set the lines", -> it "should set the lines", ->
@multi.eval @multi.eval
@ -564,6 +570,11 @@ describe "RedisManager", ->
.calledWith("Pathname:#{@doc_id}", @pathname) .calledWith("Pathname:#{@doc_id}", @pathname)
.should.equal true .should.equal true
it "should set the projectHistoryId for the doc", ->
@multi.set
.calledWith("ProjectHistoryId:#{@doc_id}", @projectHistoryId)
.should.equal true
it "should add the doc_id to the project set", -> it "should add the doc_id to the project set", ->
@rclient.sadd @rclient.sadd
.calledWith("DocsIn:#{@project_id}", @doc_id) .calledWith("DocsIn:#{@project_id}", @doc_id)
@ -575,7 +586,7 @@ describe "RedisManager", ->
describe "with empty ranges", -> describe "with empty ranges", ->
beforeEach (done) -> beforeEach (done) ->
@RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, {}, @pathname, done @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, {}, @pathname, @projectHistoryId, done
it "should delete the ranges key", -> it "should delete the ranges key", ->
@multi.del @multi.del
@ -590,7 +601,7 @@ describe "RedisManager", ->
describe "with a corrupted write", -> describe "with a corrupted write", ->
beforeEach (done) -> beforeEach (done) ->
@multi.exec = sinon.stub().callsArgWith(0, null, ["INVALID-HASH-VALUE"]) @multi.exec = sinon.stub().callsArgWith(0, null, ["INVALID-HASH-VALUE"])
@RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, done @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @projectHistoryId, done
it 'should log a hash error', -> it 'should log a hash error', ->
@logger.error.calledWith() @logger.error.calledWith()
@ -600,7 +611,7 @@ describe "RedisManager", ->
beforeEach -> beforeEach ->
@_stringify = JSON.stringify @_stringify = JSON.stringify
@JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]' @JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]'
@RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @callback @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @projectHistoryId, @callback
afterEach -> afterEach ->
@JSON.stringify = @_stringify @JSON.stringify = @_stringify
@ -614,7 +625,7 @@ describe "RedisManager", ->
describe "with ranges that are too big", -> describe "with ranges that are too big", ->
beforeEach -> beforeEach ->
@RedisManager._serializeRanges = sinon.stub().yields(new Error("ranges are too large")) @RedisManager._serializeRanges = sinon.stub().yields(new Error("ranges are too large"))
@RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @callback @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, @pathname, @projectHistoryId, @callback
it 'should log an error', -> it 'should log an error', ->
@logger.error.called.should.equal true @logger.error.called.should.equal true
@ -664,6 +675,11 @@ describe "RedisManager", ->
.calledWith("Pathname:#{@doc_id}") .calledWith("Pathname:#{@doc_id}")
.should.equal true .should.equal true
it "should delete the pathname for the doc", ->
@multi.del
.calledWith("ProjectHistoryId:#{@doc_id}")
.should.equal true
describe "clearProjectState", -> describe "clearProjectState", ->
beforeEach (done) -> beforeEach (done) ->
@rclient.del = sinon.stub().callsArg(1) @rclient.del = sinon.stub().callsArg(1)

12
services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee
View file

@ -7,6 +7,7 @@ SandboxedModule = require('sandboxed-module')
describe "UpdateManager", -> describe "UpdateManager", ->
beforeEach -> beforeEach ->
@project_id = "project-id-123" @project_id = "project-id-123"
@projectHistoryId = "history-id-123"
@doc_id = "document-id-123" @doc_id = "document-id-123"
@callback = sinon.stub() @callback = sinon.stub()
@UpdateManager = SandboxedModule.require modulePath, requires: @UpdateManager = SandboxedModule.require modulePath, requires:
@ -167,7 +168,7 @@ describe "UpdateManager", ->
@doc_ops_length = sinon.stub() @doc_ops_length = sinon.stub()
@project_ops_length = sinon.stub() @project_ops_length = sinon.stub()
@pathname = '/a/b/c.tex' @pathname = '/a/b/c.tex'
@DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges, @pathname) @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges, @pathname, @projectHistoryId)
@RangesManager.applyUpdate = sinon.stub().yields(null, @updated_ranges) @RangesManager.applyUpdate = sinon.stub().yields(null, @updated_ranges)
@ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps)
@RedisManager.updateDocument = sinon.stub().yields(null, @doc_ops_length, @project_ops_length) @RedisManager.updateDocument = sinon.stub().yields(null, @doc_ops_length, @project_ops_length)
@ -196,7 +197,7 @@ describe "UpdateManager", ->
it "should add metadata to the ops" , -> it "should add metadata to the ops" , ->
@UpdateManager._addProjectHistoryMetadataToOps @UpdateManager._addProjectHistoryMetadataToOps
.calledWith(@appliedOps, @pathname, @lines) .calledWith(@appliedOps, @pathname, @projectHistoryId, @lines)
.should.equal true .should.equal true
it "should push the applied ops into the history queue", -> it "should push the applied ops into the history queue", ->
@ -239,7 +240,7 @@ describe "UpdateManager", ->
@callback.calledWith(@error).should.equal true @callback.calledWith(@error).should.equal true
describe "_addProjectHistoryMetadataToOps", -> describe "_addProjectHistoryMetadataToOps", ->
it "should add pathname and doc_length metadata to the ops", -> it "should add projectHistoryId, pathname and doc_length metadata to the ops", ->
lines = [ lines = [
'some' 'some'
'test' 'test'
@ -250,20 +251,23 @@ describe "UpdateManager", ->
{ v: 45, op: [{d: "qux", p: 4}, { i: "bazbaz", p: 14 }] }, { v: 45, op: [{d: "qux", p: 4}, { i: "bazbaz", p: 14 }] },
{ v: 49, op: [{i: "penguin", p: 18}] } { v: 49, op: [{i: "penguin", p: 18}] }
] ]
@UpdateManager._addProjectHistoryMetadataToOps(appliedOps, @pathname, lines) @UpdateManager._addProjectHistoryMetadataToOps(appliedOps, @pathname, @projectHistoryId, lines)
appliedOps.should.deep.equal [{ appliedOps.should.deep.equal [{
projectHistoryId: @projectHistoryId
v: 42 v: 42
op: [{i: "foo", p: 4}, { i: "bar", p: 6 }] op: [{i: "foo", p: 4}, { i: "bar", p: 6 }]
meta: meta:
pathname: @pathname pathname: @pathname
doc_length: 14 doc_length: 14
}, { }, {
projectHistoryId: @projectHistoryId
v: 45 v: 45
op: [{d: "qux", p: 4}, { i: "bazbaz", p: 14 }] op: [{d: "qux", p: 4}, { i: "bazbaz", p: 14 }]
meta: meta:
pathname: @pathname pathname: @pathname
doc_length: 20 # 14 + 'foo' + 'bar' doc_length: 20 # 14 + 'foo' + 'bar'
}, { }, {
projectHistoryId: @projectHistoryId
v: 49 v: 49
op: [{i: "penguin", p: 18}] op: [{i: "penguin", p: 18}]
meta: meta: