github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-10-31 21:21:03 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #10 from sharelatex/hof-version-renames

Browse source 
version entity renames
This commit is contained in:
Hayden Faulds 2017-11-09 09:15:59 +00:00 committed by GitHub
commit 569912a832
11 changed files with 385 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
services/document-updater/app.coffee
View file

@ -47,6 +47,7 @@ app.post '/project/:project_id/doc/:doc_id', HttpCont
app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLoaded
app.delete '/project/:project_id/doc/:doc_id', HttpController.flushAndDeleteDoc
app.delete '/project/:project_id', HttpController.deleteProject
app.post '/project/:project_id', HttpController.updateProject
app.post '/project/:project_id/flush', HttpController.flushProject
app.post '/project/:project_id/doc/:doc_id/change/:change_id/accept', HttpController.acceptChanges
app.post '/project/:project_id/doc/:doc_id/change/accept', HttpController.acceptChanges

23
services/document-updater/app/coffee/DocumentManager.coffee
View file

@ -7,6 +7,7 @@ HistoryManager = require "./HistoryManager"
RealTimeRedisManager = require "./RealTimeRedisManager"
Errors = require "./Errors"
RangesManager = require "./RangesManager"
async = require "async"
MAX_UNFLUSHED_AGE = 300 * 1000 # 5 mins, document should be flushed to mongo this time after a change
@ -30,20 +31,20 @@ module.exports = DocumentManager =
else
callback null, lines, version, ranges, pathname, unflushedTime, true
getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps, ranges) ->) ->
getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, ops, ranges, pathname) ->) ->
timer = new Metrics.Timer("docManager.getDocAndRecentOps")
callback = (args...) ->
timer.done()
_callback(args...)
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) ->
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname) ->
return callback(error) if error?
if fromVersion == -1
callback null, lines, version, [], ranges
callback null, lines, version, [], ranges, pathname
else
RedisManager.getPreviousDocOps doc_id, fromVersion, version, (error, ops) ->
return callback(error) if error?
callback null, lines, version, ops, ranges
callback null, lines, version, ops, ranges, pathname
setDoc: (project_id, doc_id, newLines, source, user_id, undoing, _callback = (error) ->) ->
timer = new Metrics.Timer("docManager.setDoc")
@ -155,6 +156,14 @@ module.exports = DocumentManager =
return callback(error) if error?
callback()
renameDoc: (project_id, doc_id, user_id, update, _callback = (error) ->) ->
timer = new Metrics.Timer("docManager.updateProject")
callback = (args...) ->
timer.done()
_callback(args...)
RedisManager.renameDoc project_id, doc_id, user_id, update, callback
getDocAndFlushIfOld: (project_id, doc_id, callback = (error, doc) ->) ->
DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname, unflushedTime, alreadyLoaded) ->
return callback(error) if error?
@ -170,7 +179,7 @@ module.exports = DocumentManager =
UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.getDoc, project_id, doc_id, callback
getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version) ->) ->
getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version, ops, ranges, pathname) ->) ->
UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.getDocAndRecentOps, project_id, doc_id, fromVersion, callback
@ -197,3 +206,7 @@ module.exports = DocumentManager =
deleteCommentWithLock: (project_id, doc_id, thread_id, callback = (error) ->) ->
UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.deleteComment, project_id, doc_id, thread_id, callback
renameDocWithLock: (project_id, doc_id, user_id, update, callback = (error) ->) ->
UpdateManager = require "./UpdateManager"
UpdateManager.lockUpdatesAndDo DocumentManager.renameDoc, project_id, doc_id, user_id, update, callback

17
services/document-updater/app/coffee/HttpController.coffee
View file

@ -18,7 +18,7 @@ module.exports = HttpController =
else
fromVersion = -1
DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops, ranges) ->
DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops, ranges, pathname) ->
timer.done()
return next(error) if error?
logger.log project_id: project_id, doc_id: doc_id, "got doc via http"
@ -30,6 +30,7 @@ module.exports = HttpController =
version: version
ops: ops
ranges: ranges
pathname: pathname
_getTotalSizeOfLines: (lines) ->
size = 0
@ -141,7 +142,7 @@ module.exports = HttpController =
return next(error) if error?
logger.log {project_id, doc_id}, "accepted #{ change_ids.length } changes via http"
res.send 204 # No Content
deleteComment: (req, res, next = (error) ->) ->
{project_id, doc_id, comment_id} = req.params
logger.log {project_id, doc_id, comment_id}, "deleting comment via http"
@ -151,5 +152,15 @@ module.exports = HttpController =
return next(error) if error?
logger.log {project_id, doc_id, comment_id}, "deleted comment via http"
res.send 204 # No Content
updateProject: (req, res, next = (error) ->) ->
timer = new Metrics.Timer("http.updateProject")
project_id = req.params.project_id
{userId, docUpdates, fileUpdates} = req.body
logger.log {project_id, docUpdates, fileUpdates}, "updating project via http"
ProjectManager.updateProjectWithLocks project_id, userId, docUpdates, fileUpdates, (error) ->
timer.done()
return next(error) if error?
logger.log project_id: project_id, "updated project via http"
res.send 204 # No Content

18
services/document-updater/app/coffee/ProjectManager.coffee
View file

@ -93,3 +93,21 @@ module.exports = ProjectManager =
clearProjectState: (project_id, callback = (error) ->) ->
RedisManager.clearProjectState project_id, callback
updateProjectWithLocks: (project_id, user_id, docUpdates, fileUpdates, _callback = (error) ->) ->
timer = new Metrics.Timer("projectManager.updateProject")
callback = (args...) ->
timer.done()
_callback(args...)
handleDocUpdate = (update, cb) ->
doc_id = update.id
DocumentManager.renameDocWithLock project_id, doc_id, user_id, update, cb
handleFileUpdate = (update, cb) ->
file_id = update.id
RedisManager.renameFile project_id, file_id, user_id, update, cb
async.each docUpdates, handleDocUpdate, (error) ->
return callback(error) if error?
async.each fileUpdates, handleFileUpdate, callback

27
services/document-updater/app/coffee/RedisManager.coffee
View file

@ -272,6 +272,33 @@ module.exports = RedisManager =
else
callback null, docUpdateCount
renameDoc: (project_id, doc_id, user_id, update, callback = (error) ->) ->
RedisManager.getDoc project_id, doc_id, (error, lines, version) ->
return callback(error) if error?
if lines? and version?
rclient.set keys.pathname(doc_id:doc_id), update.newPathname, (error) ->
return callback(error) if error?
RedisManager._renameEntity project_id, 'doc', doc_id, user_id, update, callback
else
RedisManager._renameEntity project_id, 'doc', doc_id, user_id, update, callback
renameFile: (project_id, file_id, user_id, update, callback = (error) ->) ->
RedisManager._renameEntity project_id, 'file', file_id, user_id, update, callback
_renameEntity: (project_id, entity_type, entity_id, user_id, update, callback = (error) ->) ->
update =
pathname: update.pathname
new_pathname: update.newPathname
meta:
user_id: user_id
ts: new Date()
update[entity_type] = entity_id
logger.log {project_id, update}, "queue rename operation to project-history"
jsonUpdate = JSON.stringify(update)
rclient.rpush projectHistoryKeys.projectHistoryOps({project_id}), jsonUpdate, callback
clearUnflushedTime: (doc_id, callback = (error) ->) ->
rclient.del keys.unflushedTime(doc_id:doc_id), callback

98
services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee Normal file
View file

@ -0,0 +1,98 @@
sinon = require "sinon"
chai = require("chai")
chai.should()
Settings = require('settings-sharelatex')
rclient_history = require("redis-sharelatex").createClient(Settings.redis.history)
ProjectHistoryKeys = Settings.redis.project_history.key_schema
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
describe "Applying updates to a project's structure", ->
before ->
@user_id = 'user-id-123'
describe "renaming a file", ->
before (done) ->
@project_id = DocUpdaterClient.randomId()
@fileUpdate =
id: DocUpdaterClient.randomId()
pathname: '/file-path'
newPathname: '/new-file-path'
@fileUpdates = [ @fileUpdate ]
DocUpdaterClient.sendProjectUpdate @project_id, @user_id, [], @fileUpdates, (error) ->
throw error if error?
setTimeout done, 200
it "should push the applied file renames to the project history changes api", (done) ->
rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) =>
throw error if error?
update = JSON.parse(updates[0])
update.file.should.equal @fileUpdate.id
update.pathname.should.equal '/file-path'
update.new_pathname.should.equal '/new-file-path'
update.meta.user_id.should.equal @user_id
update.meta.ts.should.be.a('string')
done()
describe "renaming a document", ->
before ->
@docUpdate =
id: DocUpdaterClient.randomId()
pathname: '/doc-path'
newPathname: '/new-doc-path'
@docUpdates = [ @docUpdate ]
describe "when the document is not loaded", ->
before (done) ->
@project_id = DocUpdaterClient.randomId()
DocUpdaterClient.sendProjectUpdate @project_id, @user_id, @docUpdates, [], (error) ->
throw error if error?
setTimeout done, 200
it "should push the applied doc renames to the project history changes api", (done) ->
rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) =>
throw error if error?
update = JSON.parse(updates[0])
update.doc.should.equal @docUpdate.id
update.pathname.should.equal '/doc-path'
update.new_pathname.should.equal '/new-doc-path'
update.meta.user_id.should.equal @user_id
update.meta.ts.should.be.a('string')
done()
describe "when the document is loaded", ->
before (done) ->
@project_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc @project_id, @docUpdate.id, {}
DocUpdaterClient.preloadDoc @project_id, @docUpdate.id, (error) =>
throw error if error?
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.sendProjectUpdate @project_id, @user_id, @docUpdates, [], (error) ->
throw error if error?
setTimeout done, 200
after ->
MockWebApi.getDocument.restore()
it "should update the doc", (done) ->
DocUpdaterClient.getDoc @project_id, @docUpdate.id, (error, res, doc) =>
doc.pathname.should.equal @docUpdate.newPathname
done()
it "should push the applied doc renames to the project history changes api", (done) ->
rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) =>
throw error if error?
update = JSON.parse(updates[0])
update.doc.should.equal @docUpdate.id
update.pathname.should.equal '/doc-path'
update.new_pathname.should.equal '/new-doc-path'
update.meta.user_id.should.equal @user_id
update.meta.ts.should.be.a('string')
done()

7
services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee
View file

@ -86,3 +86,10 @@ module.exports = DocUpdaterClient =
if body? and res.statusCode >= 200 and res.statusCode < 300
body = JSON.parse(body)
callback error, res, body
sendProjectUpdate: (project_id, userId, docUpdates, fileUpdates, callback = (error) ->) ->
request.post {
url: "http://localhost:3003/project/#{project_id}"
json: { userId, docUpdates, fileUpdates }
}, (error, res, body) ->
callback error, res, body

26
services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee
View file

@ -23,6 +23,7 @@ describe "DocumentManager", ->
"./RangesManager": @RangesManager = {}
@project_id = "project-id-123"
@doc_id = "doc-id-123"
@user_id = 1234
@callback = sinon.stub()
@lines = ["one", "two", "three"]
@version = 42
@ -108,7 +109,7 @@ describe "DocumentManager", ->
describe "getDocAndRecentOps", ->
describe "with a previous version specified", ->
beforeEach ->
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges)
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname)
@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
@DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback
@ -123,14 +124,14 @@ describe "DocumentManager", ->
.should.equal true
it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, @ops, @ranges).should.equal true
@callback.calledWith(null, @lines, @version, @ops, @ranges, @pathname).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "with no previous version specified", ->
beforeEach ->
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges)
@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges, @pathname)
@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
@DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback
@ -143,7 +144,7 @@ describe "DocumentManager", ->
@RedisManager.getPreviousDocOps.called.should.equal false
it "should call the callback with the doc info", ->
@callback.calledWith(null, @lines, @version, [], @ranges).should.equal true
@callback.calledWith(null, @lines, @version, [], @ranges, @pathname).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
@ -439,3 +440,20 @@ describe "DocumentManager", ->
it "should call the callback with the lines and versions", ->
@callback.calledWith(null, @lines, @version).should.equal true
describe "renameDoc", ->
beforeEach ->
@update = 'some-update'
@RedisManager.renameDoc = sinon.stub().yields()
describe "successfully", ->
beforeEach ->
@DocumentManager.renameDoc @project_id, @doc_id, @user_id, @update, @callback
it "should rename the document", ->
@RedisManager.renameDoc
.calledWith(@project_id, @doc_id, @user_id, @update)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true

42
services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee
View file

@ -28,6 +28,7 @@ describe "HttpController", ->
@version = 42
@fromVersion = 42
@ranges = { changes: "mock", comments: "mock" }
@pathname = '/a/b/c'
@req =
params:
project_id: @project_id
@ -35,7 +36,7 @@ describe "HttpController", ->
describe "when the document exists and no recent ops are requested", ->
beforeEach ->
@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @ranges)
@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @ranges, @pathname)
@HttpController.getDoc(@req, @res, @next)
it "should get the doc", ->
@ -51,6 +52,7 @@ describe "HttpController", ->
version: @version
ops: []
ranges: @ranges
pathname: @pathname
}))
.should.equal true
@ -492,3 +494,41 @@ describe "HttpController", ->
@next
.calledWith(new Error("oops"))
.should.equal true
describe "updateProject", ->
beforeEach ->
@userId = "user-id-123"
@docUpdates = sinon.stub()
@fileUpdates = sinon.stub()
@req =
body: {@userId, @docUpdates, @fileUpdates}
params:
project_id: @project_id
describe "successfully", ->
beforeEach ->
@ProjectManager.updateProjectWithLocks = sinon.stub().callsArgWith(4)
@HttpController.updateProject(@req, @res, @next)
it "should accept the change", ->
@ProjectManager.updateProjectWithLocks
.calledWith(@project_id, @userId, @docUpdates, @fileUpdates)
.should.equal true
it "should return a successful No Content response", ->
@res.send
.calledWith(204)
.should.equal true
it "should time the request", ->
@Metrics.Timer::done.called.should.equal true
describe "when an errors occurs", ->
beforeEach ->
@ProjectManager.updateProjectWithLocks = sinon.stub().callsArgWith(4, new Error("oops"))
@HttpController.updateProject(@req, @res, @next)
it "should call next with the error", ->
@next
.calledWith(new Error("oops"))
.should.equal true

73
services/document-updater/test/unit/coffee/ProjectManager/updateProjectTests.coffee Normal file
View file

@ -0,0 +1,73 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/ProjectManager.js"
SandboxedModule = require('sandboxed-module')
describe "ProjectManager", ->
beforeEach ->
@ProjectManager = SandboxedModule.require modulePath, requires:
"./RedisManager": @RedisManager = {}
"./DocumentManager": @DocumentManager = {}
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
@project_id = "project-id-123"
@user_id = "user-id-123"
@callback = sinon.stub()
describe "updateProjectWithLocks", ->
beforeEach ->
@firstDocUpdate =
id: 1
update: 'foo'
@secondDocUpdate =
id: 2
update: 'bar'
@docUpdates = [ @firstDocUpdate, @secondDocUpdate ]
@firstFileUpdate =
id: 2
update: 'bar'
@fileUpdates = [ @firstFileUpdate ]
@DocumentManager.renameDocWithLock = sinon.stub().yields()
@RedisManager.renameFile = sinon.stub().yields()
describe "successfully", ->
beforeEach ->
@ProjectManager.updateProjectWithLocks @project_id, @user_id, @docUpdates, @fileUpdates, @callback
it "should rename the docs in the updates", ->
@DocumentManager.renameDocWithLock
.calledWith(@project_id, @firstDocUpdate.id, @user_id, @firstDocUpdate)
.should.equal true
@DocumentManager.renameDocWithLock
.calledWith(@project_id, @secondDocUpdate.id, @user_id, @secondDocUpdate)
.should.equal true
it "should rename the files in the updates", ->
@RedisManager.renameFile
.calledWith(@project_id, @firstFileUpdate.id, @user_id, @firstFileUpdate)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "when renaming a doc fails", ->
beforeEach ->
@error = new Error('error')
@DocumentManager.renameDocWithLock = sinon.stub().yields(@error)
@ProjectManager.updateProjectWithLocks @project_id, @user_id, @docUpdates, @fileUpdates, @callback
it "should call the callback with the error", ->
@callback.calledWith(@error).should.equal true
describe "when renaming a file fails", ->
beforeEach ->
@error = new Error('error')
@RedisManager.renameFile = sinon.stub().yields(@error)
@ProjectManager.updateProjectWithLocks @project_id, @user_id, @docUpdates, @fileUpdates, @callback
it "should call the callback with the error", ->
@callback.calledWith(@error).should.equal true

66
services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee
View file

@ -673,3 +673,69 @@ describe "RedisManager", ->
@rclient.del
.calledWith("ProjectState:#{@project_id}")
.should.equal true
describe "renameDoc", ->
beforeEach () ->
@rclient.rpush = sinon.stub().yields()
@rclient.set = sinon.stub().yields()
@update =
id: @doc_id
pathname: @pathname = 'pathname'
newPathname: @newPathname = 'new-pathname'
describe "the document is cached in redis", ->
beforeEach ->
@RedisManager.getDoc = sinon.stub().callsArgWith(2, null, 'lines', 'version')
@RedisManager.renameDoc @project_id, @doc_id, @userId, @update, @callback
it "update the cached pathname", ->
@rclient.set
.calledWith("Pathname:#{@doc_id}", @newPathname)
.should.equal true
it "should queue an update", ->
update =
pathname: @pathname
new_pathname: @newPathname
meta:
user_id: @userId
ts: new Date()
doc: @doc_id
@rclient.rpush
.calledWith("ProjectHistory:Ops:#{@project_id}", JSON.stringify(update))
.should.equal true
it "should call the callback", ->
@callback.calledWith().should.equal true
describe "the document is not cached in redis", ->
beforeEach ->
@RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null)
@RedisManager.renameDoc @project_id, @doc_id, @userId, @update, @callback
it "does not update the cached pathname", ->
@rclient.set.called.should.equal false
describe "renameFile", ->
beforeEach () ->
@rclient.rpush = sinon.stub().yields()
@file_id = 1234
@update =
pathname: @pathname = '/old'
newPathname: @newPathname = '/new'
@RedisManager.renameFile @project_id, @file_id, @userId, @update
it "should queue an update", ->
update =
pathname: @pathname
new_pathname: @newPathname
meta:
user_id: @userId
ts: new Date()
file: @file_id
@rclient.rpush
.calledWith("ProjectHistory:Ops:#{@project_id}", JSON.stringify(update))
.should.equal true