github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #303 from sharelatex/hof-mongo-transaction-lock

Browse source 
Introduce 'mongo transaction lock'
This commit is contained in:
Hayden Faulds 2018-02-20 09:42:08 +00:00 committed by GitHub
commit 9a73c123b7
16 changed files with 346 additions and 186 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee
View file

@ -6,15 +6,32 @@ settings = require('settings-sharelatex')
CooldownManager = require '../Cooldown/CooldownManager'
Errors = require '../Errors/Errors'
Folder = require('../../models/Folder').Folder
LockManager = require('../../infrastructure/LockManager')
Project = require('../../models/Project').Project
ProjectEntityHandler = require('./ProjectEntityHandler')
ProjectGetter = require('./ProjectGetter')
ProjectLocator = require('./ProjectLocator')
SafePath = require './SafePath'
LOCK_NAMESPACE = "mongoTransaction"
wrapWithLock = (methodWithoutLock) ->
# This lock is used whenever we read or write to an existing project's
# structure. Some operations to project structure cannot be done atomically
# in mongo, this lock is used to prevent reading the structure between two
# parts of a staged update.
methodWithLock = (project_id, args..., callback) ->
LockManager.runWithLock LOCK_NAMESPACE, project_id,
(cb) -> methodWithoutLock project_id, args..., cb
callback
methodWithLock.withoutLock = methodWithoutLock
methodWithLock
module.exports = ProjectEntityMongoUpdateHandler = self =
addDoc: (project_id, folder_id, doc, callback = (err, result) ->) ->
ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project) ->
LOCK_NAMESPACE: LOCK_NAMESPACE
addDoc: wrapWithLock (project_id, folder_id, doc, callback = (err, result) ->) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder:true, name:true}, (err, project) ->
if err?
logger.err project_id:project_id, err:err, "error getting project for add doc"
return callback(err)
@ -22,8 +39,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
self._confirmFolder project, folder_id, (folder_id) =>
self._putElement project, folder_id, doc, "doc", callback
addFile: (project_id, folder_id, fileRef, callback = (error, result, project) ->)->
ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project) ->
addFile: wrapWithLock (project_id, folder_id, fileRef, callback = (error, result, project) ->)->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder:true, name:true}, (err, project) ->
if err?
logger.err project_id:project_id, err:err, "error getting project for add file"
return callback(err)
@ -31,8 +48,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
self._confirmFolder project, folder_id, (folder_id)->
self._putElement project, folder_id, fileRef, "file", callback
replaceFile: (project_id, file_id, callback) ->
ProjectGetter.getProject project_id, {rootFolder: true, name:true}, (err, project) ->
replaceFile: wrapWithLock (project_id, file_id, callback) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder: true, name:true}, (err, project) ->
return callback(err) if err?
ProjectLocator.findElement {project:project, element_id: file_id, type: 'file'}, (err, fileRef, path)=>
return callback(err) if err?
@ -48,7 +65,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
return callback(err) if err?
callback null, fileRef, project, path
mkdirp: (project_id, path, callback) ->
mkdirp: wrapWithLock (project_id, path, callback) ->
folders = path.split('/')
folders = _.select folders, (folder)->
return folder.length != 0
@ -66,10 +83,10 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
if parentFolder?
parentFolder_id = parentFolder._id
builtUpPath = "#{builtUpPath}/#{folderName}"
ProjectLocator.findElementByPath project, builtUpPath, (err, foundFolder)=>
ProjectLocator.findElementByPath project: project, path: builtUpPath, (err, foundFolder)=>
if !foundFolder?
logger.log path:path, project_id:project._id, folderName:folderName, "making folder from mkdirp"
self.addFolder project_id, parentFolder_id, folderName, (err, newFolder, parentFolder_id)->
self.addFolder.withoutLock project_id, parentFolder_id, folderName, (err, newFolder, parentFolder_id)->
return callback(err) if err?
newFolder.parentFolder_id = parentFolder_id
previousFolders.push newFolder
@ -86,8 +103,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
!folder.filterOut
callback null, folders, lastFolder
moveEntity: (project_id, entity_id, destFolderId, entityType, callback = (error) ->) ->
ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project) ->
moveEntity: wrapWithLock (project_id, entity_id, destFolderId, entityType, callback = (error) ->) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder:true, name:true}, (err, project) ->
return callback(err) if err?
ProjectLocator.findElement {project, element_id: entity_id, type: entityType}, (err, entity, entityPath)->
return callback(err) if err?
@ -106,8 +123,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
changes = {oldDocs, newDocs, oldFiles, newFiles}
callback null, project.name, startPath, endPath, entity.rev, changes, callback
deleteEntity: (project_id, entity_id, entityType, callback) ->
ProjectGetter.getProject project_id, {name:true, rootFolder:true}, (error, project) ->
deleteEntity: wrapWithLock (project_id, entity_id, entityType, callback) ->
ProjectGetter.getProjectWithoutLock project_id, {name:true, rootFolder:true}, (error, project) ->
return callback(error) if error?
ProjectLocator.findElement {project: project, element_id: entity_id, type: entityType}, (error, entity, path) ->
return callback(error) if error?
@ -115,8 +132,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
return callback(error) if error?
callback null, entity, path, project
renameEntity: (project_id, entity_id, entityType, newName, callback) ->
ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (error, project)=>
renameEntity: wrapWithLock (project_id, entity_id, entityType, newName, callback) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder:true, name:true}, (error, project)=>
return callback(error) if error?
ProjectEntityHandler.getAllEntitiesFromProject project, (error, oldDocs, oldFiles) =>
return callback(error) if error?
@ -138,8 +155,8 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
changes = {oldDocs, newDocs, oldFiles, newFiles}
callback null, project.name, startPath, endPath, entity.rev, changes, callback
addFolder: (project_id, parentFolder_id, folderName, callback) ->
ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project) ->
addFolder: wrapWithLock (project_id, parentFolder_id, folderName, callback) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder:true, name:true}, (err, project) ->
if err?
logger.err project_id:project_id, err:err, "error getting project for add folder"
return callback(err)

9
services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee
View file

@ -18,9 +18,14 @@ ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandler')
SafePath = require './SafePath'
TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
LOCK_NAMESPACE = "sequentialProjectStructureUpdateLock"
wrapWithLock = (methodWithoutLock) ->
# This lock is used to make sure that the project structure updates are made
# sequentially. In particular the updates must be made in mongo and sent to
# the doc-updater in the same order.
methodWithLock = (project_id, args..., callback) ->
LockManager.runWithLock project_id,
LockManager.runWithLock LOCK_NAMESPACE, project_id,
(cb) -> methodWithoutLock project_id, args..., cb
callback
methodWithLock.withoutLock = methodWithoutLock
@ -267,7 +272,7 @@ module.exports = ProjectEntityUpdateHandler = self =
callback null, entity_id
deleteEntityWithPath: wrapWithLock (project_id, path, userId, callback) ->
ProjectLocator.findElementByPath project_id, path, (err, element, type)->
ProjectLocator.findElementByPath project_id: project_id, path: path, (err, element, type)->
return callback(err) if err?
return callback(new Errors.NotFoundError("project not found")) if !element?
self.deleteEntity.withoutLock project_id, element._id, type, userId, callback

56
services/web/app/coffee/Features/Project/ProjectGetter.coffee
View file

@ -5,46 +5,66 @@ ObjectId = mongojs.ObjectId
async = require "async"
Project = require("../../models/Project").Project
logger = require("logger-sharelatex")
LockManager = require("../../infrastructure/LockManager")
module.exports = ProjectGetter =
EXCLUDE_DEPTH: 8
getProjectWithoutDocLines: (project_id, callback=(error, project) ->) ->
excludes = {}
for i in [1..ProjectGetter.EXCLUDE_DEPTH]
excludes["rootFolder#{Array(i).join(".folders")}.docs.lines"] = 0
db.projects.find _id: ObjectId(project_id.toString()), excludes, (error, projects = []) ->
callback error, projects[0]
ProjectGetter.getProject project_id, excludes, callback
getProjectWithOnlyFolders: (project_id, callback=(error, project) ->) ->
excludes = {}
for i in [1..ProjectGetter.EXCLUDE_DEPTH]
excludes["rootFolder#{Array(i).join(".folders")}.docs"] = 0
excludes["rootFolder#{Array(i).join(".folders")}.fileRefs"] = 0
db.projects.find _id: ObjectId(project_id.toString()), excludes, (error, projects = []) ->
callback error, projects[0]
ProjectGetter.getProject project_id, excludes, callback
getProject: (project_id, projection, callback) ->
if !project_id?
return callback(new Error("no project_id provided"))
getProject: (query, projection, callback = (error, project) ->) ->
if !query?
return callback("no query provided")
if typeof(projection) == "function"
if typeof(projection) == "function" && !callback?
callback = projection
projection = {}
if typeof query == "string"
query = _id: ObjectId(query)
else if query instanceof ObjectId
query = _id: query
else if query?.toString().length == 24 # sometimes mongoose ids are hard to identify, this will catch them
query = _id: ObjectId(query.toString())
if typeof(projection) != "object"
return callback(new Error("projection is not an object"))
if projection?.rootFolder || Object.keys(projection).length == 0
ProjectEntityMongoUpdateHandler = require './ProjectEntityMongoUpdateHandler'
LockManager.runWithLock ProjectEntityMongoUpdateHandler.LOCK_NAMESPACE, project_id,
(cb) -> ProjectGetter.getProjectWithoutLock project_id, projection, cb
callback
else
ProjectGetter.getProjectWithoutLock project_id, projection, callback
getProjectWithoutLock: (project_id, projection, callback) ->
if !project_id?
return callback(new Error("no project_id provided"))
if typeof(projection) == "function" && !callback?
callback = projection
projection = {}
if typeof(projection) != "object"
return callback(new Error("projection is not an object"))
if typeof project_id == "string"
query = _id: ObjectId(project_id)
else if project_id instanceof ObjectId
query = _id: project_id
else if project_id?.toString().length == 24 # sometimes mongoose ids are hard to identify, this will catch them
query = _id: ObjectId(project_id.toString())
else
err = new Error("malformed get request")
logger.log query:query, err:err, type:typeof(query), "malformed get request"
logger.log project_id:project_id, err:err, type:typeof(project_id), "malformed get request"
return callback(err)
db.projects.find query, projection, (err, project)->
db.projects.find query, projection, (err, project) ->
if err?
logger.err err:err, query:query, projection:projection, "error getting project"
return callback(err)

65
services/web/app/coffee/Features/Project/ProjectLocator.coffee
View file

@ -4,7 +4,6 @@ Errors = require "../Errors/Errors"
_ = require('underscore')
logger = require('logger-sharelatex')
async = require('async')
ProjectGetter = require "./ProjectGetter"
module.exports = ProjectLocator =
findElement: (options, _callback = (err, element, path, parentFolder)->)->
@ -83,8 +82,19 @@ module.exports = ProjectLocator =
else
getRootDoc project
findElementByPath: (project_or_id, needlePath, callback = (err, foundEntity, type)->)->
findElementByPath: (options, callback = (err, foundEntity, type)->)->
{project, project_id, path} = options
if !path?
return new Error('no path provided for findElementByPath')
if project?
ProjectLocator._findElementByPathWithProject project, path, callback
else
ProjectGetter.getProject project_id, {rootFolder:true, rootDoc_id:true}, (err, project)->
return callback(err) if err?
ProjectLocator._findElementByPathWithProject project, path, callback
_findElementByPathWithProject: (project, needlePath, callback = (err, foundEntity, type)->)->
getParentFolder = (haystackFolder, foldersList, level, cb)->
if foldersList.length == 0
return cb null, haystackFolder
@ -98,7 +108,7 @@ module.exports = ProjectLocator =
else
return getParentFolder(folder, foldersList, level+1, cb)
if !found
cb("not found project_or_id: #{project_or_id} search path: #{needlePath}, folder #{foldersList[level]} could not be found")
cb("not found project: #{project._id} search path: #{needlePath}, folder #{foldersList[level]} could not be found")
getEntity = (folder, entityName, cb)->
if !entityName?
@ -119,35 +129,34 @@ module.exports = ProjectLocator =
if result?
cb null, result, type
else
cb("not found project_or_id: #{project_or_id} search path: #{needlePath}, entity #{entityName} could not be found")
cb("not found project: #{project._id} search path: #{needlePath}, entity #{entityName} could not be found")
Project.getProject project_or_id, "", (err, project)->
if err?
logger.err err:err, project_or_id:project_or_id, "error getting project for finding element"
return callback(err)
if !project?
return callback("project could not be found for finding a element #{project_or_id}")
if needlePath == '' || needlePath == '/'
return callback(null, project.rootFolder[0], "folder")
if err?
logger.err err:err, project_id:project._id, "error getting project for finding element"
return callback(err)
if !project?
return callback("project could not be found for finding a element #{project._id}")
if needlePath == '' || needlePath == '/'
return callback(null, project.rootFolder[0], "folder")
if needlePath.indexOf('/') == 0
needlePath = needlePath.substring(1)
foldersList = needlePath.split('/')
needleName = foldersList.pop()
rootFolder = project.rootFolder[0]
if needlePath.indexOf('/') == 0
needlePath = needlePath.substring(1)
foldersList = needlePath.split('/')
needleName = foldersList.pop()
rootFolder = project.rootFolder[0]
logger.log project_id:project._id, path:needlePath, foldersList:foldersList, "looking for element by path"
jobs = new Array()
jobs.push(
(cb)->
getParentFolder rootFolder, foldersList, 0, cb
)
jobs.push(
(folder, cb)->
getEntity folder, needleName, cb
)
async.waterfall jobs, callback
logger.log project_id:project._id, path:needlePath, foldersList:foldersList, "looking for element by path"
jobs = new Array()
jobs.push(
(cb)->
getParentFolder rootFolder, foldersList, 0, cb
)
jobs.push(
(folder, cb)->
getEntity folder, needleName, cb
)
async.waterfall jobs, callback
findUsersProjectByName: (user_id, projectName, callback)->
ProjectGetter.findAllUsersProjects user_id, 'name archived', (err, allProjects)->

59
services/web/app/coffee/infrastructure/LockManager.coffee
View file

@ -7,12 +7,42 @@ logger = require "logger-sharelatex"
module.exports = LockManager =
LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock
MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock
REDIS_LOCK_EXPIRY: 30 # seconds. Time until lock auto expires in redis.
REDIS_LOCK_EXPIRY: 30 # seconds. Time until lock auto expires in redis
SLOW_EXECUTION_THRESHOLD: 5000 # 5s, if execution takes longer than this then log
_blockingKey : (key)-> "lock:web:{#{key}}"
runWithLock: (namespace, id, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
tryLock : (key, callback = (err, isFree)->)->
rclient.set LockManager._blockingKey(key), "locked", "EX", LockManager.REDIS_LOCK_EXPIRY, "NX", (err, gotLock)->
# The lock can expire in redis but the process carry on. This setTimout call
# is designed to log if this happens.
#
# error is defined here so we get a useful stacktrace
lockReleased = false
slowExecutionError = new Error "slow execution during lock"
countIfExceededLockTimeout = () ->
if !lockReleased
metrics.inc "lock.#{namespace}.exceeded_lock_timeout"
logger.log "exceeded lock timeout", { namespace, id, slowExecutionError }
setTimeout countIfExceededLockTimeout, LockManager.REDIS_LOCK_EXPIRY
timer = new metrics.Timer("lock.#{namespace}")
key = "lock:web:#{namespace}:#{id}"
LockManager._getLock key, (error) ->
return callback(error) if error?
runner (error1, values...) ->
LockManager._releaseLock key, (error2) ->
lockReleased = true
timeTaken = new Date - timer.start
if timeTaken > LockManager.SLOW_EXECUTION_THRESHOLD
logger.log "slow execution during lock", { namespace, id, timeTaken, slowExecutionError }
timer.done()
error = error1 or error2
return callback(error) if error?
callback null, values...
_tryLock : (key, callback = (err, isFree)->)->
rclient.set key, "locked", "EX", LockManager.REDIS_LOCK_EXPIRY, "NX", (err, gotLock)->
return callback(err) if err?
if gotLock == "OK"
metrics.inc "lock-not-blocking"
@ -22,22 +52,22 @@ module.exports = LockManager =
logger.log key: key, redis_response: gotLock, "lock is locked"
callback err, false
getLock: (key, callback = (error) ->) ->
_getLock: (key, callback = (error) ->) ->
startTime = Date.now()
do attempt = () ->
if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
return callback(new Error("Timeout"))
LockManager.tryLock key, (error, gotLock) ->
LockManager._tryLock key, (error, gotLock) ->
return callback(error) if error?
if gotLock
callback(null)
else
setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
checkLock: (key, callback = (err, isFree)->)->
_checkLock: (key, callback = (err, isFree)->)->
multi = rclient.multi()
multi.exists LockManager._blockingKey(key)
multi.exists key
multi.exec (err, replys)->
return callback(err) if err?
exists = parseInt replys[0]
@ -48,14 +78,5 @@ module.exports = LockManager =
metrics.inc "lock-not-blocking"
callback err, true
releaseLock: (key, callback)->
rclient.del LockManager._blockingKey(key), callback
runWithLock: (key, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
LockManager.getLock key, (error) ->
return callback(error) if error?
runner (error1, values...) ->
LockManager.releaseLock key, (error2) ->
error = error1 or error2
return callback(error) if error?
callback null, values...
_releaseLock: (key, callback)->
rclient.del key, callback

83
services/web/test/acceptance/coffee/ProjectStructureMongoLockTest.coffee Normal file
View file

@ -0,0 +1,83 @@
APP_PATH = "../../../app/js"
LockManager = require "#{APP_PATH}/infrastructure/LockManager"
ProjectCreationHandler = require "#{APP_PATH}/Features/Project/ProjectCreationHandler.js"
ProjectGetter = require "#{APP_PATH}/Features/Project/ProjectGetter.js"
ProjectEntityMongoUpdateHandler = require "#{APP_PATH}/Features/Project/ProjectEntityMongoUpdateHandler.js"
UserCreator = require "#{APP_PATH}/Features/User/UserCreator.js"
expect = require("chai").expect
_ = require("lodash")
# These tests are neither acceptance tests nor unit tests. It's difficult to
# test/verify that our locking is doing what we hope.
# These tests call methods in ProjectGetter and ProjectEntityMongoUpdateHandler
# to see that they DO NOT work when a lock has been taken.
#
# It is tested that these methods DO work when the lock has not been taken in
# other acceptance tests.
describe "ProjectStructureMongoLock", ->
describe "whilst a project lock is taken", ->
before (done) ->
# We want to instantly fail if the lock is taken
LockManager.MAX_LOCK_WAIT_TIME = 1
userDetails =
holdingAccount:false,
email: 'test@example.com'
UserCreator.createNewUser userDetails, (err, user) =>
@user = user
throw err if err?
ProjectCreationHandler.createBlankProject user._id, 'locked-project', (err, project) =>
throw err if err?
@locked_project = project
@lock_key = "lock:web:#{ProjectEntityMongoUpdateHandler.LOCK_NAMESPACE}:#{project._id}"
LockManager._getLock @lock_key, done
after (done) ->
LockManager._releaseLock @lock_key, done
describe 'interacting with the locked project', ->
LOCKING_UPDATE_METHODS = ['addDoc', 'addFile', 'mkdirp', 'moveEntity', 'renameEntity', 'addFolder']
for methodName in LOCKING_UPDATE_METHODS
it "cannot call ProjectEntityMongoUpdateHandler.#{methodName}", (done) ->
method = ProjectEntityMongoUpdateHandler[methodName]
args = _.times(method.length - 2, _.constant(null))
method @locked_project._id, args, (err) ->
expect(err).to.deep.equal new Error("Timeout")
done()
it "cannot get the project without a projection", (done) ->
ProjectGetter.getProject @locked_project._id, (err) ->
expect(err).to.deep.equal new Error("Timeout")
done()
it "cannot get the project if rootFolder is in the projection", (done) ->
ProjectGetter.getProject @locked_project._id, rootFolder: true, (err) ->
expect(err).to.deep.equal new Error("Timeout")
done()
it "can get the project if rootFolder is not in the projection", (done) ->
ProjectGetter.getProject @locked_project._id, _id: true, (err, project) =>
expect(err).to.equal(null)
expect(project._id).to.deep.equal(@locked_project._id)
done()
describe 'interacting with other projects', ->
before (done) ->
ProjectCreationHandler.createBlankProject @user._id, 'unlocked-project', (err, project) =>
throw err if err?
@unlocked_project = project
done()
it "can add folders to other projects", (done) ->
ProjectEntityMongoUpdateHandler.addFolder @unlocked_project._id, @unlocked_project.rootFolder[0]._id, 'new folder', (err, folder) ->
expect(err).to.equal(null)
expect(folder).to.be.defined
done()
it "can get other projects without a projection", (done) ->
ProjectGetter.getProject @unlocked_project._id, (err, project) =>
expect(err).to.equal(null)
expect(project._id).to.deep.equal(@unlocked_project._id)
done()

8
services/web/test/acceptance/coffee/ProjectStructureTests.coffee
View file

@ -80,13 +80,13 @@ describe "ProjectStructureChanges", ->
before (done) ->
MockDocUpdaterApi.clearProjectStructureUpdates()
ProjectGetter.getProject @example_project_id, (error, projects) =>
ProjectGetter.getProject @example_project_id, (error, project) =>
throw error if error?
@owner.request.post {
uri: "project/#{@example_project_id}/doc",
json:
name: 'new.tex'
parent_folder_id: projects[0].rootFolder[0]._id
parent_folder_id: project.rootFolder[0]._id
}, (error, res, body) =>
throw error if error?
if res.statusCode < 200 || res.statusCode >= 300
@ -137,9 +137,9 @@ describe "ProjectStructureChanges", ->
describe "uploading a file", ->
before (done) ->
ProjectGetter.getProject @example_project_id, (error, projects) =>
ProjectGetter.getProject @example_project_id, (error, project) =>
throw error if error?
@root_folder_id = projects[0].rootFolder[0]._id.toString()
@root_folder_id = project.rootFolder[0]._id.toString()
done()
beforeEach () ->

4
services/web/test/acceptance/coffee/TpdsUpdateTests.coffee
View file

@ -28,10 +28,10 @@ describe "TpdsUpdateTests", ->
done()
it "should have deleted the file", (done) ->
ProjectGetter.getProject @project_id, (error, [project]) ->
ProjectGetter.getProject @project_id, (error, project) ->
throw error if error?
projectFolder = project.rootFolder[0]
for doc in projectFolder.docs
if doc.name == "main.tex"
throw new Error("expected main.tex to have been deleted")
done()
done()

29
services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee
View file

@ -35,11 +35,14 @@ describe 'ProjectEntityMongoUpdateHandler', ->
}
"../Cooldown/CooldownManager": @CooldownManager = {}
'../../models/Folder': Folder:@FolderModel
"../../infrastructure/LockManager":@LockManager =
runWithLock:
sinon.spy((namespace, id, runner, callback) -> runner(callback))
'../../models/Project': Project:@ProjectModel = {}
'./ProjectEntityHandler': @ProjectEntityHandler = {}
'./ProjectLocator': @ProjectLocator = {}
"./ProjectGetter": @ProjectGetter =
getProject: sinon.stub().yields(null, @project)
getProjectWithoutLock: sinon.stub().yields(null, @project)
afterEach ->
tk.reset()
@ -53,7 +56,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.addDoc project_id, folder_id, @doc, @callback
it 'gets the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name: true})
.should.equal true
@ -76,7 +79,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.addFile project_id, folder_id, @file, @callback
it 'gets the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name: true})
.should.equal true
@ -100,7 +103,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.replaceFile project_id, file_id, @callback
it 'gets the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name: true})
.should.equal true
@ -134,15 +137,17 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@project = _id: project_id, rootFolder: [@rootFolder]
@ProjectGetter.getProjectWithOnlyFolders = sinon.stub().yields(null, @project)
@ProjectLocator.findElementByPath = (project_id, path, cb) =>
@ProjectLocator.findElementByPath = (options, cb) =>
{path} = options
@parentFolder = {_id:"parentFolder_id_here"}
lastFolder = path.substring(path.lastIndexOf("/"))
if lastFolder.indexOf("level1") == -1
cb "level1 is not the last foler "
else
cb null, @parentFolder
@subject.addFolder = (project_id, parentFolder_id, folderName, callback) =>
callback null, {name:folderName}, @parentFolder_id
@subject.addFolder =
withoutLock: (project_id, parentFolder_id, folderName, callback) =>
callback null, {name:folderName}, @parentFolder_id
it 'should return the root folder if the path is just a slash', (done)->
path = "/"
@ -217,7 +222,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.moveEntity project_id, doc_id, folder_id, "docs", @callback
it 'should get the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name:true})
.should.equal true
@ -256,7 +261,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.deleteEntity project_id, doc_id, 'doc', @callback
it "should get the project", ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {name:true, rootFolder:true})
.should.equal true
@ -284,7 +289,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@doc = _id: doc_id, name: "old.tex", rev: 1
@folder = _id: folder_id
@ProjectGetter.getProject = sinon.stub().yields(null, @project)
@ProjectGetter.getProjectWithoutLock = sinon.stub().yields(null, @project)
@ProjectEntityHandler.getAllEntitiesFromProject = sinon.stub()
@ProjectEntityHandler.getAllEntitiesFromProject
@ -301,7 +306,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.renameEntity project_id, doc_id, 'doc', @newName, @callback
it 'should get the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name:true})
.should.equal true
@ -339,7 +344,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
@subject.addFolder project_id, folder_id, @folderName, @callback
it 'gets the project', ->
@ProjectGetter.getProject
@ProjectGetter.getProjectWithoutLock
.calledWith(project_id, {rootFolder:true, name: true})
.should.equal true

4
services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee
View file

@ -53,7 +53,7 @@ describe 'ProjectEntityUpdateHandler', ->
'../FileStore/FileStoreHandler':@FileStoreHandler
"../../infrastructure/LockManager":@LockManager =
runWithLock:
sinon.spy((key, runner, callback) -> runner(callback))
sinon.spy((namespace, id, runner, callback) -> runner(callback))
'../../models/Project': Project:@ProjectModel = {}
"./ProjectGetter": @ProjectGetter = {}
'./ProjectLocator': @ProjectLocator = {}
@ -642,7 +642,7 @@ describe 'ProjectEntityUpdateHandler', ->
it 'finds the entity', ->
@ProjectLocator.findElementByPath
.calledWith(project_id, @path)
.calledWith({project_id, @path})
.should.equal true
it 'deletes the entity', ->

36
services/web/test/unit/coffee/Project/ProjectGetterTests.coffee
View file

@ -20,8 +20,9 @@ describe "ProjectGetter", ->
"../../models/Project": Project: @Project = {}
"../Collaborators/CollaboratorsHandler": @CollaboratorsHandler = {}
"../../infrastructure/LockManager": @LockManager =
mongoTransactionLock:
runWithLock : sinon.spy((key, runner, callback) -> runner(callback))
runWithLock : sinon.spy((namespace, id, runner, callback) -> runner(callback))
'./ProjectEntityMongoUpdateHandler':
lockKey: (project_id) -> project_id
"logger-sharelatex":
err:->
log:->
@ -30,16 +31,16 @@ describe "ProjectGetter", ->
beforeEach ->
@project =
_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
@ProjectGetter.getProject = sinon.stub().yields()
describe "passing an id", ->
beforeEach ->
@ProjectGetter.getProjectWithoutDocLines @project_id, @callback
it "should call find with the project id", ->
expect(@db.projects.find.lastCall.args[0]).to.deep.equal {
_id: ObjectId(@project_id)
}
@ProjectGetter.getProject
.calledWith(@project_id)
.should.equal true
it "should exclude the doc lines", ->
excludes =
@ -51,11 +52,13 @@ describe "ProjectGetter", ->
"rootFolder.folders.folders.folders.folders.folders.docs.lines": 0
"rootFolder.folders.folders.folders.folders.folders.folders.docs.lines": 0
"rootFolder.folders.folders.folders.folders.folders.folders.folders.docs.lines": 0
@db.projects.find.calledWith(sinon.match.any, excludes)
@ProjectGetter.getProject
.calledWith(@project_id, excludes)
.should.equal true
it "should call the callback with the project", ->
@callback.calledWith(null, @project).should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "getProjectWithOnlyFolders", ->
@ -63,16 +66,16 @@ describe "ProjectGetter", ->
beforeEach ()->
@project =
_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
@ProjectGetter.getProject = sinon.stub().yields()
describe "passing an id", ->
beforeEach ->
@ProjectGetter.getProjectWithOnlyFolders @project_id, @callback
it "should call find with the project id", ->
expect(@db.projects.find.lastCall.args[0]).to.deep.equal {
_id: ObjectId(@project_id)
}
@ProjectGetter.getProject
.calledWith(@project_id)
.should.equal true
it "should exclude the docs and files linesaaaa", ->
excludes =
@ -92,11 +95,12 @@ describe "ProjectGetter", ->
"rootFolder.folders.folders.folders.folders.folders.folders.fileRefs": 0
"rootFolder.folders.folders.folders.folders.folders.folders.folders.docs": 0
"rootFolder.folders.folders.folders.folders.folders.folders.folders.fileRefs": 0
@db.projects.find.calledWith(sinon.match.any, excludes).should.equal true
@ProjectGetter.getProject
.calledWith(@project_id, excludes)
.should.equal true
it "should call the callback with the project", ->
@callback.calledWith(null, @project).should.equal true
@callback.called.should.equal true
describe "getProject", ->

78
services/web/test/unit/coffee/Project/ProjectLocatorTests.coffee
View file

@ -33,12 +33,9 @@ project.rootDoc_id = rootDoc._id
describe 'ProjectLocator', ->
beforeEach ->
Project.getProject = (project_id, fields, callback)=>
callback(null, project)
Project.findById = (project_id, callback)=>
callback(null, project)
@ProjectGetter =
@ProjectGetter =
getProject: sinon.stub().callsArgWith(2, null, project)
@locator = SandboxedModule.require modulePath, requires:
'../../models/Project':{Project:Project}
@ -162,14 +159,14 @@ describe 'ProjectLocator', ->
assert !err?
doc._id.should.equal rootDoc._id
done()
it 'should return null when the project has no rootDoc', (done) ->
project.rootDoc_id = null
@locator.findRootDoc project, (err, doc)->
assert !err?
expect(doc).to.equal null
done()
it 'should return null when the rootDoc_id no longer exists', (done) ->
project.rootDoc_id = "doesntexist"
@locator.findRootDoc project, (err, doc)->
@ -181,68 +178,71 @@ describe 'ProjectLocator', ->
it 'should take a doc path and return the element for a root level document', (done)->
path = "#{doc1.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal doc1
expect(type).to.equal "doc"
done()
it 'should take a doc path and return the element for a root level document with a starting slash', (done)->
path = "/#{doc1.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal doc1
expect(type).to.equal "doc"
done()
it 'should take a doc path and return the element for a nested document', (done)->
path = "#{subFolder.name}/#{secondSubFolder.name}/#{subSubDoc.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal subSubDoc
expect(type).to.equal "doc"
done()
it 'should take a file path and return the element for a root level document', (done)->
path = "#{file1.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal file1
expect(type).to.equal "file"
done()
it 'should take a file path and return the element for a nested document', (done)->
path = "#{subFolder.name}/#{secondSubFolder.name}/#{subSubFile.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal subSubFile
expect(type).to.equal "file"
done()
it 'should take a file path and return the element for a nested document case insenstive', (done)->
path = "#{subFolder.name.toUpperCase()}/#{secondSubFolder.name.toUpperCase()}/#{subSubFile.name.toUpperCase()}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal subSubFile
expect(type).to.equal "file"
done()
it 'should take a file path and return the element for a nested folder', (done)->
path = "#{subFolder.name}/#{secondSubFolder.name}"
@locator.findElementByPath project._id, path, (err, element, type)->
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal secondSubFolder
expect(type).to.equal "folder"
done()
it 'should take a file path and return the root folder', (done)->
@locator.findElementByPath project._id, "/", (err, element, type)->
path = "/"
@locator.findElementByPath {project, path}, (err, element, type)->
element.should.deep.equal rootFolder
expect(type).to.equal "folder"
done()
it 'should return an error if the file can not be found inside know folder', (done)->
@locator.findElementByPath project._id, "#{subFolder.name}/#{secondSubFolder.name}/exist.txt", (err, element, type)->
path = "#{subFolder.name}/#{secondSubFolder.name}/exist.txt"
@locator.findElementByPath {project, path}, (err, element, type)->
err.should.not.equal undefined
assert.equal element, undefined
expect(type).to.be.undefined
done()
it 'should return an error if the file can not be found inside unknown folder', (done)->
@locator.findElementByPath project._id, "this/does/not/exist.txt", (err, element, type)->
path = "this/does/not/exist.txt"
@locator.findElementByPath {project, path}, (err, element, type)->
err.should.not.equal undefined
assert.equal element, undefined
expect(type).to.be.undefined
@ -250,7 +250,6 @@ describe 'ProjectLocator', ->
describe "where duplicate folder exists", ->
beforeEach ->
@duplicateFolder = {name:"duplicate1", _id:"1234", folders:[{
name: "1"
@ -264,17 +263,15 @@ describe 'ProjectLocator', ->
fileRefs: []
docs: []
]
Project.getProject = sinon.stub()
Project.getProject.callsArgWith(2, null, @project)
it "should not call the callback more than once", (done)->
@locator.findElementByPath project._id, "#{@duplicateFolder.name}/#{@doc.name}", ->
path = "#{@duplicateFolder.name}/#{@doc.name}"
@locator.findElementByPath {@project, path}, ->
done() #mocha will throw exception if done called multiple times
it "should not call the callback more than once when the path is longer than 1 level below the duplicate level", (done)->
@locator.findElementByPath project._id, "#{@duplicateFolder.name}/1/main.tex", ->
path = "#{@duplicateFolder.name}/1/main.tex"
@locator.findElementByPath {@project, path}, ->
done() #mocha will throw exception if done called multiple times
describe "with a null doc", ->
@ -285,33 +282,34 @@ describe 'ProjectLocator', ->
fileRefs: []
docs: [{name:"main.tex"}, null, {name:"other.tex"}]
]
Project.getProject = sinon.stub()
Project.getProject.callsArgWith(2, null, @project)
it "should not crash with a null", (done)->
callback = sinon.stub()
@locator.findElementByPath project._id, "/other.tex", (err, element)->
path = "/other.tex"
@locator.findElementByPath {@project, path}, (err, element)->
element.name.should.equal "other.tex"
done()
describe "with a null project", ->
beforeEach ->
@project =
rootFolder:[
folders: []
fileRefs: []
docs: [{name:"main.tex"}, null, {name:"other.tex"}]
]
Project.getProject = sinon.stub()
Project.getProject.callsArgWith(2, null)
@ProjectGetter =
getProject: sinon.stub().callsArg(2)
it "should not crash with a null", (done)->
callback = sinon.stub()
@locator.findElementByPath project._id, "/other.tex", (err, element)->
path = "/other.tex"
@locator.findElementByPath {project_id: @project._id, path}, (err, element)->
expect(err).to.exist
done()
done()
describe "with a project_id", ->
it 'should take a doc path and return the element for a root level document', (done)->
path = "#{doc1.name}"
@locator.findElementByPath {project_id: project._id, path}, (err, element, type)=>
@ProjectGetter.getProject
.calledWith(project._id, {rootFolder:true, rootDoc_id: true})
.should.equal true
element.should.deep.equal doc1
expect(type).to.equal "doc"
done()
describe 'findUsersProjectByName finding a project by user_id and project name', ()->
it 'should return the project from an array case insenstive', (done)->

12
services/web/test/unit/coffee/infrastructure/LockManager/CheckingTheLock.coffee
View file

@ -2,9 +2,7 @@ sinon = require('sinon')
assert = require('assert')
path = require('path')
modulePath = path.join __dirname, '../../../../../app/js/infrastructure/LockManager.js'
project_id = 1234
doc_id = 5678
blockingKey = "lock:web:{#{doc_id}}"
lockKey = "lock:web:{#{5678}}"
SandboxedModule = require('sandboxed-module')
describe 'LockManager - checking the lock', ()->
@ -29,20 +27,20 @@ describe 'LockManager - checking the lock', ()->
it 'should check if lock exists but not set or expire', (done)->
execStub.callsArgWith(0, null, ["1"])
LockManager.checkLock doc_id, (err, docIsLocked)->
existsStub.calledWith(blockingKey).should.equal true
LockManager._checkLock lockKey, (err, docIsLocked)->
existsStub.calledWith(lockKey).should.equal true
setStub.called.should.equal false
exireStub.called.should.equal false
done()
it 'should return true if the key does not exists', (done)->
execStub.callsArgWith(0, null, "0")
LockManager.checkLock doc_id, (err, free)->
LockManager._checkLock lockKey, (err, free)->
free.should.equal true
done()
it 'should return false if the key does exists', (done)->
execStub.callsArgWith(0, null, "1")
LockManager.checkLock doc_id, (err, free)->
LockManager._checkLock lockKey, (err, free)->
free.should.equal false
done()

7
services/web/test/unit/coffee/infrastructure/LockManager/ReleasingTheLock.coffee
View file

@ -2,8 +2,7 @@ sinon = require('sinon')
assert = require('assert')
path = require('path')
modulePath = path.join __dirname, '../../../../../app/js/infrastructure/LockManager.js'
project_id = 1234
doc_id = 5678
lockKey = "lock:web:{#{5678}}"
SandboxedModule = require('sandboxed-module')
describe 'LockManager - releasing the lock', ()->
@ -20,7 +19,7 @@ describe 'LockManager - releasing the lock', ()->
LockManager = SandboxedModule.require(modulePath, requires: mocks)
it 'should put a all data into memory', (done)->
LockManager.releaseLock doc_id, ->
deleteStub.calledWith("lock:web:{#{doc_id}}").should.equal true
LockManager._releaseLock lockKey, ->
deleteStub.calledWith(lockKey).should.equal true
done()

20
services/web/test/unit/coffee/infrastructure/LockManager/getLockTests.coffee
View file

@ -20,18 +20,18 @@ describe 'LockManager - getting the lock', ->
describe "when the lock is not set", ->
beforeEach (done) ->
@LockManager.tryLock = sinon.stub().callsArgWith(1, null, true)
@LockManager.getLock @doc_id, (args...) =>
@LockManager._tryLock = sinon.stub().callsArgWith(1, null, true)
@LockManager._getLock @doc_id, (args...) =>
@callback(args...)
done()
it "should try to get the lock", ->
@LockManager.tryLock
@LockManager._tryLock
.calledWith(@doc_id)
.should.equal true
it "should only need to try once", ->
@LockManager.tryLock.callCount.should.equal 1
@LockManager._tryLock.callCount.should.equal 1
it "should return the callback", ->
@callback.calledWith(null).should.equal true
@ -41,20 +41,20 @@ describe 'LockManager - getting the lock', ->
startTime = Date.now()
tries = 0
@LockManager.LOCK_TEST_INTERVAL = 5
@LockManager.tryLock = (doc_id, callback = (error, isFree) ->) ->
@LockManager._tryLock = (doc_id, callback = (error, isFree) ->) ->
if (Date.now() - startTime < 20) or (tries < 2)
tries = tries + 1
callback null, false
else
callback null, true
sinon.spy @LockManager, "tryLock"
sinon.spy @LockManager, "_tryLock"
@LockManager.getLock @doc_id, (args...) =>
@LockManager._getLock @doc_id, (args...) =>
@callback(args...)
done()
it "should call tryLock multiple times until free", ->
(@LockManager.tryLock.callCount > 1).should.equal true
(@LockManager._tryLock.callCount > 1).should.equal true
it "should return the callback", ->
@callback.calledWith(null).should.equal true
@ -63,8 +63,8 @@ describe 'LockManager - getting the lock', ->
beforeEach (done) ->
time = Date.now()
@LockManager.MAX_LOCK_WAIT_TIME = 5
@LockManager.tryLock = sinon.stub().callsArgWith(1, null, false)
@LockManager.getLock @doc_id, (args...) =>
@LockManager._tryLock = sinon.stub().callsArgWith(1, null, false)
@LockManager._getLock @doc_id, (args...) =>
@callback(args...)
done()

11
services/web/test/unit/coffee/infrastructure/LockManager/tryLockTests.coffee
View file

@ -14,17 +14,18 @@ describe 'LockManager - trying the lock', ->
auth:->
set: @set = sinon.stub()
"settings-sharelatex":{redis:{}}
"metrics-sharelatex": inc:->
"metrics-sharelatex": inc:->
@callback = sinon.stub()
@doc_id = "doc-id-123"
@key = "lock:web:{#{@doc_id}}"
describe "when the lock is not set", ->
beforeEach ->
@set.callsArgWith(5, null, "OK")
@LockManager.tryLock @doc_id, @callback
@LockManager._tryLock @key, @callback
it "should set the lock key with an expiry if it is not set", ->
@set.calledWith("lock:web:{#{@doc_id}}", "locked", "EX", 30, "NX")
@set.calledWith(@key, "locked", "EX", 30, "NX")
.should.equal true
it "should return the callback with true", ->
@ -33,7 +34,7 @@ describe 'LockManager - trying the lock', ->
describe "when the lock is already set", ->
beforeEach ->
@set.callsArgWith(5, null, null)
@LockManager.tryLock @doc_id, @callback
@LockManager._tryLock @key, @callback
it "should return the callback with false", ->
@callback.calledWith(null, false).should.equal true