diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index b4c7ac01fb..9449c68af9 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -6,7 +6,6 @@ _ = require 'underscore' async = require 'async' logger = require('logger-sharelatex') metrics = require('../../infrastructure/Metrics') -slReqIdHelper = require('soa-req-id') redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.web) Project = require("../../models/Project").Project @@ -14,8 +13,7 @@ ProjectLocator = require('../../Features/Project/ProjectLocator') module.exports = DocumentUpdaterHandler = - queueChange : (project_id, doc_id, change, sl_req_id, callback = ()->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + queueChange : (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change doc_key = keys.combineProjectIdAndDocId(project_id, doc_id) multi = rclient.multi() @@ -26,21 +24,20 @@ module.exports = DocumentUpdaterHandler = return callback(error) if error? callback() - flushProjectToMongo: (project_id, sl_req_id, callback = (error) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log project_id:project_id, sl_req_id:sl_req_id, "flushing project from document updater" + flushProjectToMongo: (project_id, callback = (error) ->)-> + logger.log project_id:project_id, "flushing project from document updater" timer = new metrics.Timer("flushing.mongo.project") url = "#{settings.apis.documentupdater.url}/project/#{project_id}/flush" request.post url, (error, res, body)-> if error? - logger.error err: error, project_id: project_id, sl_req_id: sl_req_id, "error flushing project from document updater" + logger.error err: error, project_id: project_id, "error flushing project from document updater" return callback(error) else if res.statusCode >= 200 and res.statusCode < 300 - logger.log project_id: project_id, sl_req_id: sl_req_id, "flushed project from document updater" + logger.log project_id: project_id, "flushed project from document updater" return callback(null) else error = new Error("document updater returned a failure status code: #{res.statusCode}") - logger.error err: error, project_id: project_id, sl_req_id: sl_req_id, "document updater returned failure status code: #{res.statusCode}" + logger.error err: error, project_id: project_id, "document updater returned failure status code: #{res.statusCode}" return callback(error) flushMultipleProjectsToMongo: (project_ids, callback = (error) ->) -> @@ -51,21 +48,20 @@ module.exports = DocumentUpdaterHandler = DocumentUpdaterHandler.flushProjectToMongo project_id, callback async.series jobs, callback - flushProjectToMongoAndDelete: (project_id, sl_req_id, callback = ()->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log project_id:project_id, sl_req_id:sl_req_id, "deleting project from document updater" + flushProjectToMongoAndDelete: (project_id, callback = ()->) -> + logger.log project_id:project_id, "deleting project from document updater" timer = new metrics.Timer("delete.mongo.project") url = "#{settings.apis.documentupdater.url}/project/#{project_id}" request.del url, (error, res, body)-> if error? - logger.error err: error, project_id: project_id, sl_req_id: sl_req_id, "error deleting project from document updater" + logger.error err: error, project_id: project_id, "error deleting project from document updater" return callback(error) else if res.statusCode >= 200 and res.statusCode < 300 - logger.log project_id: project_id, sl_req_id: sl_req_id, "deleted project from document updater" + logger.log project_id: project_id, "deleted project from document updater" return callback(null) else error = new Error("document updater returned a failure status code: #{res.statusCode}") - logger.error err: error, project_id: project_id, sl_req_id: sl_req_id, "document updater returned failure status code: #{res.statusCode}" + logger.error err: error, project_id: project_id, "document updater returned failure status code: #{res.statusCode}" return callback(error) flushDocToMongo: (project_id, doc_id, callback = (error) ->) -> @@ -84,28 +80,26 @@ module.exports = DocumentUpdaterHandler = logger.error err: error, project_id: project_id, doc_id: doc_id, "document updater returned failure status code: #{res.statusCode}" return callback(error) - deleteDoc : (project_id, doc_id, sl_req_id, callback = ()->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log project_id:project_id, doc_id: doc_id, sl_req_id:sl_req_id, "deleting doc from document updater" + deleteDoc : (project_id, doc_id, callback = ()->)-> + logger.log project_id:project_id, doc_id: doc_id, "deleting doc from document updater" timer = new metrics.Timer("delete.mongo.doc") url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}" request.del url, (error, res, body)-> if error? - logger.error err: error, project_id: project_id, doc_id: doc_id, sl_req_id: sl_req_id, "error deleting doc from document updater" + logger.error err: error, project_id: project_id, doc_id: doc_id, "error deleting doc from document updater" return callback(error) else if res.statusCode >= 200 and res.statusCode < 300 - logger.log project_id: project_id, doc_id: doc_id, sl_req_id: sl_req_id, "deleted doc from document updater" + logger.log project_id: project_id, doc_id: doc_id, "deleted doc from document updater" return callback(null) else error = new Error("document updater returned a failure status code: #{res.statusCode}") - logger.error err: error, project_id: project_id, doc_id: doc_id, sl_req_id: sl_req_id, "document updater returned failure status code: #{res.statusCode}" + logger.error err: error, project_id: project_id, doc_id: doc_id, "document updater returned failure status code: #{res.statusCode}" return callback(error) - getDocument: (project_id, doc_id, fromVersion, sl_req_id, callback = (error, exists, doclines, version) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> timer = new metrics.Timer("get-document") url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}" - logger.log project_id:project_id, doc_id: doc_id, sl_req_id:sl_req_id, "getting doc from document updater" + logger.log project_id:project_id, doc_id: doc_id, "getting doc from document updater" request.get url, (error, res, body)-> timer.done() if error? @@ -122,22 +116,23 @@ module.exports = DocumentUpdaterHandler = logger.error project_id:project_id, doc_id:doc_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}" callback new Error("doc updater returned a non-success status code: #{res.statusCode}") - setDocument : (project_id, doc_id, docLines, sl_req_id, callback = (error) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + setDocument : (project_id, doc_id, docLines, source, callback = (error) ->)-> timer = new metrics.Timer("set-document") url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}" body = url: url json: lines: docLines - logger.log project_id:project_id, doc_id: doc_id, sl_req_id:sl_req_id, "setting doc in document updater" + headers: + "x-sl-update-source": source + logger.log project_id:project_id, doc_id: doc_id, source: source, "setting doc in document updater" request.post body, (error, res, body)-> timer.done() if error? logger.error err:error, url:url, project_id:project_id, doc_id:doc_id, "error setting doc in doc updater" return callback(error) if res.statusCode >= 200 and res.statusCode < 300 - logger.log project_id: project_id, doc_id: doc_id, sl_req_id: sl_req_id, "set doc in document updater" + logger.log project_id: project_id, doc_id: doc_id, "set doc in document updater" return callback(null) else logger.error project_id:project_id, doc_id:doc_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}" diff --git a/services/web/app/coffee/Features/Editor/EditorController.coffee b/services/web/app/coffee/Features/Editor/EditorController.coffee index cdc21e5cec..678d34630e 100644 --- a/services/web/app/coffee/Features/Editor/EditorController.coffee +++ b/services/web/app/coffee/Features/Editor/EditorController.coffee @@ -14,7 +14,6 @@ AuthorizationManager = require("../Security/AuthorizationManager") EditorRealTimeController = require("./EditorRealTimeController") TrackChangesManager = require("../TrackChanges/TrackChangesManager") Settings = require('settings-sharelatex') -slReqIdHelper = require('soa-req-id') async = require('async') ConnectedUsersManager = require("../ConnectedUsers/ConnectedUsersManager") _ = require('underscore') @@ -161,39 +160,33 @@ module.exports = EditorController = if callback? callback() - setDoc: (project_id, doc_id, docLines, sl_req_id, callback = (err)->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - DocumentUpdaterHandler.setDocument project_id, doc_id, docLines, (err)=> + setDoc: (project_id, doc_id, docLines, source, callback = (err)->)-> + DocumentUpdaterHandler.setDocument project_id, doc_id, docLines, source, (err)=> logger.log project_id:project_id, doc_id:doc_id, "notifying users that the document has been updated" - EditorRealTimeController.emitToRoom(project_id, "entireDocUpdate", doc_id) DocumentUpdaterHandler.flushDocToMongo project_id, doc_id, callback - addDoc: (project_id, folder_id, docName, docLines, sl_req_id, callback = (error, doc)->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addDoc: (project_id, folder_id, docName, docLines, source, callback = (error, doc)->)-> docName = docName.trim() - logger.log sl_req_id:sl_req_id, "sending new doc to project #{project_id}" + logger.log {project_id, folder_id, docName, source}, "sending new doc to project" Metrics.inc "editor.add-doc" - ProjectEntityHandler.addDoc project_id, folder_id, docName, docLines, sl_req_id, (err, doc, folder_id)=> - EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc) + ProjectEntityHandler.addDoc project_id, folder_id, docName, docLines, (err, doc, folder_id)=> + EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source) callback(err, doc) - addFile: (project_id, folder_id, fileName, path, sl_req_id, callback = (error, file)->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addFile: (project_id, folder_id, fileName, path, source, callback = (error, file)->)-> fileName = fileName.trim() - logger.log sl_req_id:sl_req_id, "sending new file to project #{project_id} with folderid: #{folder_id}" + logger.log {project_id, folder_id, fileName, path}, "sending new file to project" Metrics.inc "editor.add-file" ProjectEntityHandler.addFile project_id, folder_id, fileName, path, (err, fileRef, folder_id)=> - EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef) + EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source) callback(err, fileRef) - replaceFile: (project_id, file_id, fsPath, callback)-> - ProjectEntityHandler.replaceFile project_id, file_id, fsPath, (err) -> - callback() + replaceFile: (project_id, file_id, fsPath, source, callback = (error) ->)-> + ProjectEntityHandler.replaceFile project_id, file_id, fsPath, callback - addFolder: (project_id, folder_id, folderName, sl_req_id, callback = (error, folder)->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addFolder: (project_id, folder_id, folderName, callback = (error, folder)->)-> folderName = folderName.trim() - logger.log "sending new folder to project #{project_id}" + logger.log {project_id, folder_id, folderName}, "sending new folder to project" Metrics.inc "editor.add-folder" ProjectEntityHandler.addFolder project_id, folder_id, folderName, (err, folder, folder_id)=> @p.notifyProjectUsersOfNewFolder project_id, folder_id, folder, (error) -> @@ -209,13 +202,12 @@ module.exports = EditorController = async.series jobs, (err)-> callback err, newFolders, lastFolder - deleteEntity: (project_id, entity_id, entityType, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log project_id:project_id, entity_id:entity_id, entityType:entityType, "start delete process of entity" + deleteEntity: (project_id, entity_id, entityType, source, callback)-> + logger.log {project_id, entity_id, entityType, source}, "start delete process of entity" Metrics.inc "editor.delete-entity" ProjectEntityHandler.deleteEntity project_id, entity_id, entityType, => - logger.log sl_req_id: sl_req_id, project_id:project_id, entity_id:entity_id, entityType:entityType, "telling users entity has been deleted" - EditorRealTimeController.emitToRoom(project_id, 'removeEntity', entity_id) + logger.log project_id:project_id, entity_id:entity_id, entityType:entityType, "telling users entity has been deleted" + EditorRealTimeController.emitToRoom(project_id, 'removeEntity', entity_id, source) if callback? callback() diff --git a/services/web/app/coffee/Features/Editor/EditorHttpController.coffee b/services/web/app/coffee/Features/Editor/EditorHttpController.coffee index 846f3048a5..382cf1be24 100644 --- a/services/web/app/coffee/Features/Editor/EditorHttpController.coffee +++ b/services/web/app/coffee/Features/Editor/EditorHttpController.coffee @@ -25,7 +25,7 @@ module.exports = EditorHttpController = project_id = req.params.Project_id name = req.body.name parent_folder_id = req.body.parent_folder_id - EditorController.addDoc project_id, parent_folder_id, name, [], (error, doc) -> + EditorController.addDoc project_id, parent_folder_id, name, [], "editor", (error, doc) -> return next(error) if error? res.json doc @@ -61,7 +61,7 @@ module.exports = EditorHttpController = project_id = req.params.Project_id entity_id = req.params.entity_id entity_type = req.params.entity_type - EditorController.deleteEntity project_id, entity_id, entity_type, (error) -> + EditorController.deleteEntity project_id, entity_id, entity_type, "editor", (error) -> return next(error) if error? res.send 204 diff --git a/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee b/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee index b7070786cb..6871f8e6d2 100644 --- a/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee +++ b/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee @@ -33,7 +33,7 @@ module.exports = return callback(error) if error? self._buildTemplate "mainbasic.tex", owner_id, projectName, (error, docLines)-> return callback(error) if error? - ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "main.tex", docLines, "", (error, doc)-> + ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "main.tex", docLines, (error, doc)-> return callback(error) if error? ProjectEntityHandler.setRootDoc project._id, doc._id, (error) -> callback(error, project) @@ -46,13 +46,13 @@ module.exports = (callback) -> self._buildTemplate "main.tex", owner_id, projectName, (error, docLines)-> return callback(error) if error? - ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "main.tex", docLines, "", (error, doc)-> + ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "main.tex", docLines, (error, doc)-> return callback(error) if error? ProjectEntityHandler.setRootDoc project._id, doc._id, callback (callback) -> self._buildTemplate "references.bib", owner_id, projectName, (error, docLines)-> return callback(error) if error? - ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "references.bib", docLines, "", (error, doc)-> + ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "references.bib", docLines, (error, doc)-> callback(error) (callback) -> universePath = Path.resolve(__dirname + "/../../../templates/project_files/universe.jpg") diff --git a/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee index 34525ea90c..e786c6c606 100644 --- a/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee +++ b/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee @@ -10,16 +10,14 @@ path = require "path" async = require "async" _ = require('underscore') logger = require('logger-sharelatex') -slReqIdHelper = require('soa-req-id') docComparitor = require('./DocLinesComparitor') projectUpdateHandler = require('./ProjectUpdateHandler') DocstoreManager = require "../Docstore/DocstoreManager" ProjectGetter = require "./ProjectGetter" module.exports = ProjectEntityHandler = - getAllFolders: (project_id, sl_req_id, callback) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log sl_req_id: sl_req_id, project_id:project_id, "getting all folders for project" + getAllFolders: (project_id, callback) -> + logger.log project_id:project_id, "getting all folders for project" folders = {} processFolder = (basePath, folder) -> folders[basePath] = folder @@ -31,8 +29,7 @@ module.exports = ProjectEntityHandler = processFolder "/", project.rootFolder[0] callback null, folders - getAllDocs: (project_id, sl_req_id, callback) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + getAllDocs: (project_id, callback) -> logger.log project_id:project_id, "getting all docs for project" # We get the path and name info from the project, and the lines and @@ -45,7 +42,7 @@ module.exports = ProjectEntityHandler = for docContent in docContentsArray docContents[docContent._id] = docContent - ProjectEntityHandler.getAllFolders project_id, sl_req_id, (error, folders) -> + ProjectEntityHandler.getAllFolders project_id, (error, folders) -> return callback(error) if error? docs = {} for folderPath, folder of folders @@ -60,10 +57,9 @@ module.exports = ProjectEntityHandler = logger.log count:_.keys(docs).length, project_id:project_id, "returning docs for project" callback null, docs - getAllFiles: (project_id, sl_req_id, callback) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + getAllFiles: (project_id, callback) -> logger.log project_id:project_id, "getting all files for project" - @getAllFolders project_id, sl_req_id, (err, folders) -> + @getAllFolders project_id, (err, folders) -> return callback(err) if err? files = {} for folderPath, folder of folders @@ -71,10 +67,9 @@ module.exports = ProjectEntityHandler = files[path.join(folderPath, file.name)] = file callback null, files - flushProjectToThirdPartyDataStore: (project_id, sl_req_id, callback) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + flushProjectToThirdPartyDataStore: (project_id, callback) -> self = @ - logger.log sl_req_id: sl_req_id, project_id:project_id, "flushing project to tpds" + logger.log project_id:project_id, "flushing project to tpds" documentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler') documentUpdaterHandler.flushProjectToMongo project_id, undefined, (error) -> return callback(error) if error? @@ -87,7 +82,6 @@ module.exports = ProjectEntityHandler = do (docPath, doc) -> requests.push (callback) -> tpdsUpdateSender.addDoc {project_id:project_id, doc_id:doc._id, path:docPath, project_name:project.name, rev:doc.rev||0}, - sl_req_id, callback self.getAllFiles project_id, (error, files) -> return callback(error) if error? @@ -95,20 +89,17 @@ module.exports = ProjectEntityHandler = do (filePath, file) -> requests.push (callback) -> tpdsUpdateSender.addFile {project_id:project_id, file_id:file._id, path:filePath, project_name:project.name, rev:file.rev}, - sl_req_id, callback async.series requests, (err) -> - logger.log sl_req_id: sl_req_id, project_id:project_id, "finished flushing project to tpds" + logger.log project_id:project_id, "finished flushing project to tpds" callback(err) - setRootDoc: (project_id, newRootDocID, sl_req_id, callback = (error) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log sl_req_id: sl_req_id, project_id: project_id, rootDocId: newRootDocID, "setting root doc" + setRootDoc: (project_id, newRootDocID, callback = (error) ->)-> + logger.log project_id: project_id, rootDocId: newRootDocID, "setting root doc" Project.update {_id:project_id}, {rootDoc_id:newRootDocID}, {}, callback - unsetRootDoc: (project_id, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - logger.log sl_req_id: sl_req_id, project_id: project_id, "removing root doc" + unsetRootDoc: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "removing root doc" Project.update {_id:project_id}, {$unset: {rootDoc_id: true}}, {}, callback getDoc: (project_id, doc_id, options = {}, callback = (error, lines, rev) ->) -> @@ -117,10 +108,9 @@ module.exports = ProjectEntityHandler = options = {} DocstoreManager.getDoc project_id, doc_id, options, callback - addDoc: (project_or_id, folder_id, docName, docLines, sl_req_id, callback = (error, doc, folder_id) ->)=> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addDoc: (project_or_id, folder_id, docName, docLines, callback = (error, doc, folder_id) ->)=> Project.getProject project_or_id, "", (err, project) -> - logger.log sl_req_id: sl_req_id, project: project._id, folder_id: folder_id, doc_name: docName, "adding doc" + logger.log project: project._id, folder_id: folder_id, doc_name: docName, "adding doc" return callback(err) if err? confirmFolder project, folder_id, (folder_id)=> doc = new Doc name: docName @@ -134,7 +124,7 @@ module.exports = ProjectEntityHandler = path: result.path.fileSystem, project_name: project.name, rev: 0 - }, sl_req_id, (err) -> + }, (err) -> return callback(err) if err? callback(null, doc, folder_id) @@ -145,10 +135,9 @@ module.exports = ProjectEntityHandler = return callback(error) if error? ProjectEntityHandler.addDoc project_id, null, name, lines, callback - addFile: (project_or_id, folder_id, fileName, path, sl_req_id, callback = (error, fileRef, folder_id) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addFile: (project_or_id, folder_id, fileName, path, callback = (error, fileRef, folder_id) ->)-> Project.getProject project_or_id, "", (err, project) -> - logger.log sl_req_id: sl_req_id, project_id: project._id, folder_id: folder_id, file_name: fileName, path:path, "adding file" + logger.log project_id: project._id, folder_id: folder_id, file_name: fileName, path:path, "adding file" return callback(err) if err? confirmFolder project, folder_id, (folder_id)-> fileRef = new File name : fileName @@ -157,7 +146,7 @@ module.exports = ProjectEntityHandler = logger.err err:err, project_id: project._id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3" return callback(err) Project.putElement project._id, folder_id, fileRef, "file", (err, result)=> - tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, project_name:project.name, rev:fileRef.rev}, "sl_req_id_here", -> + tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, project_name:project.name, rev:fileRef.rev}, -> callback(err, fileRef, folder_id) replaceFile: (project_or_id, file_id, fsPath, callback)-> @@ -176,7 +165,7 @@ module.exports = ProjectEntityHandler = # between them (like waiting for the file to upload.) projectLocator.findElement findOpts, (err, fileRef, path)=> return callback(err) if err? - tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:path.fileSystem, rev:fileRef.rev+1, project_name:project.name}, "sl_req_id_here", (error) -> + tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:path.fileSystem, rev:fileRef.rev+1, project_name:project.name}, (error) -> return callback(err) if err? conditons = _id:project._id inc = {} @@ -189,10 +178,9 @@ module.exports = ProjectEntityHandler = Project.update conditons, update, {}, (err, second)-> callback() - copyFileFromExistingProject: (project_or_id, folder_id, originalProject_id, origonalFileRef, sl_req_id, callback = (error, fileRef, folder_id) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + copyFileFromExistingProject: (project_or_id, folder_id, originalProject_id, origonalFileRef, callback = (error, fileRef, folder_id) ->)-> Project.getProject project_or_id, "", (err, project) -> - logger.log sl_req_id: sl_req_id, project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "copying file in s3" + logger.log project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "copying file in s3" return callback(err) if err? confirmFolder project, folder_id, (folder_id)=> fileRef = new File name : origonalFileRef.name @@ -200,11 +188,10 @@ module.exports = ProjectEntityHandler = if err? logger.err err:err, project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "error coping file in s3" Project.putElement project._id, folder_id, fileRef, "file", (err, result)=> - tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, rev:fileRef.rev, project_name:project.name}, sl_req_id, (error) -> + tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, rev:fileRef.rev, project_name:project.name}, (error) -> callback(error, fileRef, folder_id) - mkdirp: (project_or_id, path, sl_req_id, callback = (err, newlyCreatedFolders, lastFolderInPath)->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + mkdirp: (project_or_id, path, callback = (err, newlyCreatedFolders, lastFolderInPath)->)-> self = @ folders = path.split('/') folders = _.select folders, (folder)-> @@ -225,8 +212,8 @@ module.exports = ProjectEntityHandler = builtUpPath = "#{builtUpPath}/#{folderName}" projectLocator.findElementByPath project_or_id, builtUpPath, (err, foundFolder)=> if !foundFolder? - logger.log sl_req_id: sl_req_id, path:path, project_id:project._id, folderName:folderName, "making folder from mkdirp" - @addFolder project_or_id, parentFolder_id, folderName, sl_req_id, (err, newFolder, parentFolder_id)-> + logger.log path:path, project_id:project._id, folderName:folderName, "making folder from mkdirp" + @addFolder project_or_id, parentFolder_id, folderName, (err, newFolder, parentFolder_id)-> newFolder.parentFolder_id = parentFolder_id previousFolders.push newFolder callback null, previousFolders @@ -242,13 +229,12 @@ module.exports = ProjectEntityHandler = !folder.filterOut callback(null, folders, lastFolder) - addFolder: (project_or_id, parentFolder_id, folderName, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + addFolder: (project_or_id, parentFolder_id, folderName, callback) -> folder = new Folder name: folderName Project.getProject project_or_id, "", (err, project) -> return callback(err) if err? confirmFolder project, parentFolder_id, (parentFolder_id)=> - logger.log sl_req_id: sl_req_id, project: project_or_id, parentFolder_id:parentFolder_id, folderName:folderName, "new folder added" + logger.log project: project_or_id, parentFolder_id:parentFolder_id, folderName:folderName, "new folder added" Project.putElement project._id, parentFolder_id, folder, "folder", (err, result)=> if callback? callback(err, folder, parentFolder_id) @@ -318,8 +304,7 @@ module.exports = ProjectEntityHandler = rev:entity.rev tpdsUpdateSender.moveEntity opts, callback - deleteEntity: (project_id, entity_id, entityType, sl_req_id, callback = (error) ->)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + deleteEntity: (project_id, entity_id, entityType, callback = (error) ->)-> self = @ logger.log entity_id:entity_id, type:entityType, project_id:project_id, "deleting project entity" if !entityType? @@ -332,7 +317,7 @@ module.exports = ProjectEntityHandler = return callback(error) if error? ProjectEntityHandler._cleanUpEntity project, entity, entityType, (error) -> return callback(error) if error? - tpdsUpdateSender.deleteEntity project_id:project_id, path:path.fileSystem, project_name:project.name, sl_req_id, (error) -> + tpdsUpdateSender.deleteEntity project_id:project_id, path:path.fileSystem, project_name:project.name, (error) -> return callback(error) if error? self._removeElementFromMongoArray Project, project_id, path.mongo, (error) -> return callback(error) if error? @@ -359,20 +344,17 @@ module.exports = ProjectEntityHandler = if callback? callback err - _cleanUpEntity: (project, entity, entityType, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - + _cleanUpEntity: (project, entity, entityType, callback = (error) ->) -> if(entityType.indexOf("file") != -1) - ProjectEntityHandler._cleanUpFile project, entity, sl_req_id, callback + ProjectEntityHandler._cleanUpFile project, entity, callback else if (entityType.indexOf("doc") != -1) - ProjectEntityHandler._cleanUpDoc project, entity, sl_req_id, callback + ProjectEntityHandler._cleanUpDoc project, entity, callback else if (entityType.indexOf("folder") != -1) - ProjectEntityHandler._cleanUpFolder project, entity, sl_req_id, callback + ProjectEntityHandler._cleanUpFolder project, entity, callback else callback() - _cleanUpDoc: (project, doc, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + _cleanUpDoc: (project, doc, callback = (error) ->) -> project_id = project._id.toString() doc_id = doc._id.toString() unsetRootDocIfRequired = (callback) => @@ -391,27 +373,24 @@ module.exports = ProjectEntityHandler = return callback(error) if error? callback() - _cleanUpFile: (project, file, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + _cleanUpFile: (project, file, callback = (error) ->) -> project_id = project._id.toString() file_id = file._id.toString() FileStoreHandler.deleteFile project_id, file_id, callback - _cleanUpFolder: (project, folder, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - + _cleanUpFolder: (project, folder, callback = (error) ->) -> jobs = [] for doc in folder.docs do (doc) -> - jobs.push (callback) -> ProjectEntityHandler._cleanUpDoc project, doc, sl_req_id, callback + jobs.push (callback) -> ProjectEntityHandler._cleanUpDoc project, doc, callback for file in folder.fileRefs do (file) -> - jobs.push (callback) -> ProjectEntityHandler._cleanUpFile project, file, sl_req_id, callback + jobs.push (callback) -> ProjectEntityHandler._cleanUpFile project, file, callback for childFolder in folder.folders do (childFolder) -> - jobs.push (callback) -> ProjectEntityHandler._cleanUpFolder project, childFolder, sl_req_id, callback + jobs.push (callback) -> ProjectEntityHandler._cleanUpFolder project, childFolder, callback async.series jobs, callback diff --git a/services/web/app/coffee/Features/Project/ProjectRootDocManager.coffee b/services/web/app/coffee/Features/Project/ProjectRootDocManager.coffee index 1c8202ac71..ca9dba02c6 100644 --- a/services/web/app/coffee/Features/Project/ProjectRootDocManager.coffee +++ b/services/web/app/coffee/Features/Project/ProjectRootDocManager.coffee @@ -1,11 +1,9 @@ -slReqIdHelper = require('soa-req-id') ProjectEntityHandler = require "./ProjectEntityHandler" Path = require "path" module.exports = ProjectRootDocManager = - setRootDocAutomatically: (project_id, sl_req_id, callback = (error) ->) -> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) - ProjectEntityHandler.getAllDocs project_id, sl_req_id, (error, docs) -> + setRootDocAutomatically: (project_id, callback = (error) ->) -> + ProjectEntityHandler.getAllDocs project_id, (error, docs) -> return callback(error) if error? root_doc_id = null for path, doc of docs @@ -14,7 +12,7 @@ module.exports = ProjectRootDocManager = if Path.extname(path).match(/\.R?tex$/) and match and !match[1].match /%/ root_doc_id = doc._id if root_doc_id? - ProjectEntityHandler.setRootDoc project_id, root_doc_id, sl_req_id, callback + ProjectEntityHandler.setRootDoc project_id, root_doc_id, callback else callback() diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsController.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsController.coffee index 15e2179053..aa5e888375 100644 --- a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsController.coffee +++ b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsController.coffee @@ -8,9 +8,9 @@ module.exports = mergeUpdate: (req, res)-> metrics.inc("tpds.merge-update") {filePath, user_id, projectName} = parseParams(req) - logger.log user_id:user_id, filePath:filePath, fullPath:req.params[0], projectName:projectName, sl_req_id:req.sl_req_id, "reciving update request from tpds" - tpdsUpdateHandler.newUpdate user_id, projectName, filePath, req, req.sl_req_id, (err)-> - logger.log user_id:user_id, filePath:filePath, fullPath:req.params[0], sl_req_id:req.sl_req_id, "sending response that tpdsUpdate has been completed" + logger.log user_id:user_id, filePath:filePath, fullPath:req.params[0], projectName:projectName, "reciving update request from tpds" + tpdsUpdateHandler.newUpdate user_id, projectName, filePath, req, (err)-> + logger.log user_id:user_id, filePath:filePath, fullPath:req.params[0], "sending response that tpdsUpdate has been completed" if err? logger.err err:err, user_id:user_id, filePath:filePath, "error reciving update from tpds" res.send(500) @@ -23,8 +23,8 @@ module.exports = deleteUpdate: (req, res)-> metrics.inc("tpds.delete-update") {filePath, user_id, projectName} = parseParams(req) - logger.log user_id:user_id, filePath:filePath, sl_req_id:req.sl_req_id, projectName:projectName, fullPath:req.params[0], "reciving delete request from tpds" - tpdsUpdateHandler.deleteUpdate user_id, projectName, filePath, req.sl_req_id, (err)-> + logger.log user_id:user_id, filePath:filePath, projectName:projectName, fullPath:req.params[0], "reciving delete request from tpds" + tpdsUpdateHandler.deleteUpdate user_id, projectName, filePath, (err)-> if err? logger.err err:err, user_id:user_id, filePath:filePath, "error reciving update from tpds" res.send(500) @@ -36,8 +36,9 @@ module.exports = updateProjectContents: (req, res, next = (error) ->) -> {project_id} = req.params path = "/" + req.params[0] # UpdateMerger expects leading slash - logger.log project_id: project_id, path: path, "received project contents update" - UpdateMerger.mergeUpdate project_id, path, req, (error) -> + source = req.headers["x-sl-update-source"] + logger.log project_id: project_id, path: path, source: source, "received project contents update" + UpdateMerger.mergeUpdate project_id, path, req, source, (error) -> return next(error) if error? res.send(200) req.session.destroy() @@ -45,8 +46,9 @@ module.exports = deleteProjectContents: (req, res, next = (error) ->) -> {project_id} = req.params path = "/" + req.params[0] # UpdateMerger expects leading slash - logger.log project_id: project_id, path: path, "received project contents delete request" - UpdateMerger.deleteUpdate project_id, path, (error) -> + source = req.headers["x-sl-update-source"] + logger.log project_id: project_id, path: path, source: source, "received project contents delete request" + UpdateMerger.deleteUpdate project_id, path, source, (error) -> return next(error) if error? res.send(200) req.session.destroy() diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateHandler.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateHandler.coffee index 76b802a5e3..416faaaac0 100644 --- a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateHandler.coffee +++ b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateHandler.coffee @@ -9,7 +9,7 @@ commitMessage = "Before update from Dropbox" module.exports = - newUpdate: (user_id, projectName, path, updateRequest, sl_req_id, callback)-> + newUpdate: (user_id, projectName, path, updateRequest, callback)-> getOrCreateProject = (cb)=> projectLocator.findUsersProjectByName user_id, projectName, (err, project)=> logger.log user_id:user_id, filePath:path, projectName:projectName, "handling new update from tpds" @@ -17,16 +17,16 @@ module.exports = projectCreationHandler.createBlankProject user_id, projectName, (err, project)=> # have a crack at setting the root doc after a while, on creation we won't have it yet, but should have # been sent it it within 30 seconds - setTimeout (-> ProjectRootDocManager.setRootDocAutomatically project._id, sl_req_id ), @_rootDocTimeoutLength + setTimeout (-> ProjectRootDocManager.setRootDocAutomatically project._id ), @_rootDocTimeoutLength cb err, project else cb err, project getOrCreateProject (err, project)-> - updateMerger.mergeUpdate project._id, path, updateRequest, sl_req_id, (err)-> + updateMerger.mergeUpdate project._id, path, updateRequest, (err)-> callback(err) - deleteUpdate: (user_id, projectName, path, sl_req_id, callback)-> + deleteUpdate: (user_id, projectName, path, callback)-> logger.log user_id:user_id, filePath:path, "handling delete update from tpds" projectLocator.findUsersProjectByName user_id, projectName, (err, project)-> if !project? @@ -36,7 +36,7 @@ module.exports = logger.log user_id:user_id, filePath:path, projectName:projectName, project_id:project._id, "project found for delete update, path is root so marking project as deleted" return projectDeleter.markAsDeletedByExternalSource project._id, callback else - updateMerger.deleteUpdate project._id, path, sl_req_id, (err)-> + updateMerger.deleteUpdate project._id, path, (err)-> callback(err) diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee index b341e7b574..f964720115 100644 --- a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee +++ b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee @@ -1,6 +1,5 @@ settings = require('settings-sharelatex') logger = require('logger-sharelatex') -slReqIdHelper = require('soa-req-id') path = require('path') Project = require('../../models/Project').Project keys = require('../../infrastructure/Keys') @@ -16,13 +15,12 @@ queue = require('fairy').connect(settings.redis.fairy).queue(keys.queue.web_to_t module.exports = - _addEntity: (options, sl_req_id, callback = (err)->)-> + _addEntity: (options, callback = (err)->)-> getProjectsUsersIds options.project_id, (err, user_id, allUserIds)-> - logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store" + logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, rev:options.rev, "sending file to third party data store" postOptions = method : "post" - headers: - "sl_req_id":sl_req_id + headers: sl_entity_rev:options.rev sl_project_id:options.project_id sl_all_user_ids:JSON.stringify(allUserIds) @@ -30,25 +28,22 @@ module.exports = title: "addFile" streamOrigin : options.streamOrigin queue.enqueue options.project_id, "pipeStreamFrom", postOptions, -> - logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store queued up for processing" + logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, rev:options.rev, "sending file to third party data store queued up for processing" callback() - addFile : (options, sl_req_id, callback = (err)->)-> + addFile : (options, callback = (err)->)-> metrics.inc("tpds.add-file") - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) options.streamOrigin = settings.apis.filestore.url + path.join("/project/#{options.project_id}/file/","#{options.file_id}") - @_addEntity(options, sl_req_id, callback) + @_addEntity(options, callback) - addDoc : (options, sl_req_id, callback = (err)->)-> + addDoc : (options, callback = (err)->)-> metrics.inc("tpds.add-doc") - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) options.streamOrigin = settings.apis.docstore.pubUrl + path.join("/project/#{options.project_id}/doc/","#{options.doc_id}/raw") - @_addEntity(options, sl_req_id, callback) + @_addEntity(options, callback)    - moveEntity : (options, sl_req_id, callback = (err)->)-> + moveEntity : (options, callback = (err)->)-> metrics.inc("tpds.move-entity") - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) if options.newProjectName? startPath = path.join("/#{options.project_name}/") endPath = path.join("/#{options.newProjectName}/") @@ -56,13 +51,12 @@ module.exports = startPath = mergeProjectNameAndPath(options.project_name, options.startPath) endPath = mergeProjectNameAndPath(options.project_name, options.endPath) getProjectsUsersIds options.project_id, (err, user_id, allUserIds)-> - logger.log project_id: options.project_id, user_id:user_id, startPath:startPath, endPath:endPath, uri:options.uri, sl_req_id:sl_req_id, "moving entity in third party data store" + logger.log project_id: options.project_id, user_id:user_id, startPath:startPath, endPath:endPath, uri:options.uri, "moving entity in third party data store" moveOptions = method : "put" title:"moveEntity" uri : "#{settings.apis.thirdPartyDataStore.url}/user/#{user_id}/entity" headers: - "sl_req_id":sl_req_id, sl_project_id:options.project_id, sl_entity_rev:options.rev sl_all_user_ids:JSON.stringify(allUserIds) @@ -72,15 +66,13 @@ module.exports = startPath: startPath queue.enqueue options.project_id, "standardHttpRequest", moveOptions, callback - deleteEntity : (options, sl_req_id, callback = (err)->)-> + deleteEntity : (options, callback = (err)->)-> metrics.inc("tpds.delete-entity") - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) getProjectsUsersIds options.project_id, (err, user_id, allUserIds)-> - logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, "deleting entity in third party data store" + logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, "deleting entity in third party data store" deleteOptions = method : "DELETE" - headers: - "sl_req_id":sl_req_id, + headers: sl_project_id:options.project_id sl_all_user_ids:JSON.stringify(allUserIds) uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}" diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee index 823b4ba7e0..c026f44e98 100644 --- a/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee +++ b/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee @@ -3,18 +3,16 @@ projectLocator = require('../Project/ProjectLocator') editorController = require('../Editor/EditorController') logger = require('logger-sharelatex') Settings = require('settings-sharelatex') -slReqIdHelper = require('soa-req-id') FileTypeManager = require('../Uploads/FileTypeManager') uuid = require('node-uuid') fs = require('fs') module.exports = - mergeUpdate: (project_id, path, updateRequest, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + mergeUpdate: (project_id, path, updateRequest, source, callback = (error) ->)-> self = @ - logger.log sl_req_id: sl_req_id, project_id:project_id, path:path, "merging update from tpds" + logger.log project_id:project_id, path:path, "merging update from tpds" projectLocator.findElementByPath project_id, path, (err, element)=> - logger.log sl_req_id: sl_req_id, project_id:project_id, path:path, "found element by path for merging update from tpds" + logger.log project_id:project_id, path:path, "found element by path for merging update from tpds" elementId = undefined if element? elementId = element._id @@ -24,54 +22,51 @@ module.exports = return callback() FileTypeManager.isBinary path, fsPath, (err, isFile)-> if isFile - self.p.processFile project_id, elementId, fsPath, path, callback #TODO clean up the stream written to disk here + self.p.processFile project_id, elementId, fsPath, path, source, callback #TODO clean up the stream written to disk here else - self.p.processDoc project_id, elementId, fsPath, path, sl_req_id, callback + self.p.processDoc project_id, elementId, fsPath, path, source, callback - deleteUpdate: (project_id, path, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + deleteUpdate: (project_id, path, source, callback)-> projectLocator.findElementByPath project_id, path, (err, element)-> type = 'file' if err? || !element? - logger.log sl_req_id: sl_req_id, element:element, project_id:project_id, path:path, "could not find entity for deleting, assuming it was already deleted" + logger.log element:element, project_id:project_id, path:path, "could not find entity for deleting, assuming it was already deleted" return callback() if element.lines? type = 'doc' else if element.folders? type = 'folder' - logger.log sl_req_id: sl_req_id, project_id:project_id, path:path, type:type, element:element, "processing update to delete entity from tpds" - editorController.deleteEntity project_id, element._id, type, sl_req_id, (err)-> - logger.log sl_req_id: sl_req_id, project_id:project_id, path:path, "finished processing update to delete entity from tpds" + logger.log project_id:project_id, path:path, type:type, element:element, "processing update to delete entity from tpds" + editorController.deleteEntity project_id, element._id, type, source, (err)-> + logger.log project_id:project_id, path:path, "finished processing update to delete entity from tpds" callback() p: - processDoc: (project_id, doc_id, fsPath, path, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + processDoc: (project_id, doc_id, fsPath, path, source, callback)-> readFileIntoTextArray fsPath, (err, docLines)-> if err? logger.err project_id:project_id, doc_id:doc_id, fsPath:fsPath, "error reading file into text array for process doc update" return callback(err) - logger.log docLines:docLines, doc_id:doc_id, project_id:project_id, sl_req_id:sl_req_id, "processing doc update from tpds" + logger.log docLines:docLines, doc_id:doc_id, project_id:project_id, "processing doc update from tpds" if doc_id? - editorController.setDoc project_id, doc_id, docLines, sl_req_id, (err)-> + editorController.setDoc project_id, doc_id, docLines, source, (err)-> callback() else setupNewEntity project_id, path, (err, folder, fileName)-> - editorController.addDoc project_id, folder._id, fileName, docLines, sl_req_id, (err)-> + editorController.addDoc project_id, folder._id, fileName, docLines, source, (err)-> callback() - processFile: (project_id, file_id, fsPath, path, sl_req_id, callback)-> - {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) + processFile: (project_id, file_id, fsPath, path, source, callback)-> finish = (err)-> - logger.log sl_req_id: sl_req_id, project_id:project_id, file_id:file_id, path:path, "completed processing file update from tpds" + logger.log project_id:project_id, file_id:file_id, path:path, "completed processing file update from tpds" callback(err) - logger.log sl_req_id: sl_req_id, project_id:project_id, file_id:file_id, path:path, "processing file update from tpds" + logger.log project_id:project_id, file_id:file_id, path:path, "processing file update from tpds" setupNewEntity project_id, path, (err, folder, fileName) => if file_id? - editorController.replaceFile project_id, file_id, fsPath, finish + editorController.replaceFile project_id, file_id, fsPath, source, finish else - editorController.addFile project_id, folder._id, fileName, fsPath, finish + editorController.addFile project_id, folder._id, fileName, fsPath, source, finish writeStreamToDisk: (project_id, file_id, stream, callback = (err, fsPath)->)-> if !file_id? diff --git a/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee b/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee index 0562255fa2..f24770ff8f 100644 --- a/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee +++ b/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee @@ -11,7 +11,7 @@ module.exports = FileSystemImportManager = return callback(error) if error? content = content.replace(/\r/g, "") lines = content.split("\n") - EditorController.addDoc project_id, folder_id, name, lines, callback + EditorController.addDoc project_id, folder_id, name, lines, "upload", callback addFile: (project_id, folder_id, name, path, replace, callback = (error, file)-> )-> if replace @@ -24,14 +24,14 @@ module.exports = FileSystemImportManager = existingFile = fileRef break if existingFile? - EditorController.replaceFile project_id, existingFile._id, path, callback + EditorController.replaceFile project_id, existingFile._id, path, "upload", callback else - EditorController.addFile project_id, folder_id, name, path, callback + EditorController.addFile project_id, folder_id, name, path, "upload", callback else - EditorController.addFile project_id, folder_id, name, path, callback + EditorController.addFile project_id, folder_id, name, path, "upload", callback addFolder: (project_id, folder_id, name, path, replace, callback = (error)-> ) -> - EditorController.addFolder project_id, folder_id, name, (error, new_folder) => + EditorController.addFolder project_id, folder_id, name, "upload", (error, new_folder) => return callback(error) if error? @addFolderContents project_id, new_folder._id, path, replace, (error) -> return callback(error) if error? diff --git a/services/web/app/coffee/infrastructure/Server.coffee b/services/web/app/coffee/infrastructure/Server.coffee index 45d1ad889b..9b92c3d2aa 100644 --- a/services/web/app/coffee/infrastructure/Server.coffee +++ b/services/web/app/coffee/infrastructure/Server.coffee @@ -6,7 +6,6 @@ metrics = require('./Metrics') crawlerLogger = require('./CrawlerLogger') expressLocals = require('./ExpressLocals') socketIoConfig = require('./SocketIoConfig') -soareqid = require('soa-req-id') Router = require('../router') metrics.inc("startup") SessionSockets = require('session.socket.io') diff --git a/services/web/package.json b/services/web/package.json index 00488fc2c5..9b43687fd0 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -37,7 +37,6 @@ "sanitizer": "0.1.1", "session.socket.io": "0.1.4", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", - "soa-req-id": "git+https://github.com/sharelatex/soa-req-id.git#v1.0.0", "socket.io": "0.9.16", "translations-sharelatex": "git+https://github.com/sharelatex/translations-sharelatex.git#master", "underscore": "1.6.0", diff --git a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee index 484c78b6ad..3b5c4fc54e 100644 --- a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee @@ -213,11 +213,12 @@ describe 'Flushing documents :', -> describe "setDocument", -> beforeEach -> @callback = sinon.stub() + @source = "dropbox" describe "successfully", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 204}, "") - @handler.setDocument @project_id, @doc_id, @lines, @callback + @handler.setDocument @project_id, @doc_id, @lines, @source, @callback it 'should set the document in the document updater', -> url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}" @@ -226,6 +227,8 @@ describe 'Flushing documents :', -> url: url json: lines: @lines + headers: + "x-sl-update-source": @source }) .should.equal true @@ -235,7 +238,7 @@ describe 'Flushing documents :', -> describe "when the document updater API returns an error", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null) - @handler.setDocument @project_id, @doc_id, @lines, @callback + @handler.setDocument @project_id, @doc_id, @lines, @source, @callback it "should return an error to the callback", -> @callback.calledWith(@error).should.equal true @@ -243,7 +246,7 @@ describe 'Flushing documents :', -> describe "when the document updater returns a failure error code", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") - @handler.setDocument @project_id, @doc_id, @lines, @callback + @handler.setDocument @project_id, @doc_id, @lines, @source, @callback it "should return the callback with an error", -> @callback diff --git a/services/web/test/UnitTests/coffee/Editor/EditorControllerTests.coffee b/services/web/test/UnitTests/coffee/Editor/EditorControllerTests.coffee index d7e754605a..8ecf946901 100644 --- a/services/web/test/UnitTests/coffee/Editor/EditorControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/Editor/EditorControllerTests.coffee @@ -14,6 +14,7 @@ describe "EditorController", -> @doc_id = "test-doc-id" + @source = "dropbox" @projectModelView = "projectModelView" @@ -410,38 +411,32 @@ describe "EditorController", -> @ProjectOptionsHandler.setSpellCheckLanguage.args[0][2]() - describe 'set document', -> + describe 'setDoc', -> beforeEach -> @docLines = ["foo", "bar"] @DocumentUpdaterHandler.flushDocToMongo = sinon.stub().callsArg(2) - @DocumentUpdaterHandler.setDocument = sinon.stub().callsArg(3) - @EditorRealTimeController.emitToRoom = sinon.stub() + @DocumentUpdaterHandler.setDocument = sinon.stub().callsArg(4) it 'should send the document to the documentUpdaterHandler', (done)-> - @DocumentUpdaterHandler.setDocument = sinon.stub().withArgs(@project_id, @doc_id, @docLines).callsArg(3) - @EditorController.setDoc @project_id, @doc_id, @docLines, (err)-> - done() - - it 'should send the update to the connected users', (done)-> - @EditorController.setDoc @project_id, @doc_id, @docLines, (err)=> - @EditorRealTimeController.emitToRoom.calledWith(@project_id, "entireDocUpdate", @doc_id).should.equal true + @DocumentUpdaterHandler.setDocument = sinon.stub().withArgs(@project_id, @doc_id, @docLines, @source).callsArg(4) + @EditorController.setDoc @project_id, @doc_id, @docLines, @source, (err)-> done() it 'should send the new doc lines to the doucment updater', (done)-> @DocumentUpdaterHandler.setDocument = -> - mock = sinon.mock(@DocumentUpdaterHandler).expects("setDocument").withArgs(@project_id, @doc_id, @docLines).once().callsArg(3) + mock = sinon.mock(@DocumentUpdaterHandler).expects("setDocument").withArgs(@project_id, @doc_id, @docLines, @source).once().callsArg(4) - @EditorController.setDoc @project_id, @doc_id, @docLines, (err)=> + @EditorController.setDoc @project_id, @doc_id, @docLines, @source, (err)=> mock.verify() done() it 'should flush the doc to mongo', (done)-> - @EditorController.setDoc @project_id, @doc_id, @docLines, (err)=> + @EditorController.setDoc @project_id, @doc_id, @docLines, @source, (err)=> @DocumentUpdaterHandler.flushDocToMongo.calledWith(@project_id, @doc_id).should.equal true done() - describe 'add doc', -> + describe 'addDoc', -> beforeEach -> @ProjectEntityHandler.addDoc = ()-> @EditorRealTimeController.emitToRoom = sinon.stub() @@ -453,24 +448,24 @@ describe "EditorController", -> @docLines = ["1234","dskl"] it 'should add the doc using the project entity handler', (done)-> - mock = sinon.mock(@ProjectEntityHandler).expects("addDoc").withArgs(@project_id, @folder_id, @docName, @docLines).callsArg(5) + mock = sinon.mock(@ProjectEntityHandler).expects("addDoc").withArgs(@project_id, @folder_id, @docName, @docLines).callsArg(4) - @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, "", -> + @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, @source, -> mock.verify() done() it 'should send the update out to the users in the project', (done)-> - @ProjectEntityHandler.addDoc = sinon.stub().callsArgWith(5, null, @doc, @folder_id) + @ProjectEntityHandler.addDoc = sinon.stub().callsArgWith(4, null, @doc, @folder_id) - @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, "", => + @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, @source, => @EditorRealTimeController.emitToRoom - .calledWith(@project_id, "reciveNewDoc", @folder_id, @doc) + .calledWith(@project_id, "reciveNewDoc", @folder_id, @doc, @source) .should.equal true done() it 'should return the doc to the callback', (done) -> - @ProjectEntityHandler.addDoc = sinon.stub().callsArgWith(5, null, @doc, @folder_id) - @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, "", (error, doc) => + @ProjectEntityHandler.addDoc = sinon.stub().callsArgWith(4, null, @doc, @folder_id) + @EditorController.addDoc @project_id, @folder_id, @docName, @docLines, @source, (error, doc) => doc.should.equal @doc done() @@ -487,22 +482,22 @@ describe "EditorController", -> it 'should add the folder using the project entity handler', (done)-> mock = sinon.mock(@ProjectEntityHandler).expects("addFile").withArgs(@project_id).callsArg(4) - @EditorController.addFile @project_id, @folder_id, @fileName, @stream, => + @EditorController.addFile @project_id, @folder_id, @fileName, @stream, @source, => mock.verify() done() it 'should send the update of a new folder out to the users in the project', (done)-> @ProjectEntityHandler.addFile = sinon.stub().callsArgWith(4, null, @file, @folder_id) - @EditorController.addFile @project_id, @folder_id, @fileName, @stream, => + @EditorController.addFile @project_id, @folder_id, @fileName, @stream, @source, => @EditorRealTimeController.emitToRoom - .calledWith(@project_id, "reciveNewFile", @folder_id, @file) + .calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source) .should.equal true done() it "should return the file in the callback", (done) -> @ProjectEntityHandler.addFile = sinon.stub().callsArgWith(4, null, @file, @folder_id) - @EditorController.addFile @project_id, @folder_id, @fileName, @stream, (error, file) => + @EditorController.addFile @project_id, @folder_id, @fileName, @stream, @source, (error, file) => file.should.equal @file done() @@ -515,7 +510,7 @@ describe "EditorController", -> it 'should send the replace file message to the editor controller', (done)-> @ProjectEntityHandler.replaceFile = sinon.stub().callsArgWith(3) - @EditorController.replaceFile @project_id, @file_id, @fsPath, => + @EditorController.replaceFile @project_id, @file_id, @fsPath, @source, => @ProjectEntityHandler.replaceFile.calledWith(@project_id, @file_id, @fsPath).should.equal true done() @@ -585,14 +580,14 @@ describe "EditorController", -> it 'should delete the folder using the project entity handler', (done)-> mock = sinon.mock(@ProjectEntityHandler).expects("deleteEntity").withArgs(@project_id, @entity_id, @type).callsArg(3) - @EditorController.deleteEntity @project_id, @entity_id, @type, -> + @EditorController.deleteEntity @project_id, @entity_id, @type, @source, -> mock.verify() done() it 'notify users an entity has been deleted', (done)-> - @EditorController.deleteEntity @project_id, @entity_id, @type, => + @EditorController.deleteEntity @project_id, @entity_id, @type, @source, => @EditorRealTimeController.emitToRoom - .calledWith(@project_id, "removeEntity", @entity_id) + .calledWith(@project_id, "removeEntity", @entity_id, @source) .should.equal true done() diff --git a/services/web/test/UnitTests/coffee/Editor/EditorHttpControllerTests.coffee b/services/web/test/UnitTests/coffee/Editor/EditorHttpControllerTests.coffee index fe00e13cb3..a2081a9464 100644 --- a/services/web/test/UnitTests/coffee/Editor/EditorHttpControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/Editor/EditorHttpControllerTests.coffee @@ -55,12 +55,12 @@ describe "EditorHttpController", -> @req.body = name: @name = "doc-name" parent_folder_id: @parent_folder_id - @EditorController.addDoc = sinon.stub().callsArgWith(4, null, @doc) + @EditorController.addDoc = sinon.stub().callsArgWith(5, null, @doc) @EditorHttpController.addDoc @req, @res it "should call EditorController.addDoc", -> @EditorController.addDoc - .calledWith(@project_id, @parent_folder_id, @name, []) + .calledWith(@project_id, @parent_folder_id, @name, [], "editor") .should.equal true it "should send the doc back as JSON", -> @@ -147,12 +147,12 @@ describe "EditorHttpController", -> Project_id: @project_id entity_id: @entity_id = "entity-id-123" entity_type: @entity_type = "entity-type" - @EditorController.deleteEntity = sinon.stub().callsArg(3) + @EditorController.deleteEntity = sinon.stub().callsArg(4) @EditorHttpController.deleteEntity @req, @res it "should call EditorController.deleteEntity", -> @EditorController.deleteEntity - .calledWith(@project_id, @entity_id, @entity_type) + .calledWith(@project_id, @entity_id, @entity_type, "editor") .should.equal true it "should send back a success response", -> diff --git a/services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee b/services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee index e9de417c02..d6004bffa2 100644 --- a/services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee @@ -30,7 +30,7 @@ describe 'ProjectCreationHandler', -> constructor:(options)-> {@name} = options @ProjectEntityHandler = - addDoc: sinon.stub().callsArgWith(5, null, {_id: docId}) + addDoc: sinon.stub().callsArgWith(4, null, {_id: docId}) addFile: sinon.stub().callsArg(4) setRootDoc: sinon.stub().callsArg(2) diff --git a/services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee b/services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee index 1426f6943f..bff3542419 100644 --- a/services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee @@ -20,9 +20,9 @@ describe 'ProjectEntityHandler', -> uploadFileFromDisk:(project_id, fileRef, localImagePath, callback)->callback() copyFile: sinon.stub().callsArgWith(4, null) @tpdsUpdateSender = - addDoc:sinon.stub().callsArg(2) - addFile:sinon.stub().callsArg(2) - addFolder:sinon.stub().callsArg(2) + addDoc:sinon.stub().callsArg(1) + addFile:sinon.stub().callsArg(1) + addFolder:sinon.stub().callsArg(1) @rootFolder = _id:rootFolderId, folders:[ @@ -84,7 +84,7 @@ describe 'ProjectEntityHandler', -> cb "level1 is not the last foler " else cb null, @parentFolder - @ProjectEntityHandler.addFolder = (project_id, parentFolder_id, folderName, sl_req_id, callback)=> + @ProjectEntityHandler.addFolder = (project_id, parentFolder_id, folderName, callback)=> callback null, {name:folderName}, @parentFolder_id it 'should return the root folder if the path is just a slash', (done)-> @@ -134,10 +134,10 @@ describe 'ProjectEntityHandler', -> lastFolder.parentFolder_id.should.equal @parentFolder_id done() - describe 'deleting an element', -> + describe 'deleteEntity', -> entity_id = "4eecaffcbffa66588e000009" beforeEach -> - @tpdsUpdateSender.deleteEntity = sinon.stub().callsArg(2) + @tpdsUpdateSender.deleteEntity = sinon.stub().callsArg(1) @ProjectEntityHandler._removeElementFromMongoArray = sinon.stub().callsArg(3) @ProjectEntityHandler._cleanUpEntity = sinon.stub().callsArg(3) @path = mongo: "mongo.path", fileSystem: "/file/system/path" @@ -186,7 +186,7 @@ describe 'ProjectEntityHandler', -> describe "a doc", -> beforeEach (done) -> - @ProjectEntityHandler._cleanUpDoc = sinon.stub().callsArg(3) + @ProjectEntityHandler._cleanUpDoc = sinon.stub().callsArg(2) @ProjectEntityHandler._cleanUpEntity @project, @entity = {_id: @entity_id}, 'doc', done it "should clean up the doc", -> @@ -205,8 +205,8 @@ describe 'ProjectEntityHandler', -> fileRefs: [ @file2 = { _id: "file-id-2" } ] docs: [ @doc2 = { _id: "doc-id-2" } ] - @ProjectEntityHandler._cleanUpDoc = sinon.stub().callsArg(3) - @ProjectEntityHandler._cleanUpFile = sinon.stub().callsArg(3) + @ProjectEntityHandler._cleanUpDoc = sinon.stub().callsArg(2) + @ProjectEntityHandler._cleanUpFile = sinon.stub().callsArg(2) @ProjectEntityHandler._cleanUpEntity @project, @folder, "folder", done it "should clean up all sub files", -> @@ -406,7 +406,7 @@ describe 'ProjectEntityHandler', -> @ProjectModel.putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:@path}}) @callback = sinon.stub() - @tpdsUpdateSender.addDoc = sinon.stub().callsArg(2) + @tpdsUpdateSender.addDoc = sinon.stub().callsArg(1) @DocstoreManager.updateDoc = sinon.stub().callsArgWith(3, null, true, 0) @ProjectEntityHandler.addDoc project_id, folder_id, @name, @lines, @callback @@ -465,7 +465,7 @@ describe 'ProjectEntityHandler', -> it "should call the callback with the new folder and doc", -> @callback.calledWith(null, @doc, @folder_id).should.equal true - describe 'adding file', -> + describe 'addFile', -> fileName = "something.jpg" beforeEach -> @filePath = "somewhere" @@ -786,8 +786,8 @@ describe 'ProjectEntityHandler', -> } @ProjectModel.findById = sinon.stub().callsArgWith(1, null, @project) @documentUpdaterHandler.flushProjectToMongo = sinon.stub().callsArg(2) - @tpdsUpdateSender.addDoc = sinon.stub().callsArg(2) - @tpdsUpdateSender.addFile = sinon.stub().callsArg(2) + @tpdsUpdateSender.addDoc = sinon.stub().callsArg(1) + @tpdsUpdateSender.addFile = sinon.stub().callsArg(1) @docs = { "/doc/one": @doc1 = { _id: "mock-doc-1", lines: ["one"], rev: 5 } "/doc/two": @doc2 = { _id: "mock-doc-2", lines: ["two"], rev: 6 } diff --git a/services/web/test/UnitTests/coffee/Project/ProjectRootDocManagerTests.coffee b/services/web/test/UnitTests/coffee/Project/ProjectRootDocManagerTests.coffee index 3bcec72b0f..fbd12c6ed4 100644 --- a/services/web/test/UnitTests/coffee/Project/ProjectRootDocManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Project/ProjectRootDocManagerTests.coffee @@ -8,6 +8,7 @@ describe 'ProjectRootDocManager', -> beforeEach -> @project_id = "project-123" @sl_req_id = "sl-req-id-123" + @callback = sinon.stub() @ProjectRootDocManager = SandboxedModule.require modulePath, requires: "./ProjectEntityHandler" : @ProjectEntityHandler = {} @@ -21,9 +22,9 @@ describe 'ProjectRootDocManager', -> "/main.tex": _id: "doc-id-2" lines: ["\\documentclass{article}", "\\input{chapter1}"] - @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(2, null, @docs) - @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(3) - @ProjectRootDocManager.setRootDocAutomatically @project_id, @sl_req_id, @callback + @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs) + @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(2) + @ProjectRootDocManager.setRootDocAutomatically @project_id, @callback it "should check the docs of the project", -> @ProjectEntityHandler.getAllDocs.calledWith(@project_id) @@ -36,12 +37,6 @@ describe 'ProjectRootDocManager', -> it "should call the callback", -> @callback.called.should.equal true - it "should pass on the sl_req_id", -> - @ProjectEntityHandler.getAllDocs.calledWith(sinon.match.any, @sl_req_id) - .should.equal true - @ProjectEntityHandler.setRootDoc.calledWith(sinon.match.any, sinon.match.any, @sl_req_id) - .should.equal true - describe "when the root doc is an Rtex file", -> beforeEach -> @docs = @@ -51,9 +46,9 @@ describe 'ProjectRootDocManager', -> "/main.Rtex": _id: "doc-id-2" lines: ["\\documentclass{article}", "\\input{chapter1}"] - @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(2, null, @docs) - @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(3) - @ProjectRootDocManager.setRootDocAutomatically @project_id, @sl_req_id, @callback + @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs) + @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(2) + @ProjectRootDocManager.setRootDocAutomatically @project_id, @callback it "should set the root doc to the doc containing a documentclass", -> @ProjectEntityHandler.setRootDoc.calledWith(@project_id, "doc-id-2") @@ -68,8 +63,8 @@ describe 'ProjectRootDocManager', -> "/style.bst": _id: "doc-id-2" lines: ["%Example: \\documentclass{article}"] - @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(2, null, @docs) - @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(3) + @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs) + @ProjectEntityHandler.setRootDoc = sinon.stub().callsArgWith(2) @ProjectRootDocManager.setRootDocAutomatically @project_id, @callback it "should not set the root doc to the doc containing a documentclass", -> diff --git a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsControllerTests.coffee b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsControllerTests.coffee index 12bc57df8d..e0b424e571 100644 --- a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsControllerTests.coffee @@ -25,7 +25,7 @@ describe 'TpdsController', -> params:{0:path, "user_id":@user_id} session: destroy:-> - @TpdsUpdateHandler.newUpdate = sinon.stub().callsArg(5) + @TpdsUpdateHandler.newUpdate = sinon.stub().callsArg(4) res = send: => @TpdsUpdateHandler.newUpdate.calledWith(@user_id, "projectName","/here.txt", req).should.equal true done() @@ -38,7 +38,7 @@ describe 'TpdsController', -> params:{0:path, "user_id":@user_id} session: destroy:-> - @TpdsUpdateHandler.deleteUpdate = sinon.stub().callsArg(4) + @TpdsUpdateHandler.deleteUpdate = sinon.stub().callsArg(3) res = send: => @TpdsUpdateHandler.deleteUpdate.calledWith(@user_id, "projectName", "/here.txt").should.equal true done() @@ -72,13 +72,15 @@ describe 'TpdsController', -> describe 'updateProjectContents', -> beforeEach -> - @UpdateMerger.mergeUpdate = sinon.stub().callsArg(3) + @UpdateMerger.mergeUpdate = sinon.stub().callsArg(4) @req = params: 0: @path = "chapters/main.tex" project_id: @project_id = "project-id-123" session: destroy: sinon.stub() + headers: + "x-sl-update-source": @source = "github" @res = send: sinon.stub() @@ -86,7 +88,7 @@ describe 'TpdsController', -> it "should merge the update", -> @UpdateMerger.mergeUpdate - .calledWith(@project_id, "/" + @path, @req) + .calledWith(@project_id, "/" + @path, @req, @source) .should.equal true it "should return a success", -> @@ -97,13 +99,15 @@ describe 'TpdsController', -> describe 'deleteProjectContents', -> beforeEach -> - @UpdateMerger.deleteUpdate = sinon.stub().callsArg(2) + @UpdateMerger.deleteUpdate = sinon.stub().callsArg(3) @req = params: 0: @path = "chapters/main.tex" project_id: @project_id = "project-id-123" session: destroy: sinon.stub() + headers: + "x-sl-update-source": @source = "github" @res = send: sinon.stub() @@ -111,7 +115,7 @@ describe 'TpdsController', -> it "should delete the file", -> @UpdateMerger.deleteUpdate - .calledWith(@project_id, "/" + @path) + .calledWith(@project_id, "/" + @path, @source) .should.equal true it "should return a success", -> diff --git a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateHandlerTests.coffee b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateHandlerTests.coffee index 3a6e31809f..5ff8b509be 100644 --- a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateHandlerTests.coffee @@ -3,12 +3,12 @@ sinon = require('sinon') require('chai').should() modulePath = require('path').join __dirname, '../../../../app/js/Features/ThirdPartyDataStore/TpdsUpdateHandler.js' -describe 'third party data store reciver :', -> +describe 'TpdsUpdateHandler', -> beforeEach -> @requestQueuer = {} @updateMerger = - deleteUpdate: (user_id, path, sl_req_id, cb)->cb() - mergeUpdate:(user_id, path, update, sl_req_id, cb)->cb() + deleteUpdate: (user_id, path, cb)->cb() + mergeUpdate:(user_id, path, update, cb)->cb() @editorController = {} @project_id = "dsjajilknaksdn" @project = {_id:@project_id, name:"projectNameHere"} @@ -26,21 +26,22 @@ describe 'third party data store reciver :', -> "../Project/ProjectRootDocManager" : @rootDocManager 'logger-sharelatex': log:-> @user_id = "dsad29jlkjas" + @source = "dropbox" describe 'getting an update', -> it 'should send the update to the update merger', (done)-> path = "/path/here" update = {} @updateMerger.mergeUpdate = sinon.stub() - @updateMerger.mergeUpdate.withArgs(@project_id, path, update).callsArg(4) - @handler.newUpdate @user_id, @project.name, path, update, "", => + @updateMerger.mergeUpdate.withArgs(@project_id, path, update).callsArg(3) + @handler.newUpdate @user_id, @project.name, path, update, => @projectCreationHandler.createBlankProject.called.should.equal false done() it 'should create a new project if one does not already exit', (done)-> @projectLocator.findUsersProjectByName = sinon.stub().callsArgWith(2) path = "/" - @handler.newUpdate @user_id, @project.name, path, {}, "", => + @handler.newUpdate @user_id, @project.name, path, {}, => @projectCreationHandler.createBlankProject.calledWith(@user_id, @project.name).should.equal true done() @@ -48,7 +49,7 @@ describe 'third party data store reciver :', -> @projectLocator.findUsersProjectByName = sinon.stub().callsArgWith(2) @handler._rootDocTimeoutLength = 0 path = "/" - @handler.newUpdate @user_id, @project.name, path, {}, "", => + @handler.newUpdate @user_id, @project.name, path, {}, => setTimeout (=> @rootDocManager.setRootDocAutomatically.calledWith(@project._id).should.equal true done() @@ -60,16 +61,16 @@ describe 'third party data store reciver :', -> it 'should call deleteEntity in the collaberation manager', (done)-> path = "/delete/this" update = {} - @updateMerger.deleteUpdate = sinon.stub().callsArg(3) + @updateMerger.deleteUpdate = sinon.stub().callsArg(2) - @handler.deleteUpdate @user_id, @project.name,path, "sl_req_id", => + @handler.deleteUpdate @user_id, @project.name, path, => @projectDeleter.markAsDeletedByExternalSource.calledWith(@project._id).should.equal false @updateMerger.deleteUpdate.calledWith(@project_id, path).should.equal true done() it 'should mark the project as deleted by external source if path is a single slash', (done)-> path = "/" - @handler.deleteUpdate @user_id, @project.name, path, "sl_req_id", => + @handler.deleteUpdate @user_id, @project.name, path, => @projectDeleter.markAsDeletedByExternalSource.calledWith(@project._id).should.equal true done() diff --git a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee index 4088c0791e..08f330e028 100644 --- a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee +++ b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee @@ -23,6 +23,7 @@ describe 'UpdateMerger :', -> log: -> err: -> @project_id = "project_id_here" + @source = "dropbox" @update = new BufferedStream() @update.headers = {} @@ -38,7 +39,7 @@ describe 'UpdateMerger :', -> @projectLocator.findElementByPath = sinon.spy() - @updateMerger.mergeUpdate @project_id, @path, @update, "", => + @updateMerger.mergeUpdate @project_id, @path, @update, @source, => @projectLocator.findElementByPath.calledWith(@project_id, @path).should.equal true done() @@ -48,7 +49,7 @@ describe 'UpdateMerger :', -> filePath = ".gitignore" @projectLocator.findElementByPath = (_, __, cb)->cb(null, {_id:"id"}) @FileTypeManager.shouldIgnore.callsArgWith(1, null, true) - @updateMerger.mergeUpdate @project_id, filePath, @update, "", => + @updateMerger.mergeUpdate @project_id, filePath, @update, @source, => @FileTypeManager.isBinary.called.should.equal false @FileTypeManager.shouldIgnore.calledWith(filePath).should.equal true done() @@ -62,9 +63,9 @@ describe 'UpdateMerger :', -> @updateMerger.p.processDoc = sinon.stub().callsArgWith(5) filePath = "/folder/doc.tex" - @updateMerger.mergeUpdate @project_id, filePath, @update, "", => + @updateMerger.mergeUpdate @project_id, filePath, @update, @source, => @FileTypeManager.isBinary.calledWith(filePath, @fsPath).should.equal true - @updateMerger.p.processDoc.calledWith(@project_id, doc_id, @fsPath).should.equal true + @updateMerger.p.processDoc.calledWith(@project_id, doc_id, @fsPath, filePath, @source).should.equal true done() it 'should process update as file when it is not a doc', (done)-> @@ -72,11 +73,11 @@ describe 'UpdateMerger :', -> @projectLocator.findElementByPath = (_, __, cb)->cb(null, {_id:file_id}) @FileTypeManager.isBinary.callsArgWith(2, null, true) @FileTypeManager.shouldIgnore.callsArgWith(1, null, false) - @updateMerger.p.processFile = sinon.stub().callsArgWith(4) + @updateMerger.p.processFile = sinon.stub().callsArgWith(5) filePath = "/folder/file1.png" - @updateMerger.mergeUpdate @project_id, filePath, @update, "", => - @updateMerger.p.processFile.calledWith(@project_id, file_id, @fsPath).should.equal true + @updateMerger.mergeUpdate @project_id, filePath, @update, @source, => + @updateMerger.p.processFile.calledWith(@project_id, file_id, @fsPath, filePath, @source).should.equal true @FileTypeManager.isBinary.calledWith(filePath, @fsPath).should.equal true done() @@ -90,12 +91,12 @@ describe 'UpdateMerger :', -> it 'should set the doc text in the editor controller', (done)-> @editorController.setDoc = -> - mock = sinon.mock(@editorController).expects("setDoc").withArgs(@project_id, @doc_id, @splitDocLines).callsArg(4) + mock = sinon.mock(@editorController).expects("setDoc").withArgs(@project_id, @doc_id, @splitDocLines, @source).callsArg(4) @update.write(@docLines) @update.end() - @updateMerger.p.processDoc @project_id, @doc_id, @update, "path", "", -> + @updateMerger.p.processDoc @project_id, @doc_id, @update, "path", @source, -> mock.verify() done() @@ -105,12 +106,12 @@ describe 'UpdateMerger :', -> path = "folder1/folder2/#{docName}" @editorController.mkdirp = sinon.stub().withArgs(@project_id).callsArgWith(2, null, [folder], folder) @editorController.addDoc = -> - mock = sinon.mock(@editorController).expects("addDoc").withArgs(@project_id, folder._id, docName, @splitDocLines).callsArg(5) + mock = sinon.mock(@editorController).expects("addDoc").withArgs(@project_id, folder._id, docName, @splitDocLines, @source).callsArg(5) @update.write(@docLines) @update.end() - @updateMerger.p.processDoc @project_id, undefined, @update, path, "", -> + @updateMerger.p.processDoc @project_id, undefined, @update, path, @source, -> mock.verify() done() @@ -122,22 +123,22 @@ describe 'UpdateMerger :', -> @folder = _id: @folder_id @fileName = "file.png" @fsPath = "fs/path.tex" - @editorController.addFile = sinon.stub().callsArg(4) - @editorController.replaceFile = sinon.stub().callsArg(3) + @editorController.addFile = sinon.stub().callsArg(5) + @editorController.replaceFile = sinon.stub().callsArg(4) @editorController.deleteEntity = sinon.stub() @editorController.mkdirp = sinon.stub().withArgs(@project_id).callsArgWith(2, null, [@folder], @folder) @updateMerger.p.writeStreamToDisk = sinon.stub().withArgs(@project_id, @file_id, @update).callsArgWith(3, null, @fsPath) it 'should replace file if the file already exists', (done)-> - @updateMerger.p.processFile @project_id, @file_id, @fsPath, @path, => + @updateMerger.p.processFile @project_id, @file_id, @fsPath, @path, @source, => @editorController.addFile.called.should.equal false - @editorController.replaceFile.calledWith(@project_id, @file_id, @fsPath).should.equal true + @editorController.replaceFile.calledWith(@project_id, @file_id, @fsPath, @source).should.equal true done() it 'should call add file if the file does not exist', (done)-> - @updateMerger.p.processFile @project_id, undefined, @fsPath, @path, => + @updateMerger.p.processFile @project_id, undefined, @fsPath, @path, @source, => @editorController.mkdirp.calledWith(@project_id, "folder/").should.equal true - @editorController.addFile.calledWith(@project_id, @folder_id, @fileName, @fsPath).should.equal true + @editorController.addFile.calledWith(@project_id, @folder_id, @fileName, @fsPath, @source).should.equal true @editorController.replaceFile.called.should.equal false done() @@ -152,26 +153,26 @@ describe 'UpdateMerger :', -> it 'should get the element id', -> @projectLocator.findElementByPath = sinon.spy() - @updateMerger.deleteUpdate @project_id, @path, "", -> + @updateMerger.deleteUpdate @project_id, @path, @source, -> @projectLocator.findElementByPath.calledWith(@project_id, @path).should.equal true it 'should delete the entity in the editor controller with type doc when entity has docLines array', (done)-> @entity.lines = [] - mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "doc").callsArg(4) - @updateMerger.deleteUpdate @project_id, @path, "", -> + mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "doc", @source).callsArg(4) + @updateMerger.deleteUpdate @project_id, @path, @source, -> mock.verify() done() it 'should delete the entity in the editor controller with type folder when entity has folders array', (done)-> @entity.folders = [] - mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "folder").callsArg(4) - @updateMerger.deleteUpdate @project_id, @path, "", -> + mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "folder", @source).callsArg(4) + @updateMerger.deleteUpdate @project_id, @path, @source, -> mock.verify() done() it 'should delete the entity in the editor controller with type file when entity has no interesting properties', (done)-> - mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "file").callsArg(4) - @updateMerger.deleteUpdate @project_id, @path, "", -> + mock = sinon.mock(@editorController).expects("deleteEntity").withArgs(@project_id, @entity_id, "file", @source).callsArg(4) + @updateMerger.deleteUpdate @project_id, @path, @source, -> mock.verify() done() diff --git a/services/web/test/UnitTests/coffee/Uploads/FileSystemImportManagerTests.coffee b/services/web/test/UnitTests/coffee/Uploads/FileSystemImportManagerTests.coffee index e0a38c974b..6771151f59 100644 --- a/services/web/test/UnitTests/coffee/Uploads/FileSystemImportManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Uploads/FileSystemImportManagerTests.coffee @@ -22,14 +22,14 @@ describe "FileSystemImportManager", -> @docContent = "one\ntwo\nthree" @docLines = @docContent.split("\n") @fs.readFile = sinon.stub().callsArgWith(2, null, @docContent) - @EditorController.addDoc = sinon.stub().callsArg(4) + @EditorController.addDoc = sinon.stub().callsArg(5) @FileSystemImportManager.addDoc @project_id, @folder_id, @name, @path_on_disk, false, @callback it "should read the file from disk", -> @fs.readFile.calledWith(@path_on_disk, "utf8").should.equal true it "should insert the doc", -> - @EditorController.addDoc.calledWith(@project_id, @folder_id, @name, @docLines) + @EditorController.addDoc.calledWith(@project_id, @folder_id, @name, @docLines, "upload") .should.equal true describe "addDoc with windows line ending", -> @@ -37,20 +37,20 @@ describe "FileSystemImportManager", -> @docContent = "one\r\ntwo\r\nthree" @docLines = ["one", "two", "three"] @fs.readFile = sinon.stub().callsArgWith(2, null, @docContent) - @EditorController.addDoc = sinon.stub().callsArg(4) + @EditorController.addDoc = sinon.stub().callsArg(5) @FileSystemImportManager.addDoc @project_id, @folder_id, @name, @path_on_disk, false, @callback it "should strip the \\r characters before adding", -> - @EditorController.addDoc.calledWith(@project_id, @folder_id, @name, @docLines) + @EditorController.addDoc.calledWith(@project_id, @folder_id, @name, @docLines, "upload") .should.equal true describe "addFile with replace set to false", -> beforeEach -> - @EditorController.addFile = sinon.stub().callsArg(4) + @EditorController.addFile = sinon.stub().callsArg(5) @FileSystemImportManager.addFile @project_id, @folder_id, @name, @path_on_disk, false, @callback it "should add the file", -> - @EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk) + @EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload") .should.equal true describe "addFile with replace set to true", -> @@ -63,7 +63,7 @@ describe "FileSystemImportManager", -> }] } @ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder) - @EditorController.addFile = sinon.stub().callsArg(4) + @EditorController.addFile = sinon.stub().callsArg(5) @FileSystemImportManager.addFile @project_id, @folder_id, @name, @path_on_disk, true, @callback it "should look up the folder", -> @@ -72,7 +72,7 @@ describe "FileSystemImportManager", -> .should.equal true it "should add the file", -> - @EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk) + @EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload") .should.equal true describe "when the file does exist", -> @@ -87,7 +87,7 @@ describe "FileSystemImportManager", -> }] } @ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder) - @EditorController.replaceFile = sinon.stub().callsArg(3) + @EditorController.replaceFile = sinon.stub().callsArg(4) @FileSystemImportManager.addFile @project_id, @folder_id, @name, @path_on_disk, true, @callback it "should look up the folder", -> @@ -96,18 +96,18 @@ describe "FileSystemImportManager", -> .should.equal true it "should replace the file", -> - @EditorController.replaceFile.calledWith(@project_id, @file_id, @path_on_disk) + @EditorController.replaceFile.calledWith(@project_id, @file_id, @path_on_disk, "upload") .should.equal true describe "addFolder", -> beforeEach -> @new_folder_id = "new-folder-id" - @EditorController.addFolder = sinon.stub().callsArgWith(3, null, _id: @new_folder_id) + @EditorController.addFolder = sinon.stub().callsArgWith(4, null, _id: @new_folder_id) @FileSystemImportManager.addFolderContents = sinon.stub().callsArg(4) @FileSystemImportManager.addFolder @project_id, @folder_id, @name, @path_on_disk, @replace, @callback it "should add a folder to the project", -> - @EditorController.addFolder.calledWith(@project_id, @folder_id, @name) + @EditorController.addFolder.calledWith(@project_id, @folder_id, @name, "upload") .should.equal true it "should add the folders contents", ->