Get doc lines from docstore when performing batch operations

services/web/app/coffee/Features/Docstore/DocstoreManager.coffee

@@ -4,7 +4,7 @@ settings = require "settings-sharelatex"
 
 module.exports = DocstoreManager =
 	deleteDoc: (project_id, doc_id, callback = (error) ->) ->
-		logger.log project_id: project_id, "deleting doc in docstore api"
+		logger.log project_id: project_id, doc_id: doc_id, "deleting doc in docstore api"
 		url = "#{settings.apis.docstore.url}/project/#{project_id}/doc/#{doc_id}"
 		request.del url, (error, res, body) ->
 			return callback(error) if error?
@@ -13,4 +13,19 @@ module.exports = DocstoreManager =
 			else
 				error = new Error("docstore api responded with non-success code: #{res.statusCode}")
 				logger.error err: error, project_id: project_id, doc_id: doc_id, "error deleting doc in docstore"
+				callback(error)
+
+	getAllDocs: (project_id, callback = (error) ->) ->
+		logger.log project_id: project_id, "getting all docs for project in docstore api"
+		url = "#{settings.apis.docstore.url}/project/#{project_id}/doc"
+		request.get {
+			url: url
+			json: true
+		}, (error, res, docs) ->
+			return callback(error) if error?
+			if 200 <= res.statusCode < 300
+				callback(null, docs)
+			else
+				error = new Error("docstore api responded with non-success code: #{res.statusCode}")
+				logger.error err: error, project_id: project_id, "error getting all docs in docstore"
 				callback(error)

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -33,14 +33,31 @@ module.exports = ProjectEntityHandler =
 	getAllDocs: (project_id, sl_req_id, callback) ->
 		{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
 		logger.log project_id:project_id, "getting all docs for project"
-		@getAllFolders project_id, sl_req_id, (err, folders) ->
+
-			return callback(err) if err?
+		# We get the path and name info from the project, and the lines and
-			docs = {}
+		# version info from the doc store.
-			for folderPath, folder of folders
+		DocstoreManager.getAllDocs project_id, (error, docContentsArray) ->
-				for doc in folder.docs
+			return callback(error) if error?
-					docs[path.join(folderPath, doc.name)] = doc
+
-			logger.log count:_.keys(docs).length, project_id:project_id, "returning docs for project"
+			# Turn array from docstore into a dictionary based on doc id
-			callback null, docs
+			docContents = {}
+			for docContent in docContentsArray
+				docContents[docContent._id] = docContent
+
+			ProjectEntityHandler.getAllFolders project_id, sl_req_id, (error, folders) ->
+				return callback(error) if error?
+				docs = {}
+				for folderPath, folder of folders
+					for doc in folder.docs
+						content = docContents[doc._id.toString()]
+						docs[path.join(folderPath, doc.name)] = {
+							_id:   doc._id
+							name:  doc.name
+							lines: content.lines
+							rev:   content.rev
+						}
+				logger.log count:_.keys(docs).length, project_id:project_id, "returning docs for project"
+				callback null, docs
 
 	getAllFiles: (project_id, sl_req_id, callback) ->
 		{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
@@ -59,17 +76,20 @@ module.exports = ProjectEntityHandler =
 		logger.log sl_req_id: sl_req_id, project_id:project_id, "flushing project to tpds"
 		documentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
 		documentUpdaterHandler.flushProjectToMongo project_id, undefined, (error) ->
-			Project.findById project_id, (err, project) ->
+			return callback(error) if error?
+			Project.findById project_id, (error, project) ->
 				return callback(error) if error?
 				requests = []
-				self.getAllDocs project_id, (err, docs) ->
+				self.getAllDocs project_id, (error, docs) ->
+					return callback(error) if error?
 					for docPath, doc of docs
 						do (docPath, doc) ->
 							requests.push (callback) ->
 								tpdsUpdateSender.addDoc {project_id:project_id, docLines:doc.lines, path:docPath, project_name:project.name, rev:doc.rev||0},
 									sl_req_id,
 									callback
-					self.getAllFiles project_id, (err, files) ->
+					self.getAllFiles project_id, (error, files) ->
+						return callback(error) if error?
 						for filePath, file of files
 							do (filePath, file) ->
 								requests.push (callback) ->

services/web/config/settings.defaults.coffee

@@ -72,6 +72,8 @@ module.exports =
 			url :"http://localhost:3012"
 		spelling:
 			url : "http://localhost:3005"
+		trackchanges:
+			url : "http://localhost:3015"
 		docstore:
 			url : "http://localhost:3016"
 		versioning:

services/web/test/UnitTests/coffee/Docstore/DocstoreManagerTests.coffee

@@ -52,3 +52,36 @@ describe "DocstoreManager", ->
 					}, "error deleting doc in docstore")
 					.should.equal true
 
+
+	describe "getAllDocs", ->
+		describe "with a successful response code", ->
+			beforeEach ->
+				@request.get = sinon.stub().callsArgWith(1, null, statusCode: 204, @docs = [{ _id: "mock-doc-id" }])
+				@DocstoreManager.getAllDocs @project_id, @callback
+
+			it "should get all the project docs in the docstore api", ->
+				@request.get
+					.calledWith({
+						url: "#{@settings.apis.docstore.url}/project/#{@project_id}/doc"
+						json: true
+					})
+					.should.equal true
+
+			it "should call the callback with the docs", ->
+				@callback.calledWith(null, @docs).should.equal true
+
+		describe "with a failed response code", ->
+			beforeEach ->
+				@request.get = sinon.stub().callsArgWith(1, null, statusCode: 500, "")
+				@DocstoreManager.getAllDocs @project_id, @callback
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error("docstore api responded with non-success code: 500")).should.equal true
+
+			it "should log the error", ->
+				@logger.error
+					.calledWith({
+						err: new Error("docstore api responded with a non-success code: 500")
+						project_id: @project_id
+					}, "error getting all docs in docstore")
+					.should.equal true

services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee

@@ -7,7 +7,7 @@ SandboxedModule = require('sandboxed-module')
 ObjectId = require("mongoose").Types.ObjectId
 tk = require 'timekeeper'
 
-describe 'project entity handler', ->
+describe 'ProjectEntityHandler', ->
 	project_id = '4eecb1c1bffa66588e0000a1'
 	folder_id = "4eecaffcbffa66588e000008"
 	rootFolderId = "4eecaffcbffa66588e000007"
@@ -481,137 +481,162 @@ describe 'project entity handler', ->
 				@projectUpdater.markAsUpdated.calledWith(project_id).should.equal true
 			@ProjectEntityHandler.updateDocLines project_id, docId, docLines, done
 
-	describe "flushing folders, docs and files", ->
+	describe "getting folders, docs and files", ->
 		beforeEach ->
 			@project.rootFolder = [
-				folders: [{
+				docs: [@doc1 = {
-					name    : "folder1"
+					name  : "doc1"
-					folders : [{
+					_id   : "doc1_id"
-						name    : "folder2"
-						folders : [{
-							name:"folder4",
-							docs:[{name:"doc3", rev:3,lines:["doc3"]}],
-							fileRefs:[{_id:"file3_id", rev:3,name:"file3"}],
-							folders:[]
-							}]
-						docs    : [{
-							rev:2
-							name: "doc2"
-							lines: ["doc2", "lines"]
-						}]
-						fileRefs   : []
-					}]
-					docs    : []
-					fileRefs   : [{
-						rev:2
-						name : "file2"
-						_id  : "file2_id"
-					}]
-				}, {
-					name    : "folder3"
-					folders : []
-					docs    : []
-					fileRefs   : []
 				}]
-				docs: [{
+				fileRefs: [@file1 = {
-					rev:1
+					rev  : 1
-					name: "doc1"
-					lines: ["doc1", "lines"]
-				}]
-				fileRefs: [{
-					rev:1
 					_id  : "file1_id"
 					name : "file1"
 				}]
+				folders: [@folder1 = {
+					name    : "folder1"
+					docs    : [@doc2 = {
+						name  : "doc2"
+						_id   : "doc2_id"
+					}]
+					fileRefs   : [@file2 = {
+						rev  : 2
+						name : "file2"
+						_id  : "file2_id"
+					}]
+					folders : []
+				}]
 			]
 
-		it "should work for a very small project", (done)->
+		describe "getAllFolders", ->
-			@project.rootFolder[0].folders = []
+			beforeEach ->
-			@ProjectEntityHandler.getAllDocs project_id, (err, docs) =>
+				@callback = sinon.stub()
-				docs["/doc1"].name.should.equal "doc1"
+				@ProjectEntityHandler.getAllFolders project_id, @callback
-				@ProjectEntityHandler.getAllFiles project_id, (err, files) =>
-					files["/file1"].name.should.equal "file1"
-					done()
 
-		it "should be able to get all folders", (done) ->
+			it "should call the callback with the folders", ->
-			@ProjectEntityHandler.getAllFolders project_id, (err, folders) ->
+				@callback
-				should.exist folders["/"]
+					.calledWith(null, {
-				should.exist folders["/folder1"]
+						"/": @project.rootFolder[0]
-				should.exist folders["/folder1/folder2"]
+						"/folder1": @folder1
-				should.exist folders["/folder1/folder2/folder4"]
+					})
-				folders["/folder1/folder2/folder4"].name.should.equal "folder4"
+					.should.equal true
-				should.exist folders["/folder3"]
-				done()
 
-		it "should be able to get all docs", (done) ->
+		describe "getAllFiles", ->
-			@ProjectEntityHandler.getAllDocs project_id, (err, docs) ->
+			beforeEach ->
-				docs["/doc1"].name.should.equal "doc1"
+				@callback = sinon.stub()
-				docs["/folder1/folder2/doc2"].name.should.equal "doc2"
+				@ProjectEntityHandler.getAllFiles project_id, @callback
-				docs["/folder1/folder2/folder4/doc3"].lines.should.deep.equal ["doc3"]
-				done()
 
-		it "should be able to get all files", (done) ->
+			it "should call the callback with the files", ->
-			@ProjectEntityHandler.getAllFiles project_id, (err, files) ->
+				@callback
-				files["/file1"].name.should.equal "file1"
+					.calledWith(null, {
-				files["/folder1/file2"].name.should.equal "file2"
+						"/file1": @file1
-				files["/folder1/folder2/folder4/file3"].name.should.equal "file3"
+						"/folder1/file2": @file2
-				done()
+					})
+					.should.equal true
 
-		describe "flushProjectToThirdPartyDataStore", ->
+		describe "getAllDocs", ->
-			beforeEach (done) ->
+			beforeEach ->
-				@addedDocs = {}
+				@docs = [{
-				@addedFiles = {}
+					_id:   @doc1._id
-				@tpdsUpdateSender.addDoc = (options, _, callback) =>
+					lines: @lines1 = ["one"]
-					callback()
+					rev:   @rev1 = 1
-				sinon.spy @tpdsUpdateSender, "addDoc"
+				}, {
-				@tpdsUpdateSender.addFile = (options, _, callback) =>
+					_id:   @doc2._id
-					callback()
+					lines: @lines2 = ["two"]
-				sinon.spy @tpdsUpdateSender, "addFile"
+					rev:   @rev2 = 2
-				@documentUpdaterHandler.flushProjectToMongo = (project_id, sl_req_id, callback) ->
+				}]
-					callback()
+				@DocstoreManager.getAllDocs = sinon.stub().callsArgWith(1, null, @docs)
-				sinon.spy @documentUpdaterHandler, "flushProjectToMongo"
+				@ProjectEntityHandler.getAllDocs project_id, @callback
 
-				@ProjectEntityHandler.flushProjectToThirdPartyDataStore project_id, (err) -> done()
+			it "should get the doc lines and rev from the docstore", ->
+				@DocstoreManager.getAllDocs
+					.calledWith(project_id)
+					.should.equal true
 
-			it "should flush the documents from the document updater", ->
+			it "should call the callback with the docs with the lines and rev included", ->
-				@documentUpdaterHandler.flushProjectToMongo
+				@callback
-					.calledWith(@project._id).should.equal true
+					.calledWith(null, {
-				@documentUpdaterHandler.flushProjectToMongo
+						"/doc1": {
-					.calledBefore(@tpdsUpdateSender.addDoc).should.equal true
+							_id:   @doc1._id
-				@documentUpdaterHandler.flushProjectToMongo
+							lines: @lines1
-					.calledBefore(@tpdsUpdateSender.addFile).should.equal true
+							name:  @doc1.name
+							rev:   @rev1
+						}
+						"/folder1/doc2":  {
+							_id:   @doc2._id
+							lines: @lines2
+							name:  @doc2.name
+							rev:   @rev2
+						}
+					})
+					.should.equal true
 
-			it "should call addDoc for each doc", ->
+	describe "flushProjectToThirdPartyDataStore", ->
-				@tpdsUpdateSender.addDoc.calledWith(
+		beforeEach (done) ->
-					project_id : @project._id
+			@project = {
-					path : "/doc1"
+				_id: project_id
-					docLines: ["doc1", "lines"]
+				name: "Mock project name"
-					project_name: @project.name
+			}
-					rev:1
+			@ProjectModel.findById = sinon.stub().callsArgWith(1, null, @project)
-				).should.equal true
+			@documentUpdaterHandler.flushProjectToMongo = sinon.stub().callsArg(2)
-				@tpdsUpdateSender.addDoc.calledWith(
+			@tpdsUpdateSender.addDoc = sinon.stub().callsArg(2)
-					project_id : @project._id
+			@tpdsUpdateSender.addFile = sinon.stub().callsArg(2)
-					path : "/folder1/folder2/doc2"
+			@docs = {
-					docLines: ["doc2", "lines"]
+				"/doc/one": @doc1 = { _id: "mock-doc-1", lines: ["one"], rev: 5 }
-					project_name: @project.name
+				"/doc/two": @doc2 = { _id: "mock-doc-2", lines: ["two"], rev: 6 }
-					rev:2
+			}
-				).should.equal true
+			@files = {
+				"/file/one": @file1 = { _id: "mock-file-1", rev: 7 }
+				"/file/two": @file2 = { _id: "mock-file-2", rev: 8 }
+			}
+			@ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs)
+			@ProjectEntityHandler.getAllFiles = sinon.stub().callsArgWith(1, null, @files)
 
-			it "should call addFile for each file", ->
+			@ProjectEntityHandler.flushProjectToThirdPartyDataStore project_id, () -> done()
-				@tpdsUpdateSender.addFile.calledWith(
+
-					project_id : @project._id
+		it "should flush the project from the doc updater", ->
-					file_id    : "file1_id"
+			@documentUpdaterHandler.flushProjectToMongo
-					path       : "/file1"
+				.calledWith(project_id)
-					project_name: @project.name
+				.should.equal true
-					rev:1
+
-				).should.equal true
+		it "should look up the project in mongo", ->
-				@tpdsUpdateSender.addFile.calledWith(
+			@ProjectModel.findById
-					project_id : @project._id
+				.calledWith(project_id)
-					file_id    : "file2_id"
+				.should.equal true
-					path       : "/folder1/file2"
+
-					project_name: @project.name
+		it "should get all the docs in the project", ->
-					rev:2
+			@ProjectEntityHandler.getAllDocs
-				).should.equal true
+				.calledWith(project_id)
+				.should.equal true
+
+		it "should get all the files in the project", ->
+			@ProjectEntityHandler.getAllFiles
+				.calledWith(project_id)
+				.should.equal true
+
+		it "should flush each doc to the TPDS", ->
+			for path, doc of @docs
+				@tpdsUpdateSender.addDoc
+					.calledWith({
+						project_id: project_id,
+						docLines: doc.lines
+						project_name: @project.name
+						rev: doc.rev
+						path: path
+					})
+					.should.equal true
+
+		it "should flush each file to the TPDS", ->
+			for path, file of @files
+				@tpdsUpdateSender.addFile
+					.calledWith({
+						project_id: project_id,
+						file_id: file._id
+						project_name: @project.name
+						rev: file.rev
+						path: path
+					})
+					.should.equal true
 	
 	describe "setRootDoc", ->
 		it "should call Project.update", ->