change send doc lines using tpds to work with stream and doc store

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -86,7 +86,7 @@ module.exports = ProjectEntityHandler =
 					for docPath, doc of docs
 						do (docPath, doc) ->
 							requests.push (callback) ->
-								tpdsUpdateSender.addDoc {project_id:project_id, docLines:doc.lines, path:docPath, project_name:project.name, rev:doc.rev||0},
+								tpdsUpdateSender.addDoc {project_id:project_id, doc_id:doc._id, path:docPath, project_name:project.name, rev:doc.rev||0},
 									sl_req_id,
 									callback
 					self.getAllFiles project_id, (error, files) ->
@@ -127,7 +127,7 @@ module.exports = ProjectEntityHandler =
 						return callback(err) if err? 
 						tpdsUpdateSender.addDoc {
 							project_id:   project._id,
-							docLines:     docLines,
+							doc_id:		  doc._id	
 							path:         result.path.fileSystem,
 							project_name: project.name,
 							rev:          0
@@ -256,7 +256,7 @@ module.exports = ProjectEntityHandler =
 					if modified
 						# Don't need to block for marking as updated
 						projectUpdateHandler.markAsUpdated project_id
-						tpdsUpdateSender.addDoc {project_id:project_id, path:path.fileSystem, docLines:lines, project_name:project.name, rev:rev}, callback
+						tpdsUpdateSender.addDoc {project_id:project_id, path:path.fileSystem, doc_id:doc_id, project_name:project.name, rev:rev}, callback
 					else
 						callback()
 

services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee

@@ -15,10 +15,8 @@ buildPath = (user_id, project_name, filePath)->
 queue = require('fairy').connect(settings.redis.fairy).queue(keys.queue.web_to_tpds_http_requests)
 
 module.exports =
- 
-	addFile : (options, sl_req_id, callback = (err)->)->
-		metrics.inc("tpds.add-file")
-		{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
+
+	_addEntity: (options, sl_req_id, callback = (err)->)->
 		getProjectsUsersIds options.project_id, (err, user_id, allUserIds)->
 			logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store"
 			postOptions =
@@ -29,29 +27,23 @@ module.exports =
 					sl_project_id:options.project_id
 					sl_all_user_ids:JSON.stringify(allUserIds)
 				uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}" 
-				title:"addFile"
-				streamOrigin : settings.apis.filestore.url + path.join("/project/#{options.project_id}/file/","#{options.file_id}")
+				title: "addFile"
+				streamOrigin : options.streamOrigin
 			queue.enqueue options.project_id, "pipeStreamFrom", postOptions, ->
 				logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store queued up for processing"
 				callback()
+	
+	addFile : (options, sl_req_id, callback = (err)->)->
+		metrics.inc("tpds.add-file")
+		{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
+		options.streamOrigin = settings.apis.filestore.url + path.join("/project/#{options.project_id}/file/","#{options.file_id}")
+		@_addEntity(options, sl_req_id, callback)
 
 	addDoc : (options, sl_req_id, callback = (err)->)->
 		metrics.inc("tpds.add-doc")
 		{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
-		getProjectsUsersIds options.project_id, (err, user_id, allUserIds)->
-			return callback(err) if err?
-			logger.log project_id: options.project_id, user_id:user_id, path: options.path, rev:options.rev, uri:options.uri, project_name:options.project_name, docLines:options.docLines, sl_req_id:sl_req_id, "sending doc to third party data store"
-			postOptions =
-				method : "post"
-				headers: 
-					"sl_req_id":sl_req_id, 
-					sl_entity_rev:options.rev, 
-					sl_project_id:options.project_id
-					sl_all_user_ids:JSON.stringify(allUserIds)
-				uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}"
-				title: "addDoc"
-				docLines: options.docLines
-			queue.enqueue options.project_id, "sendDoc", postOptions, callback
+		options.streamOrigin = settings.apis.docstore.url + path.join("/project/#{options.project_id}/doc/","#{options.doc_id}")
+		@_addEntity(options, sl_req_id, callback)
   
 
 	moveEntity : (options, sl_req_id, callback = (err)->)->

services/web/app/coffee/Features/User/UserController.coffee

@@ -47,7 +47,7 @@ module.exports =
 					return res.send(400)
 				else if newEmail == user.email
 					return res.send 200
-				else 
+				else
 					UserUpdater.changeEmailAddress user_id, newEmail, (err)->
 						if err?
 							logger.err err:err, user_id:user_id, newEmail:newEmail, "problem updaing users email address"

services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee

@@ -35,10 +35,11 @@ describe 'ProjectEntityHandler', ->
 				@rev = 0
 			save:(callback)->callback()
 			rootFolder:[@rootFolder]
+
 		@DocModel = class Doc
 			constructor:(options)->
 				{@name, @lines} = options
-				@_id = "mock-id"	
+				@_id = doc_id
 				@rev = 0
 		@FileModel =  class File
 			constructor:(options)->
@@ -326,7 +327,7 @@ describe 'ProjectEntityHandler', ->
 			@tpdsUpdateSender.addDoc
 				.calledWith({
 					project_id: project_id
-					docLines: @lines
+					doc_id: doc_id
 					path: @path
 					project_name: @project.name
 					rev: 0
@@ -504,7 +505,7 @@ describe 'ProjectEntityHandler', ->
 					.calledWith({
 						project_id: project_id
 						project_name: @project.name
-						docLines: @lines
+						doc_id: doc_id
 						rev: @rev
 						path: @path
 					})
@@ -699,7 +700,7 @@ describe 'ProjectEntityHandler', ->
 				@tpdsUpdateSender.addDoc
 					.calledWith({
 						project_id: project_id,
-						docLines: doc.lines
+						doc_id: doc._id
 						project_name: @project.name
 						rev: doc.rev
 						path: path

services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee

@@ -22,6 +22,7 @@ describe 'TpdsUpdateSender', ->
 		@requestQueuer = regist:(queue, meth, opts, callback)->
 		project = {owner_ref:user_id,readOnly_refs:[read_only_ref_1], collaberator_refs:[collaberator_ref_1]}
 		@Project = findById:sinon.stub().callsArgWith(2, null, project)
+		@docstoreUrl = "docstore.sharelatex.env"
 		@updateSender = SandboxedModule.require modulePath, requires:
 			'fairy':{connect:=>{queue:=>@requestQueuer}}
 			"settings-sharelatex": 
@@ -31,6 +32,8 @@ describe 'TpdsUpdateSender', ->
 					thirdPartyDataStore: {url: thirdPartyDataStoreApiUrl}
 					filestore:
 						url: filestoreUrl
+					docstore:
+						url: @docstoreUrl
 				redis:fairy:{}
 			"logger-sharelatex":{log:->}
 			'../../models/Project': Project:@Project
@@ -56,12 +59,12 @@ describe 'TpdsUpdateSender', ->
 			path = "/some/path/here.tex"
 			lines = ["line1", "line2", "line3"]
 
-			@requestQueuer.enqueue = (uid, method, job, callback)->
+			@requestQueuer.enqueue = (uid, method, job, callback)=>
 				uid.should.equal project_id
 				job.method.should.equal "post"
 				expectedUrl = "#{thirdPartyDataStoreApiUrl}/user/#{user_id}/entity/#{encodeURIComponent(project_name)}#{encodeURIComponent(path)}"
 				job.uri.should.equal expectedUrl
-				assert.deepEqual job.docLines, lines
+				job.streamOrigin.should.equal "#{@docstoreUrl}/project/#{project_id}/doc/#{doc_id}"
 				job.headers.sl_all_user_ids.should.eql(JSON.stringify([collaberator_ref_1, read_only_ref_1, user_id]))
 				done()
 			@updateSender.addDoc {project_id:project_id, doc_id:doc_id, path:path, docLines:lines,project_name:project_name}