persist cookie in redis for compiles.

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -6,8 +6,19 @@ Project = require("../../models/Project").Project
 ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 logger = require "logger-sharelatex"
 url = require("url")
+ClsiRequestManager = require("./ClsiRequestManager")
+
 
 module.exports = ClsiManager =
+
+	_makeRequest: (project_id, opts, callback)->
+		ClsiRequestManager.getCookieJar project_id, (err, jar)->
+			if err?
+				logger.err err:err, "error getting cookie jar for clsi request"
+				return callback(err)
+			opts.jar = jar
+			request opts, callback
+
 	sendRequest: (project_id, options = {}, callback = (error, success) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
 			return callback(error) if error?
@@ -23,7 +34,10 @@ module.exports = ClsiManager =
 
 	deleteAuxFiles: (project_id, options, callback = (error) ->) ->
 		compilerUrl = @_getCompilerUrl(options?.compileGroup)
-		request.del "#{compilerUrl}/project/#{project_id}", callback
+		opts =
+			url:"#{compilerUrl}/project/#{project_id}"
+			method:"DELETE"
+		ClsiManager._makeRequest project_id, opts, callback
 
 	_getCompilerUrl: (compileGroup) ->
 		if compileGroup == "priority"
@@ -33,13 +47,11 @@ module.exports = ClsiManager =
 
 	_postToClsi: (project_id, req, compileGroup, callback = (error, response) ->) ->
 		compilerUrl = @_getCompilerUrl(compileGroup)
-		request.post {
+		opts = 
 			url:  "#{compilerUrl}/project/#{project_id}/compile"
 			json: req
-			jar:  false
-			query:
-				project_id:project_id
-		}, (error, response, body) ->
+			method: "POST"
+		ClsiManager._makeRequest project_id, opts, (error, response, body) ->
 			return callback(error) if error?
 			if 200 <= response.statusCode < 300
 				callback null, body
@@ -117,9 +129,10 @@ module.exports = ClsiManager =
 			wordcount_url = "#{compilerUrl}/project/#{project_id}/wordcount?file=#{encodeURIComponent(filename)}"
 			if req.compile.options.imageName?
 				wordcount_url += "&image=#{encodeURIComponent(req.compile.options.imageName)}"
-			request.get {
+			opts =
 				url: wordcount_url
-			}, (error, response, body) ->
+				method: "GET"
+			ClsiManager._makeRequest project_id, opts, (error, response, body) ->
 				return callback(error) if error?
 				if 200 <= response.statusCode < 300
 					callback null, body

services/web/app/coffee/Features/Compile/ClsiRequestManager.coffee

@@ -0,0 +1,60 @@
+Settings = require "settings-sharelatex"
+request = require('request')
+redis = require("redis-sharelatex")
+rclient = redis.createClient(Settings.redis.web)
+cookie = require('cookie')
+
+buildKey = (project_id)->
+	return "clsiserver:#{project_id}"
+
+
+ONE_WEEK_IN_SECONDS = 60 * 60 * 24 * 7
+
+module.exports = ClsiRequestManager =
+
+	_getServerId : (project_id, callback = (err, serverId)->)->
+		multi = rclient.multi()
+		multi.get buildKey(project_id)
+		multi.expire buildKey(project_id), ONE_WEEK_IN_SECONDS
+		multi.exec (err, results)->
+			if err?
+				return callback(err)
+			serverId = results[0]
+			if serverId?
+				return callback(null, serverId)
+			else
+				return ClsiRequestManager._getServerIdViaRequest project_id, callback
+
+
+	_getServerIdViaRequest :(project_id, callback = (err, serverId)->)->
+		url = "#{Settings.apis.clsi.url}/project/#{project_id}/status"
+		request.get url, (err, res, body)->
+			if err?
+				logger.err err:err, project_id:project_id, "error getting initial server id for project"
+				return callback(err)
+			ClsiRequestManager.setServerId project_id, res, callback
+
+	_parseServerIdFromResponse : (response)->
+		console.log response.headers
+		cookies = cookie.parse(response.headers["set-cookie"]?[0] or "")
+		return cookies?.clsiserver
+
+	setServerId: (project_id, response, callback = ->)->
+		serverId = ClsiRequestManager._parseServerIdFromResponse(response)
+		multi = rclient.multi()
+		multi.set buildKey(project_id), serverId
+		multi.expire buildKey(project_id), ONE_WEEK_IN_SECONDS
+		multi.exec callback
+
+
+	getCookieJar: (project_id, opts, callback = (err, jar)->)->
+		ClsiRequestManager._getServerId project_id, (err, serverId)=>
+			if err?
+				logger.err err:err, project_id:project_id, "error getting server id"
+				return callback(err)
+			cookie = request.cookie("clsiserver=#{serverId}")
+			jar = request.jar()
+			jar.setCookie cookie, Settings.apis.clsi.url
+			callback(null, jar)
+
+

services/web/app/coffee/Features/Compile/CompileController.coffee

@@ -8,6 +8,8 @@ Settings = require "settings-sharelatex"
 AuthenticationController = require "../Authentication/AuthenticationController"
 UserGetter = require "../User/UserGetter"
 RateLimiter = require("../../infrastructure/RateLimiter")
+ClsiRequestManager = require("./ClsiRequestManager")
+
 
 module.exports = CompileController =
 	compile: (req, res, next = (error) ->) ->
@@ -107,31 +109,34 @@ module.exports = CompileController =
 				CompileController.proxyToClsiWithLimits(project_id, url, limits, req, res, next)
 
 	proxyToClsiWithLimits: (project_id, url, limits, req, res, next = (error) ->) ->
-		if limits.compileGroup == "priority"
-			compilerUrl = Settings.apis.clsi_priority.url
-		else
-			compilerUrl = Settings.apis.clsi.url
-		url = "#{compilerUrl}#{url}"
-		logger.log url: url, "proxying to CLSI"
-		oneMinute = 60 * 1000
-		# the base request
-		options = { url: url, method: req.method,	timeout: oneMinute }
-		# if we have a build parameter, pass it through to the clsi
-		if req.query?.pdfng && req.query?.build? # only for new pdf viewer
-			options.qs = {}
-			options.qs.build = req.query.build
-		# if we are byte serving pdfs, pass through If-* and Range headers
-		# do not send any others, there's a proxying loop if Host: is passed!
-		if req.query?.pdfng
-			newHeaders = {}
-			for h, v of req.headers
-				newHeaders[h] = req.headers[h] if h.match /^(If-|Range)/i
-			options.headers = newHeaders
-		req.query.project_id = project_id
-		proxy = request(options)
-		proxy.pipe(res)
-		proxy.on "error", (error) ->
-			logger.warn err: error, url: url, "CLSI proxy error"
+		ClsiRequestManager.getCookieJar project_id, (err, jar)->
+			if err?
+				logger.err err:err, "error getting cookie jar for clsi request"
+				return callback(err)
+			if limits.compileGroup == "priority"
+				compilerUrl = Settings.apis.clsi_priority.url
+			else
+				compilerUrl = Settings.apis.clsi.url
+			url = "#{compilerUrl}#{url}"
+			logger.log url: url, "proxying to CLSI"
+			oneMinute = 60 * 1000
+			# the base request
+			options = { url: url, method: req.method, timeout: oneMinute, jar : jar }
+			# if we have a build parameter, pass it through to the clsi
+			if req.query?.pdfng && req.query?.build? # only for new pdf viewer
+				options.qs = {}
+				options.qs.build = req.query.build
+			# if we are byte serving pdfs, pass through If-* and Range headers
+			# do not send any others, there's a proxying loop if Host: is passed!
+			if req.query?.pdfng
+				newHeaders = {}
+				for h, v of req.headers
+					newHeaders[h] = req.headers[h] if h.match /^(If-|Range)/i
+				options.headers = newHeaders
+			proxy = request(options)
+			proxy.pipe(res)
+			proxy.on "error", (error) ->
+				logger.warn err: error, url: url, "CLSI proxy error"
 
 	wordCount: (req, res, next) ->
 		project_id = req.params.Project_id

services/web/app/coffee/infrastructure/Server.coffee

@@ -125,7 +125,7 @@ apiRouter.get "/profile", (req, res) ->
 	, time
 
 app.get "/heapdump", (req, res)->
-	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.clsi.heapsnapshot', (err, filename)->
+	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.web.heapsnapshot', (err, filename)->
 		res.send filename
 
 logger.info ("creating HTTP server").yellow

services/web/package.json

@@ -18,6 +18,7 @@
     "body-parser": "^1.13.1",
     "bufferedstream": "1.6.0",
     "connect-redis": "2.3.0",
+    "cookie": "^0.2.3",
     "cookie-parser": "1.3.5",
     "csurf": "^1.8.3",
     "dateformat": "1.0.4-1.2.3",

services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee

@@ -7,6 +7,9 @@ SandboxedModule = require('sandboxed-module')
 
 describe "ClsiManager", ->
 	beforeEach ->
+		@jar = {cookie:"stuff"}
+		@ClsiRequestManager = 
+			getCookieJar: sinon.stub().callsArgWith(1, null, @jar)
 		@ClsiManager = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": @settings =
 				apis:
@@ -19,8 +22,9 @@ describe "ClsiManager", ->
 						url: "https://clsipremium.example.com"
 			"../../models/Project": Project: @Project = {}
 			"../Project/ProjectEntityHandler": @ProjectEntityHandler = {}
+			"./ClsiRequestManager": @ClsiRequestManager
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
-			"request": @request = {}
+			"request": @request = sinon.stub()
 		@project_id = "project-id"
 		@callback = sinon.stub()
 
@@ -80,15 +84,15 @@ describe "ClsiManager", ->
 
 	describe "deleteAuxFiles", ->
 		beforeEach ->
-			@request.del = sinon.stub().callsArg(1)
+			@ClsiManager._makeRequest = sinon.stub().callsArg(2)
 			
 		describe "with the standard compileGroup", ->
 			beforeEach ->
 				@ClsiManager.deleteAuxFiles @project_id, {compileGroup: "standard"}, @callback
 
 			it "should call the delete method in the standard CLSI", ->
-				@request.del
-					.calledWith("#{@settings.apis.clsi.url}/project/#{@project_id}")
+				@ClsiManager._makeRequest
+					.calledWith(@project_id, { method:"DELETE", url:"#{@settings.apis.clsi.url}/project/#{@project_id}"})
 					.should.equal true
 
 			it "should call the callback", ->
@@ -99,8 +103,8 @@ describe "ClsiManager", ->
 				@ClsiManager.deleteAuxFiles @project_id, {compileGroup: "priority"}, @callback
 
 			it "should call the delete method in the CLSI", ->
-				@request.del
-					.calledWith("#{@settings.apis.clsi_priority.url}/project/#{@project_id}")
+				@ClsiManager._makeRequest
+					.calledWith(@project_id, { method:"DELETE", url:"#{@settings.apis.clsi_priority.url}/project/#{@project_id}"})
 					.should.equal true
 
 	describe "_buildRequest", ->
@@ -235,15 +239,15 @@ describe "ClsiManager", ->
 
 		describe "successfully", ->
 			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, {statusCode: 204}, @body = { mock: "foo" })
+				@ClsiManager._makeRequest = sinon.stub().callsArgWith(2, null, {statusCode: 204}, @body = { mock: "foo" })
 				@ClsiManager._postToClsi @project_id, @req, "standard", @callback
 
 			it 'should send the request to the CLSI', ->
 				url = "#{@settings.apis.clsi.url}/project/#{@project_id}/compile"
-				@request.post.calledWith({
+				@ClsiManager._makeRequest.calledWith(@project_id, {
+					method: "POST",
 					url: url
 					json: @req
-					jar: false
 				}).should.equal true
 
 			it "should call the callback with the body and no error", ->
@@ -251,7 +255,7 @@ describe "ClsiManager", ->
 
 		describe "when the CLSI returns an error", ->
 			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, @body = { mock: "foo" })
+				@ClsiManager._makeRequest = sinon.stub().callsArgWith(2, null, {statusCode: 500}, @body = { mock: "foo" })
 				@ClsiManager._postToClsi @project_id, @req, "standard", @callback
 
 			it "should call the callback with the body and the error", ->
@@ -259,20 +263,20 @@ describe "ClsiManager", ->
 
 		describe "when the compiler is priority", ->
 			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, @body = { mock: "foo" })
+				@ClsiManager._makeRequest = sinon.stub()
 				@ClsiManager._postToClsi @project_id, @req, "priority", @callback
 
 			it "should use the clsi_priority url", ->
 				url = "#{@settings.apis.clsi_priority.url}/project/#{@project_id}/compile"
-				@request.post.calledWith({
+				@ClsiManager._makeRequest.calledWith(@project_id, {
+					method: "POST",
 					url: url
 					json: @req
-					jar: false
 				}).should.equal true
 
 	describe "wordCount", ->
 		beforeEach ->
-			@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body = { mock: "foo" })
+			@ClsiManager._makeRequest = sinon.stub().callsArgWith(2, null, {statusCode: 200}, @body = { mock: "foo" })
 			@ClsiManager._buildRequest = sinon.stub().callsArgWith(2, null, @req = { compile: { rootResourcePath: "rootfile.text", options: {} } })
 			@ClsiManager._getCompilerUrl = sinon.stub().returns "compiler.url"
 
@@ -281,8 +285,8 @@ describe "ClsiManager", ->
 				@ClsiManager.wordCount @project_id, false, {}, @callback
 
 			it "should call wordCount with root file", ->
-				@request.get
-					.calledWith({ url: "compiler.url/project/#{@project_id}/wordcount?file=rootfile.text" })
+				@ClsiManager._makeRequest
+					.calledWith(@project_id, { method: "GET", url: "compiler.url/project/#{@project_id}/wordcount?file=rootfile.text" })
 					.should.equal true
 
 			it "should call the callback", ->
@@ -293,8 +297,8 @@ describe "ClsiManager", ->
 				@ClsiManager.wordCount @project_id, "main.tex", {}, @callback
 
 			it "should call wordCount with param file", ->
-				@request.get
-					.calledWith({ url: "compiler.url/project/#{@project_id}/wordcount?file=main.tex" })
+				@ClsiManager._makeRequest
+					.calledWith(@project_id, { method: "GET", url: "compiler.url/project/#{@project_id}/wordcount?file=main.tex" })
 					.should.equal true
 					
 		describe "with image", ->
@@ -303,6 +307,6 @@ describe "ClsiManager", ->
 				@ClsiManager.wordCount @project_id, "main.tex", {}, @callback
 
 			it "should call wordCount with file and image", ->
-				@request.get
-					.calledWith({ url: "compiler.url/project/#{@project_id}/wordcount?file=main.tex&image=#{encodeURIComponent(@image)}" })
+				@ClsiManager._makeRequest
+					.calledWith(@project_id, { method: "GET", url: "compiler.url/project/#{@project_id}/wordcount?file=main.tex&image=#{encodeURIComponent(@image)}" })
 					.should.equal true

services/web/test/UnitTests/coffee/Compile/ClsiRequestManagerTests.coffee

@@ -0,0 +1,111 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../../app/js/Features/Compile/ClsiRequestManager.js"
+SandboxedModule = require('sandboxed-module')
+realRequst = require("request")
+describe "ClsiRequestManager", ->
+	beforeEach ->
+		@redisMulti =
+			set:sinon.stub()
+			get:sinon.stub()
+			expire:sinon.stub()
+			exec:sinon.stub()
+		self = @
+		@project_id = "123423431321"
+		@request =
+			get: sinon.stub()
+			cookie:realRequst.cookie
+			jar: realRequst.jar
+		@ClsiRequestManager = SandboxedModule.require modulePath, requires:
+			"redis-sharelatex" :
+				createClient: =>
+					auth:->
+					multi: -> return self.redisMulti
+			"settings-sharelatex": @settings =
+				redis:
+					web:"redis.something"
+				apis:
+					clsi:
+						url: "http://clsi.example.com"
+			"request": @request
+
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
+
+
+
+	describe "getServerId", ->
+
+		it "should call get for the key", (done)->
+			@redisMulti.exec.callsArgWith(0, null, ["clsi-7"])
+			@ClsiRequestManager._getServerId @project_id, (err, serverId)=>
+				@redisMulti.get.calledWith("clsiserver:#{@project_id}").should.equal true
+				serverId.should.equal "clsi-7"
+				done()
+
+		it "should expire the key", (done)->
+			@redisMulti.exec.callsArgWith(0, null, ["clsi-7"])
+			@ClsiRequestManager._getServerId @project_id, (err, serverId)=>
+				@redisMulti.expire.calledWith("clsiserver:#{@project_id}", 60 * 60 * 24 * 7).should.equal true
+				done()
+
+		it "should _getServerIdViaRequest if no key is found", (done)->
+			@ClsiRequestManager._getServerIdViaRequest = sinon.stub().callsArgWith(1)
+			@redisMulti.exec.callsArgWith(0, null, [])
+			@ClsiRequestManager._getServerId @project_id, (err, serverId)=>
+				@ClsiRequestManager._getServerIdViaRequest.calledWith(@project_id).should.equal true
+				done()
+
+
+	describe "_getServerIdViaRequest", ->
+
+		it "should make a request to the clsi", (done)->
+			response  = "some data"
+			@request.get.callsArgWith(1, null, response)
+			@ClsiRequestManager.setServerId = sinon.stub().callsArgWith(2)
+			@ClsiRequestManager._getServerIdViaRequest @project_id, (err, serverId)=>
+				args = @ClsiRequestManager.setServerId.args[0]
+				args[0].should.equal @project_id
+				args[1].should.deep.equal response
+				done()
+
+	describe "setServerId", ->
+
+		it "should set the server id with a ttl", (done)->
+			@ClsiRequestManager._parseServerIdFromResponse = sinon.stub().returns("clsi-8")
+			response = "dsadsakj"
+			@redisMulti.exec.callsArgWith(0)
+			@ClsiRequestManager.setServerId @project_id, response, (err)=>
+				@redisMulti.set.calledWith("clsiserver:#{@project_id}", "clsi-8").should.equal true
+				@redisMulti.expire.calledWith("clsiserver:#{@project_id}", 60 * 60 * 24 * 7).should.equal true
+				done()
+
+
+	describe "getCookieJar", ->
+
+		it "should return a jar with the cookie set populated from redis", (done)->
+			@ClsiRequestManager._getServerId = sinon.stub().callsArgWith(1, null, "clsi-11")
+			opts = {}
+			@ClsiRequestManager.getCookieJar @project_id, opts, (err, jar)->
+				jar._jar.store.idx["clsi.example.com"]["/"].clsiserver.key.should.equal "clsiserver"
+				jar._jar.store.idx["clsi.example.com"]["/"].clsiserver.value.should.equal "clsi-11"
+				done()
+
+
+	# describe "_parseServerIdFromResponse", ->
+	# 	it "take the cookie from the response", (done)->
+
+	# 		a.should.equal 
+
+
+
+
+
+
+
+
+
+
+
+

services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee

@@ -22,6 +22,9 @@ describe "CompileController", ->
 					url: "clsi.example.com"
 				clsi_priority:
 					url: "clsi-priority.example.com"
+		@jar = {cookie:"stuff"}
+		@ClsiRequestManager = 
+			getCookieJar:sinon.stub().callsArgWith(1, null, @jar)
 		@CompileController = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": @settings
 			"request": @request = sinon.stub()
@@ -33,6 +36,7 @@ describe "CompileController", ->
 			"./ClsiManager": @ClsiManager
 			"../Authentication/AuthenticationController": @AuthenticationController = {}
 			"../../infrastructure/RateLimiter":@RateLimiter
+			"./ClsiRequestManager":@ClsiRequestManager
 		@project_id = "project-id"
 		@user = 
 			features:
@@ -182,6 +186,7 @@ describe "CompileController", ->
 				it "should open a request to the CLSI", ->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi.url}#{@url}",
 							timeout: 60 * 1000
@@ -204,6 +209,7 @@ describe "CompileController", ->
 				it "should proxy to the priority url if the user has the feature", ()->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi_priority.url}#{@url}",
 							timeout: 60 * 1000
@@ -218,6 +224,7 @@ describe "CompileController", ->
 				it "should open a request to the CLSI", ->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi.url}#{@url}",
 							timeout: 60 * 1000
@@ -240,6 +247,7 @@ describe "CompileController", ->
 				it "should proxy to the priority url if the user has the feature", ()->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi_priority.url}#{@url}",
 							timeout: 60 * 1000
@@ -254,6 +262,7 @@ describe "CompileController", ->
 				it "should proxy to the standard url", ()->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi.url}#{@url}",
 							timeout: 60 * 1000
@@ -269,6 +278,7 @@ describe "CompileController", ->
 				it "should proxy to the standard url without the build parameter", ()->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi.url}#{@url}",
 							timeout: 60 * 1000
@@ -286,6 +296,7 @@ describe "CompileController", ->
 				it "should open a request to the CLSI", ->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi.url}#{@url}",
 							timeout: 60 * 1000
@@ -313,6 +324,7 @@ describe "CompileController", ->
 				it "should proxy to the priority url if the user has the feature", ()->
 					@request
 						.calledWith(
+							jar:@jar
 							method: @req.method
 							url: "#{@settings.apis.clsi_priority.url}#{@url}",
 							timeout: 60 * 1000
@@ -331,8 +343,8 @@ describe "CompileController", ->
 					@CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next)
 
 				it "should proxy to the standard url with the build parameter", ()->
-					@request
-						.calledWith(
+					@request.calledWith(
+							jar:@jar
 							method: @req.method
 							qs: {build: 1234}
 							url: "#{@settings.apis.clsi.url}#{@url}",