Merge pull request #671 from sharelatex/ho-clsi-double-backend

send compile requests to 2 backends

services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee

@@ -7,31 +7,35 @@ if Settings.redis.clsi_cookie_secondary?
 Cookie = require('cookie')
 logger = require "logger-sharelatex"
 
-buildKey = (project_id)->
-	return "clsiserver:#{project_id}"
 
 clsiCookiesEnabled = Settings.clsiCookie?.key? and Settings.clsiCookie.key.length != 0
 
 
-module.exports = ClsiCookieManager =
+module.exports = (backendGroup)->
+
+	buildKey : (project_id)->
+		if backendGroup?
+			return "clsiserver:#{backendGroup}:#{project_id}"
+		else
+			return "clsiserver:#{project_id}"
 
 	_getServerId : (project_id, callback = (err, serverId)->)->
-		rclient.get buildKey(project_id), (err, serverId)->
+		rclient.get @buildKey(project_id), (err, serverId)=>
 			if err?
 				return callback(err)
 			if !serverId? or serverId == ""
-				return ClsiCookieManager._populateServerIdViaRequest project_id, callback
+				return @_populateServerIdViaRequest project_id, callback
 			else
 				return callback(null, serverId)
 
 
 	_populateServerIdViaRequest :(project_id, callback = (err, serverId)->)->
 		url = "#{Settings.apis.clsi.url}/project/#{project_id}/status"
-		request.get url, (err, res, body)->
+		request.get url, (err, res, body)=>
 			if err?
 				logger.err err:err, project_id:project_id, "error getting initial server id for project"
 				return callback(err)
-			ClsiCookieManager.setServerId project_id, res, (err, serverId)->
+			@setServerId project_id, res, (err, serverId)->
 				if err?
 					logger.err err:err, project_id:project_id, "error setting server id via populate request"
 				callback(err, serverId)
@@ -43,7 +47,7 @@ module.exports = ClsiCookieManager =
 	setServerId: (project_id, response, callback = (err, serverId)->)->
 		if !clsiCookiesEnabled
 			return callback()
-		serverId = ClsiCookieManager._parseServerIdFromResponse(response)
+		serverId = @_parseServerIdFromResponse(response)
 		if !serverId? # We don't get a cookie back if it hasn't changed
 			return callback()
 		if rclient_secondary?
@@ -53,19 +57,19 @@ module.exports = ClsiCookieManager =
 
 	_setServerIdInRedis: (rclient, project_id, serverId, callback = (err) ->) ->
 		multi = rclient.multi()
-		multi.set buildKey(project_id), serverId
+		multi.set @buildKey(project_id), serverId
-		multi.expire buildKey(project_id), Settings.clsiCookie.ttl
+		multi.expire @buildKey(project_id), Settings.clsiCookie.ttl
 		multi.exec callback
 
 	clearServerId: (project_id, callback = (err)->)->
 		if !clsiCookiesEnabled
 			return callback()
-		rclient.del buildKey(project_id), callback
+		rclient.del @buildKey(project_id), callback
 
 	getCookieJar: (project_id, callback = (err, jar)->)->
 		if !clsiCookiesEnabled
 			return callback(null, request.jar())
-		ClsiCookieManager._getServerId project_id, (err, serverId)=>
+		@_getServerId project_id, (err, serverId)=>
 			if err?
 				logger.err err:err, project_id:project_id, "error getting server id"
 				return callback(err)

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -7,7 +7,8 @@ ProjectGetter = require("../Project/ProjectGetter")
 ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 logger = require "logger-sharelatex"
 Url = require("url")
-ClsiCookieManager = require("./ClsiCookieManager")
+ClsiCookieManager = require("./ClsiCookieManager")()
+NewBackendCloudClsiCookieManager = require("./ClsiCookieManager")("newBackendcloud")
 ClsiStateManager = require("./ClsiStateManager")
 _ = require("underscore")
 async = require("async")
@@ -88,21 +89,60 @@ module.exports = ClsiManager =
 					callback(null, response?.compile?.status, outputFiles, clsiServerId)
 
 	_makeRequest: (project_id, opts, callback)->
+		async.series {
+			currentBackend: (cb)-> 
+				startTime = new Date()
 				ClsiCookieManager.getCookieJar project_id, (err, jar)->
 					if err?
 						logger.err err:err, "error getting cookie jar for clsi request"
 						return callback(err)
 					opts.jar = jar
+					timer = new Metrics.Timer("compile.currentBackend")
 					request opts, (err, response, body)->
+						timer.done()
 						if err?
 							logger.err err:err, project_id:project_id, url:opts?.url, "error making request to clsi"
 							return callback(err)
 						ClsiCookieManager.setServerId project_id, response, (err)->
 							if err?
 								logger.warn err:err, project_id:project_id, "error setting server id"
+							callback err, response, body #return as soon as the standard compile has returned
+							cb(err, {response:response, body:body, finishTime:new Date() - startTime })
+			newBackend: (cb)-> 
+				startTime = new Date()
+				ClsiManager._makeNewBackendRequest project_id, opts, (err, response, body)->
+					cb(err, {response:response, body:body, finishTime:new Date() - startTime})
+		}, (err, results)->
+			timeDifference = results.newBackend?.finishTime - results.currentBackend?.finishTime 
+			statusCodeSame = results.newBackend?.response?.statusCode == results.currentBackend?.response?.statusCode
+			currentCompileTime = results.currentBackend?.finishTime
+			newBackendCompileTime = results.newBackend?.finishTime
+			logger.log {statusCodeSame, timeDifference, currentCompileTime, newBackendCompileTime}, "both clsi requests returned"
 
+
+
+	_makeNewBackendRequest: (project_id, baseOpts, callback)->
+		if !Settings.apis.clsi_new?.url?
+			return callback()
+		opts = _.clone(baseOpts)
+		opts.url = opts.url.replace(Settings.apis.clsi.url, Settings.apis.clsi_new?.url)
+		NewBackendCloudClsiCookieManager.getCookieJar project_id, (err, jar)->
+			if err?
+				logger.err err:err, "error getting cookie jar for clsi request"
+				return callback(err)
+			opts.jar = jar
+			timer = new Metrics.Timer("compile.newBackend")
+			request opts, (err, response, body)->
+				timer.done()
+				if err?
+					logger.warn err:err, project_id:project_id, url:opts?.url, "error making request to new clsi"
+					return callback(err)
+				NewBackendCloudClsiCookieManager.setServerId project_id, response, (err)->
+					if err?
+						logger.warn err:err, project_id:project_id, "error setting server id new backend"
 					return callback err, response, body
 
+
 	_getCompilerUrl: (compileGroup, project_id, user_id, action) ->
 		host = Settings.apis.clsi.url
 		path = "/project/#{project_id}"

services/web/app/coffee/Features/Compile/CompileController.coffee

@@ -9,7 +9,7 @@ Settings = require "settings-sharelatex"
 AuthenticationController = require "../Authentication/AuthenticationController"
 UserGetter = require "../User/UserGetter"
 RateLimiter = require("../../infrastructure/RateLimiter")
-ClsiCookieManager = require("./ClsiCookieManager")
+ClsiCookieManager = require("./ClsiCookieManager")()
 Path = require("path")
 
 module.exports = CompileController =

services/web/test/unit/coffee/Compile/ClsiCookieManagerTests.coffee

@@ -40,7 +40,7 @@ describe "ClsiCookieManager", ->
 			"request": @request
 
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
-		@ClsiCookieManager = SandboxedModule.require modulePath, requires:@requires
+		@ClsiCookieManager = SandboxedModule.require(modulePath, {requires:@requires})()
 
 
 
@@ -107,7 +107,7 @@ describe "ClsiCookieManager", ->
 
 		it "should not set the server id if clsiCookies are not enabled", (done)->
 			delete @settings.clsiCookie.key 
-			@ClsiCookieManager = SandboxedModule.require modulePath, requires:@requires
+			@ClsiCookieManager = SandboxedModule.require(modulePath, requires:@requires)()
 			@ClsiCookieManager.setServerId @project_id, @response, (err, serverId)=>
 				@redisMulti.exec.called.should.equal false
 				done()
@@ -129,7 +129,7 @@ describe "ClsiCookieManager", ->
 			@RedisWrapper.client = sinon.stub()
 			@RedisWrapper.client.withArgs("clsi_cookie").returns(@redis)
 			@RedisWrapper.client.withArgs("clsi_cookie_secondary").returns(@redis_secondary)
-			@ClsiCookieManager = SandboxedModule.require modulePath, requires:@requires
+			@ClsiCookieManager = SandboxedModule.require(modulePath, requires:@requires)()
 			@ClsiCookieManager._parseServerIdFromResponse = sinon.stub().returns("clsi-8")
 			@ClsiCookieManager.setServerId @project_id, @response, (err, serverId)=>
 				@redisSecondaryMulti.set.calledWith("clsiserver:#{@project_id}", "clsi-8").should.equal true
@@ -150,7 +150,7 @@ describe "ClsiCookieManager", ->
 
 		it "should return empty cookie jar if clsiCookies are not enabled", (done)->
 			delete @settings.clsiCookie.key 
-			@ClsiCookieManager = SandboxedModule.require modulePath, requires:@requires
+			@ClsiCookieManager = SandboxedModule.require(modulePath, requires:@requires)()
 			@ClsiCookieManager.getCookieJar @project_id, (err, jar)->
 				assert.deepEqual jar, realRequst.jar()
 				done()

services/web/test/unit/coffee/Compile/ClsiManagerTests.coffee

@@ -31,7 +31,7 @@ describe "ClsiManager", ->
 			"../Project/ProjectGetter": @ProjectGetter = {}
 			"../DocumentUpdater/DocumentUpdaterHandler": @DocumentUpdaterHandler =
 				getProjectDocsIfMatch: sinon.stub().callsArgWith(2,null,null)
-			"./ClsiCookieManager": @ClsiCookieManager
+			"./ClsiCookieManager": => @ClsiCookieManager
 			"./ClsiStateManager": @ClsiStateManager
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err: sinon.stub(), warn: sinon.stub() }
 			"request": @request = sinon.stub()
@@ -557,6 +557,28 @@ describe "ClsiManager", ->
 				done()
 
 
+	describe "_makeGoogleCloudRequest", ->
+
+		beforeEach ->
+			@settings.apis.clsi_new = 
+				url : "https://compiles.somewhere.test"
+			@response = {there:"something"}
+			@request.callsArgWith(1, null, @response)
+			@opts = 
+				url: @ClsiManager._getCompilerUrl(null, @project_id)
+				
+		it "should change the domain on the url", (done)->
+			@ClsiManager._makeNewBackendRequest @project_id, @opts, =>
+				args = @request.args[0]
+				args[0].url.should.equal "https://compiles.somewhere.test/project/#{@project_id}"
+				done()
+
+		it "should not make a request if there is not clsi_new url", (done)->
+			@settings.apis.clsi_new = undefined
+			@ClsiManager._makeNewBackendRequest @project_id, @opts, (err)=>
+				expect(err).to.equal undefined
+				@request.callCount.should.equal 0
+				done()
 
 
 

services/web/test/unit/coffee/Compile/CompileControllerTests.coffee

@@ -51,7 +51,7 @@ describe "CompileController", ->
 			"./ClsiManager": @ClsiManager
 			"../Authentication/AuthenticationController": @AuthenticationController
 			"../../infrastructure/RateLimiter":@RateLimiter
-			"./ClsiCookieManager":@ClsiCookieManager
+			"./ClsiCookieManager": ()=> @ClsiCookieManager
 		@project_id = "project-id"
 		@next = sinon.stub()
 		@req = new MockRequest()