Merge pull request #69 from overleaf/ho-drain-connections-timewindow

add shutdownDrainTimeWindow, drains all connections within time range

services/real-time/app.coffee

@@ -116,17 +116,10 @@ shutdownCleanly = (signal) ->
 			shutdownCleanly(signal)
 		, 10000
 
-forceDrain = ->
-	logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout"
-	setTimeout ()-> 
-		logger.log "starting drain"
-		DrainManager.startDrain(io, 4)
-	, Settings.forceDrainMsDelay
-
 shutDownInProgress = false
-if Settings.forceDrainMsDelay?
+if Settings.shutdownDrainTimeWindow?
-	Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10)
+	Settings.forceDrainMsDelay = parseInt(Settings.shutdownDrainTimeWindow, 10)
-	logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled"
+	logger.log shutdownDrainTimeWindow: Settings.shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled"
 	for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
 		process.on signal, ->
 			if shutDownInProgress
@@ -134,9 +127,9 @@ if Settings.forceDrainMsDelay?
 				return
 			else
 				shutDownInProgress = true
-				logger.log signal: signal, "received interrupt, cleaning up"
+				logger.log signal: signal, "received interrupt, starting drain over #{Settings.shutdownDrainTimeWindow} mins"
+				DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow)
 				shutdownCleanly(signal)
-				forceDrain()
 
 
 

services/real-time/app/coffee/DrainManager.coffee

@@ -1,6 +1,11 @@
 logger = require "logger-sharelatex"
 
-module.exports =
+module.exports = DrainManager =
+
+	startDrainTimeWindow: (io, minsToDrain)->
+		drainPerMin = io.sockets.clients().length / minsToDrain
+		DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate
+
 	startDrain: (io, rate) ->
 		# Clear out any old interval
 		clearInterval @interval

services/real-time/config/settings.defaults.coffee

@@ -48,7 +48,7 @@ settings =
 	
 	max_doc_length: 2 * 1024 * 1024 # 2mb
 
-	forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false
+	shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9
 
 	continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false
 

services/real-time/test/unit/coffee/DrainManagerTests.coffee

@@ -12,6 +12,23 @@ describe "DrainManager", ->
 			sockets:
 				clients: sinon.stub()
 
+	describe "startDrainTimeWindow", ->
+		beforeEach ->
+			@clients = []
+			for i in [0..5399]
+				@clients[i] = {
+					id: i
+					emit: sinon.stub()
+				}
+			@io.sockets.clients.returns @clients
+			@DrainManager.startDrain = sinon.stub()
+
+		it "should set a drain rate fast enough", (done)->
+			@DrainManager.startDrainTimeWindow(@io, 9)
+			@DrainManager.startDrain.calledWith(@io, 10).should.equal true
+			done()
+
+
 	describe "reconnectNClients", ->
 		beforeEach ->
 			@clients = []