Merge pull request #16 from sharelatex/cluster-restart

Graceful cluster restart on domain exception

services/filestore/Gruntfile.coffee

@@ -19,7 +19,7 @@ module.exports = (grunt) ->
 			app_server: 
 				expand: true,
 				flatten: false,
-				src: ['app.coffee'],
+				src: ['app.coffee', 'cluster.coffee'],
 				dest: './',
 				ext: '.js'
 
@@ -41,7 +41,7 @@ module.exports = (grunt) ->
 
 		watch:
 			server_coffee:
-				files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee"]
+				files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee", "cluster.coffee"]
 				tasks: ["clean", 'coffee', 'mochaTest']
 
 		clean: ["app/js", "test/unit/js", "app.js"]

services/filestore/app.coffee

@@ -37,12 +37,12 @@ app.use (req, res, next) ->
 	requestDomain.add req
 	requestDomain.add res
 	requestDomain.on "error", (err)->
-		# request a shutdown to prevent memory leaks
+		try
 			appIsOk = false
-		setTimeout(->
+			# request a shutdown to prevent memory leaks
+			beginShutdown()
 			if !res.headerSent
-				res.send(500)
-		, 3000)
+				res.send(500, "uncaught exception")
 			logger = require('logger-sharelatex')
 			req =
 				body:req.body
@@ -57,8 +57,16 @@ app.use (req, res, next) ->
 				type: err.type
 				arguments: err.arguments
 			logger.err err:err, req:req, res:res, "uncaught exception thrown on request"
+		catch exception
+			logger.err err: exception, "exception in request domain handler"
 	requestDomain.run next
 
+app.use (req, res, next) ->
+	if not appIsOk
+		# when shutting down, close any HTTP keep-alive connections
+		res.set 'Connection', 'close'
+	next()
+
 app.get  "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile
 app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile
 
@@ -95,19 +103,30 @@ app.get "/health_check", (req, res)->
 			else
 				res.send(503)
 
-
-
 app.get '*', (req, res)->
 	res.send 404
 
-serverDomain = domain.create()
-serverDomain.run ->
-	server = require('http').createServer(app)
-	port = settings.internal.filestore.port or 3009
-	host = settings.internal.filestore.host or "localhost"
-	server.listen port, host, ->
-		logger.log("filestore store listening on #{host}:#{port}")
+server = require('http').createServer(app)
+port = settings.internal.filestore.port or 3009
+host = settings.internal.filestore.host or "localhost"
 
-serverDomain.on "error", (err)->
-	logger.log err:err, "top level uncaught exception"
+beginShutdown = () ->
+	if appIsOk
+		appIsOk = false
+		# hard-terminate this process if graceful shutdown fails
+		killTimer = setTimeout () ->
+			process.exit 1
+		, 120*1000
+		killTimer.unref?() # prevent timer from keeping process alive
+		server.close () ->
+			logger.log "closed all connections"
+			Metrics.close()
+			process.disconnect?()
+		logger.log "server will stop accepting connections"
 
+server.listen port, host, ->
+	logger.log("filestore listening on #{host}:#{port}")
+
+process.on 'SIGTERM', () ->
+	logger.log("filestore got SIGTERM, shutting down gracefully")
+	beginShutdown()

services/filestore/cluster.coffee

@@ -0,0 +1,9 @@
+recluster = require "recluster" # https://github.com/doxout/recluster
+path = require "path"
+
+cluster = recluster path.join(__dirname, 'app.js'), {
+		workers: 2,
+		backoff: 0,
+		readyWhen: "listening"
+}
+cluster.run()

services/filestore/package.json

@@ -20,6 +20,7 @@
     "node-transloadit": "0.0.4",
     "node-uuid": "~1.4.1",
     "pngcrush": "0.0.3",
+    "recluster": "^0.3.7",
     "request": "2.14.0",
     "response": "0.14.0",
     "rimraf": "2.2.8",