Catch errors from socket.io and attempt graceful cleanup

services/real-time/app.coffee

@@ -119,22 +119,32 @@ shutdownCleanly = (signal) ->
 			shutdownCleanly(signal)
 		, 10000
 
+drainAndShutdown = (signal) ->
+	if Settings.shutDownInProgress
+		logger.log signal: signal, "shutdown already in progress, ignoring signal"
+		return
+	else
+		Settings.shutDownInProgress = true
+		logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins"
+		DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
+		shutdownCleanly(signal)
+
+
 Settings.shutDownInProgress = false
 if Settings.shutdownDrainTimeWindow?
 	shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10)
 	logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled"
 	for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
 		process.on signal, ->
-			if Settings.shutDownInProgress
-				logger.log signal: signal, "shutdown already in progress, ignoring signal"
-				return
-			else
-				Settings.shutDownInProgress = true
-				logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins"
-				DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
-				shutdownCleanly(signal)
-
+			drainAndShutdown(signal)
 
+	# global exception handler
+	if Settings.errors?.catchUncaughtErrors
+		process.removeAllListeners('uncaughtException')
+		process.on 'uncaughtException', (error) ->
+			logger.error err: error, 'uncaught exception'
+			if Settings.errors?.shutdownOnUncaughtError
+				drainAndShutdown('SIGABRT')
 
 if Settings.continualPubsubTraffic
 	console.log "continualPubsubTraffic enabled"

services/real-time/app/coffee/DrainManager.coffee

@@ -36,4 +36,4 @@ module.exports = DrainManager =
 			if haveDrainedNClients
 				break
 		if drainedCount < N
-			logger.log "All clients have been told to reconnectGracefully"
+			logger.log "All clients have been told to reconnectGracefully"

services/real-time/app/coffee/Router.coffee

@@ -42,6 +42,12 @@ module.exports = Router =
 		app.post "/drain", httpAuth, HttpApiController.startDrain
 
 		session.on 'connection', (error, client, session) ->
+			client?.on "error", (err) ->
+				logger.err "socket.io client error", { err }
+				if client.connected
+					client.emit("reconnectGracefully")
+					client.disconnect()
+
 			if settings.shutDownInProgress
 				client.emit("connectionRejected", {message: "retry"})
 				client.disconnect()

services/real-time/config/settings.defaults.coffee

@@ -62,6 +62,10 @@ settings =
 
 	sentry:
 		dsn: process.env.SENTRY_DSN
+
+	errors:
+		catchUncaughtErrors: true
+		shutdownOnUncaughtError: true
 	
 # console.log settings.redis
 module.exports = settings