diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 708e5f467c..c2f9e7fa9c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -119,22 +119,32 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 +drainAndShutdown = (signal) -> + if Settings.shutDownInProgress + logger.log signal: signal, "shutdown already in progress, ignoring signal" + return + else + Settings.shutDownInProgress = true + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + shutdownCleanly(signal) + + Settings.shutDownInProgress = false if Settings.shutdownDrainTimeWindow? shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if Settings.shutDownInProgress - logger.log signal: signal, "shutdown already in progress, ignoring signal" - return - else - Settings.shutDownInProgress = true - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) - + drainAndShutdown(signal) + # global exception handler + if Settings.errors?.catchUncaughtErrors + process.removeAllListeners('uncaughtException') + process.on 'uncaughtException', (error) -> + logger.error err: error, 'uncaught exception' + if Settings.errors?.shutdownOnUncaughtError + drainAndShutdown('SIGABRT') if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index ad142ccc47..2590a96726 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -36,4 +36,4 @@ module.exports = DrainManager = if haveDrainedNClients break if drainedCount < N - logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file + logger.log "All clients have been told to reconnectGracefully" diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 92a17729b7..ae4d404e48 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -42,6 +42,12 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + client?.on "error", (err) -> + logger.err "socket.io client error", { err } + if client.connected + client.emit("reconnectGracefully") + client.disconnect() + if settings.shutDownInProgress client.emit("connectionRejected", {message: "retry"}) client.disconnect() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7fba6ea0d6..75b39ab2ee 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -62,6 +62,10 @@ settings = sentry: dsn: process.env.SENTRY_DSN + + errors: + catchUncaughtErrors: true + shutdownOnUncaughtError: true # console.log settings.redis module.exports = settings