From 7543f2fcbd39b4b7c2756cb61445d077a6bdc511 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 17 Oct 2019 12:45:56 +0100 Subject: [PATCH 1/4] Catch errors from socket.io and attempt graceful cleanup --- services/real-time/app.coffee | 28 +++++++++++++------ .../real-time/app/coffee/DrainManager.coffee | 2 +- services/real-time/app/coffee/Router.coffee | 6 ++++ .../real-time/config/settings.defaults.coffee | 4 +++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 708e5f467c..c2f9e7fa9c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -119,22 +119,32 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 +drainAndShutdown = (signal) -> + if Settings.shutDownInProgress + logger.log signal: signal, "shutdown already in progress, ignoring signal" + return + else + Settings.shutDownInProgress = true + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + shutdownCleanly(signal) + + Settings.shutDownInProgress = false if Settings.shutdownDrainTimeWindow? shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if Settings.shutDownInProgress - logger.log signal: signal, "shutdown already in progress, ignoring signal" - return - else - Settings.shutDownInProgress = true - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) - + drainAndShutdown(signal) + # global exception handler + if Settings.errors?.catchUncaughtErrors + process.removeAllListeners('uncaughtException') + process.on 'uncaughtException', (error) -> + logger.error err: error, 'uncaught exception' + if Settings.errors?.shutdownOnUncaughtError + drainAndShutdown('SIGABRT') if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index ad142ccc47..2590a96726 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -36,4 +36,4 @@ module.exports = DrainManager = if haveDrainedNClients break if drainedCount < N - logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file + logger.log "All clients have been told to reconnectGracefully" diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 92a17729b7..ae4d404e48 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -42,6 +42,12 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + client?.on "error", (err) -> + logger.err "socket.io client error", { err } + if client.connected + client.emit("reconnectGracefully") + client.disconnect() + if settings.shutDownInProgress client.emit("connectionRejected", {message: "retry"}) client.disconnect() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7fba6ea0d6..75b39ab2ee 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -62,6 +62,10 @@ settings = sentry: dsn: process.env.SENTRY_DSN + + errors: + catchUncaughtErrors: true + shutdownOnUncaughtError: true # console.log settings.redis module.exports = settings From ce366fdbeeb35955a107739262b7881a6f8dd974 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 17 Oct 2019 12:46:07 +0100 Subject: [PATCH 2/4] Bump Dockerfile to node 10 --- services/real-time/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index e7243c5291..7656cf4526 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.15.1 as app +FROM node:10.16.3 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:6.15.1 +FROM node:10.16.3 COPY --from=app /app /app From e04b6e1e49ca7710afcdd7e28581103f9e59ed4d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Feb 2020 14:46:14 +0000 Subject: [PATCH 3/4] Update app/coffee/Router.coffee Co-Authored-By: Jakob Ackermann --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ae4d404e48..0ec305ed3a 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -43,7 +43,7 @@ module.exports = Router = session.on 'connection', (error, client, session) -> client?.on "error", (err) -> - logger.err "socket.io client error", { err } + logger.err { err }, "socket.io client error" if client.connected client.emit("reconnectGracefully") client.disconnect() From 49a8e1214b476eeae2e80d3a6fc442759c20bc8b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Feb 2020 14:47:45 +0000 Subject: [PATCH 4/4] use a separate field for client errors --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 0ec305ed3a..fcdb0c93dc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -43,7 +43,7 @@ module.exports = Router = session.on 'connection', (error, client, session) -> client?.on "error", (err) -> - logger.err { err }, "socket.io client error" + logger.err { clientErr: err }, "socket.io client error" if client.connected client.emit("reconnectGracefully") client.disconnect()