From 7f2decae4d4e7b33787693729516898548c5bc52 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:06:39 +0000 Subject: [PATCH] start drain after 3 hours --- services/real-time/app.coffee | 39 ++++++++++++------- .../app/coffee/ConnectedUsersManager.coffee | 2 - .../coffee/DocumentUpdaterController.coffee | 2 - .../real-time/config/settings.defaults.coffee | 4 +- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 034dc2d915..20c89a2681 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -19,6 +19,8 @@ Metrics.initialize(Settings.appName or "real-time") Metrics.event_loop.monitor(logger) +DrainManager = require("./app/js/DrainManager") + # Set up socket.io server app = express() server = require('http').createServer(app) @@ -81,17 +83,28 @@ Error.stackTraceLimit = 10 shutdownCleanly = (signal) -> - return () -> - logger.log signal: signal, "received interrupt, cleaning up" - connectedClients = io.sockets.clients()?.length - logger.log {connectedClients, signal}, "looking to shut down process" - if connectedClients == 0 - logger.log("no clients connected, exiting") - process.exit() - else - setTimeout () -> - shutdownCleanly(signal) - , 10000 + connectedClients = io.sockets.clients()?.length + logger.log {connectedClients, signal}, "looking to shut down process" + if connectedClients == 0 + logger.log("no clients connected, exiting") + process.exit() + else + setTimeout () -> + shutdownCleanly(signal) + , 10000 -for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] - process.on signal, shutdownCleanly(signal) +forceDrain = -> + THREE_HOURS = 60 * 1000 * 60 * 3 + setTimeout( -> , + logger.log({delay_ms:THREE_HOURS}, "starting drain") + DrainManager.startDrain(io, 4) + , THREE_HOURS) + + +if Settings.drainBeforeShutdown + logger.log "drainBeforeShutdown enabled" + for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] + logger.log signal: signal, "received interrupt, cleaning up" + process.on signal, -> + shutdownCleanly(signal) + forceDrain() diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 38370da464..831a237356 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -5,8 +5,6 @@ redis = require("redis-sharelatex") rclient = redis.createClient(Settings.redis.realtime) Keys = Settings.redis.realtime.key_schema -console.log Settings.redis.realtime, "REALTIME" - ONE_HOUR_IN_S = 60 * 60 ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 05a70fe3a7..451dc812bc 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,8 +4,6 @@ redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" -console.log "REDIS", settings.redis - MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 485789cb12..270c32aa77 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -40,4 +40,6 @@ module.exports = cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid" - max_doc_length: 2 * 1024 * 1024 # 2mb \ No newline at end of file + max_doc_length: 2 * 1024 * 1024 # 2mb + + drainBeforeShutdown: process.env['DRAIN_BEFORE_SHUTDOWN'] or false \ No newline at end of file