From 00cca29d9ef23156657631c56994c4055dcdd820 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 13 Aug 2019 11:12:04 +0100 Subject: [PATCH 1/4] add shutdownDrainTimeWindow, drains all connections within time range --- services/real-time/app.coffee | 4 ++-- .../real-time/app/coffee/DrainManager.coffee | 7 ++++++- .../real-time/config/settings.defaults.coffee | 2 ++ .../test/unit/coffee/DrainManagerTests.coffee | 18 ++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 7b53730ce6..8b3486dc0a 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -113,8 +113,8 @@ shutdownCleanly = (signal) -> forceDrain = -> logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" setTimeout ()-> - logger.log "starting drain" - DrainManager.startDrain(io, 4) + logger.log "starting drain over #{Settings.shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) , Settings.forceDrainMsDelay shutDownInProgress = false diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index da6b331e23..a71ffa8ed9 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -1,6 +1,11 @@ logger = require "logger-sharelatex" -module.exports = +module.exports = DrainManager = + + startDrainTimeWindow: (io, minsToDrain)-> + drainPerMin = io.sockets.clients().length / minsToDrain + DrainManager.startDrain(io, drainPerMin / 60) + startDrain: (io, rate) -> # Clear out any old interval clearInterval @interval diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index a3128e84d1..b48de98981 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -50,6 +50,8 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false + shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 + continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee index b3cdaf1ca0..cd34aea4c4 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -12,6 +12,24 @@ describe "DrainManager", -> sockets: clients: sinon.stub() + describe "startDrainTimeWindow", -> + beforeEach -> + @clients = [] + for i in [0..1619] + @clients[i] = { + id: i + emit: sinon.stub() + } + @io.sockets.clients.returns @clients + @DrainManager.startDrain = sinon.stub() + + it "should set a drain rate fast enough", (done)-> + @DrainManager.startDrainTimeWindow(@io, 9) + console.log(@DrainManager.startDrain.args[0]) + @DrainManager.startDrain.calledWith(@io, 3).should.equal true + done() + + describe "reconnectNClients", -> beforeEach -> @clients = [] From b3e5709b649be38aeef72a1f86ccfb537013a4cf Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:15:30 +0100 Subject: [PATCH 2/4] enforce a minimum drain rate --- services/real-time/app/coffee/DrainManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index a71ffa8ed9..2219291267 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -4,7 +4,7 @@ module.exports = DrainManager = startDrainTimeWindow: (io, minsToDrain)-> drainPerMin = io.sockets.clients().length / minsToDrain - DrainManager.startDrain(io, drainPerMin / 60) + DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate startDrain: (io, rate) -> # Clear out any old interval From 7db882f3398452ef98c62010e49d76d3c6f1186e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 17:26:49 +0100 Subject: [PATCH 3/4] fix unit tests --- services/real-time/test/unit/coffee/DrainManagerTests.coffee | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee index cd34aea4c4..88009f02cd 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -15,7 +15,7 @@ describe "DrainManager", -> describe "startDrainTimeWindow", -> beforeEach -> @clients = [] - for i in [0..1619] + for i in [0..5399] @clients[i] = { id: i emit: sinon.stub() @@ -25,8 +25,7 @@ describe "DrainManager", -> it "should set a drain rate fast enough", (done)-> @DrainManager.startDrainTimeWindow(@io, 9) - console.log(@DrainManager.startDrain.args[0]) - @DrainManager.startDrain.calledWith(@io, 3).should.equal true + @DrainManager.startDrain.calledWith(@io, 10).should.equal true done() From 4a984f533e1db3f1bb927f98ecb399843bf69ee3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 14 Aug 2019 11:51:25 +0100 Subject: [PATCH 4/4] remove forceDrainMsDelay as soon as a pod is marked as being killed we should start draining --- services/real-time/app.coffee | 17 +++++------------ .../real-time/config/settings.defaults.coffee | 2 -- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 8b3486dc0a..50f7eb975b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -110,17 +110,10 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 -forceDrain = -> - logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" - setTimeout ()-> - logger.log "starting drain over #{Settings.shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) - , Settings.forceDrainMsDelay - shutDownInProgress = false -if Settings.forceDrainMsDelay? - Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) - logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" +if Settings.shutdownDrainTimeWindow? + Settings.forceDrainMsDelay = parseInt(Settings.shutdownDrainTimeWindow, 10) + logger.log shutdownDrainTimeWindow: Settings.shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> if shutDownInProgress @@ -128,9 +121,9 @@ if Settings.forceDrainMsDelay? return else shutDownInProgress = true - logger.log signal: signal, "received interrupt, cleaning up" + logger.log signal: signal, "received interrupt, starting drain over #{Settings.shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) shutdownCleanly(signal) - forceDrain() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index b48de98981..6fe0f4bd1e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -48,8 +48,6 @@ settings = max_doc_length: 2 * 1024 * 1024 # 2mb - forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false - shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false