From 73b4262186e51b5117d6d110831ddf54bd245409 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 30 Sep 2019 15:35:05 +0100 Subject: [PATCH 1/6] add continuous background flush --- services/document-updater/app.coffee | 5 +++++ .../app/coffee/DeleteQueueManager.coffee | 12 +++++++++++- .../document-updater/config/settings.defaults.coffee | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee index 2962860027..5eb47d9c9e 100644 --- a/services/document-updater/app.coffee +++ b/services/document-updater/app.coffee @@ -14,6 +14,7 @@ if Settings.sentry?.dsn? RedisManager = require('./app/js/RedisManager') DispatchManager = require('./app/js/DispatchManager') +DeleteQueueManager = require('./app/js/DeleteQueueManager') Errors = require "./app/js/Errors" HttpController = require "./app/js/HttpController" mongojs = require "./app/js/mongojs" @@ -146,3 +147,7 @@ module.exports = app for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, shutdownCleanly(signal) + +if Settings.continuousBackgroundFlush + logger.info "Starting continuous background flush" + DeleteQueueManager.startBackgroundFlush() \ No newline at end of file diff --git a/services/document-updater/app/coffee/DeleteQueueManager.coffee b/services/document-updater/app/coffee/DeleteQueueManager.coffee index f1af0ba244..3fc6b3f644 100644 --- a/services/document-updater/app/coffee/DeleteQueueManager.coffee +++ b/services/document-updater/app/coffee/DeleteQueueManager.coffee @@ -1,3 +1,4 @@ +Settings = require('settings-sharelatex') RedisManager = require "./RedisManager" ProjectManager = require "./ProjectManager" logger = require "logger-sharelatex" @@ -60,4 +61,13 @@ module.exports = DeleteQueueManager = count++ if flushed flushNextProject() - flushNextProject() \ No newline at end of file + flushNextProject() + + startBackgroundFlush: () -> + doFlush = () -> + if Settings.shuttingDown + logger.warn "discontinuing background flush due to shutdown" + return + DeleteQueueManager.flushAndDeleteOldProjects {timeout:1000,min_delete_age:3*60*1000,limit:1000}, () -> + setTimeout doFlush, 10 + doFlush() diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index bc83433484..327bf98ee4 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -95,3 +95,5 @@ module.exports = dsn: process.env.SENTRY_DSN publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false + + continuousBackgroundFlush: process.env['CONTINUOUS_BACKGROUND_FLUSH'] or false From a32495d2b4184b7ec7256cb0258bf9bd323e02f2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Oct 2019 14:09:41 +0100 Subject: [PATCH 2/6] make background flush more adaptive --- .../document-updater/app/coffee/DeleteQueueManager.coffee | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/DeleteQueueManager.coffee b/services/document-updater/app/coffee/DeleteQueueManager.coffee index 3fc6b3f644..731cdf77b7 100644 --- a/services/document-updater/app/coffee/DeleteQueueManager.coffee +++ b/services/document-updater/app/coffee/DeleteQueueManager.coffee @@ -64,10 +64,12 @@ module.exports = DeleteQueueManager = flushNextProject() startBackgroundFlush: () -> + SHORT_DELAY = 10 + LONG_DELAY = 1000 doFlush = () -> if Settings.shuttingDown logger.warn "discontinuing background flush due to shutdown" return - DeleteQueueManager.flushAndDeleteOldProjects {timeout:1000,min_delete_age:3*60*1000,limit:1000}, () -> - setTimeout doFlush, 10 + DeleteQueueManager.flushAndDeleteOldProjects {timeout:1000,min_delete_age:3*60*1000,limit:1000}, (err, flushed) -> + setTimeout doFlush, (if flushed > 10 then SHORT_DELAY else LONG_DELAY) doFlush() From 2c22a60052b04b47756f5c461164bb49c9f2e4b4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Oct 2019 15:01:20 +0100 Subject: [PATCH 3/6] add random jitter to cutoff time --- services/document-updater/app/coffee/DeleteQueueManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/DeleteQueueManager.coffee b/services/document-updater/app/coffee/DeleteQueueManager.coffee index 731cdf77b7..8cd4a66c21 100644 --- a/services/document-updater/app/coffee/DeleteQueueManager.coffee +++ b/services/document-updater/app/coffee/DeleteQueueManager.coffee @@ -23,7 +23,7 @@ async = require "async" module.exports = DeleteQueueManager = flushAndDeleteOldProjects: (options, callback) -> startTime = Date.now() - cutoffTime = startTime - options.min_delete_age + cutoffTime = startTime - options.min_delete_age + 100 * (Math.random() - 0.5) count = 0 flushProjectIfNotModified = (project_id, flushTimestamp, cb) -> From 2845b23b7030c227b226511ca88ade019bb161a7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Oct 2019 15:01:53 +0100 Subject: [PATCH 4/6] add smoothing of delete spikes --- services/document-updater/app/coffee/RedisManager.coffee | 5 +++-- services/document-updater/config/settings.defaults.coffee | 2 ++ .../test/acceptance/coffee/DeletingAProjectTests.coffee | 5 ++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 1490ac87f4..63086320d7 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -295,8 +295,9 @@ module.exports = RedisManager = multi.exec callback queueFlushAndDeleteProject: (project_id, callback) -> - # store the project id in a sorted set ordered by time - rclient.zadd keys.flushAndDeleteQueue(), Date.now(), project_id, callback + # store the project id in a sorted set ordered by time with a random offset to smooth out spikes + SMOOTHING_OFFSET = if Settings.smoothingOffset > 0 then Math.round(Settings.smoothingOffset * Math.random()) else 0 + rclient.zadd keys.flushAndDeleteQueue(), Date.now() + SMOOTHING_OFFSET, project_id, callback getNextProjectToFlushAndDelete: (cutoffTime, callback = (error, key, timestamp)->) -> # find the oldest queued flush that is before the cutoff time diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 327bf98ee4..2b070e562c 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -97,3 +97,5 @@ module.exports = publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false continuousBackgroundFlush: process.env['CONTINUOUS_BACKGROUND_FLUSH'] or false + + smoothingOffset: process.env['SMOOTHING_OFFSET'] or 1000 # milliseconds \ No newline at end of file diff --git a/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee b/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee index 91e4378dc2..6c1f7fe8ed 100644 --- a/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee +++ b/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee @@ -147,9 +147,8 @@ describe "Deleting a project", -> @statusCode = res.statusCode # after deleting the project and putting it in the queue, flush the queue setTimeout () -> - DocUpdaterClient.flushOldProjects (error, res, body) => - setTimeout done, 1000 # allow time for the flush to complete - , 100 + DocUpdaterClient.flushOldProjects done + , 2000 , 200 after -> From ae3ebf2db6411b9ba5121b4bbb3d388151c24a0b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Oct 2019 15:02:27 +0100 Subject: [PATCH 5/6] start background flush after http server has started --- services/document-updater/app.coffee | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee index 5eb47d9c9e..82598bd93f 100644 --- a/services/document-updater/app.coffee +++ b/services/document-updater/app.coffee @@ -143,11 +143,12 @@ host = Settings.internal.documentupdater.host or "localhost" if !module.parent # Called directly app.listen port, host, -> logger.info "Document-updater starting up, listening on #{host}:#{port}" + if Settings.continuousBackgroundFlush + logger.info "Starting continuous background flush" + DeleteQueueManager.startBackgroundFlush() + module.exports = app for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, shutdownCleanly(signal) -if Settings.continuousBackgroundFlush - logger.info "Starting continuous background flush" - DeleteQueueManager.startBackgroundFlush() \ No newline at end of file From 0c14b7d2f8b98ceeb7cf4585bf7b714b06cf9249 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Oct 2019 15:06:01 +0100 Subject: [PATCH 6/6] add comment about background flush limit --- .../document-updater/app/coffee/DeleteQueueManager.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/DeleteQueueManager.coffee b/services/document-updater/app/coffee/DeleteQueueManager.coffee index 8cd4a66c21..985222df69 100644 --- a/services/document-updater/app/coffee/DeleteQueueManager.coffee +++ b/services/document-updater/app/coffee/DeleteQueueManager.coffee @@ -70,6 +70,10 @@ module.exports = DeleteQueueManager = if Settings.shuttingDown logger.warn "discontinuing background flush due to shutdown" return - DeleteQueueManager.flushAndDeleteOldProjects {timeout:1000,min_delete_age:3*60*1000,limit:1000}, (err, flushed) -> + DeleteQueueManager.flushAndDeleteOldProjects { + timeout:1000, + min_delete_age:3*60*1000, + limit:1000 # high value, to ensure we always flush enough projects + }, (err, flushed) -> setTimeout doFlush, (if flushed > 10 then SHORT_DELAY else LONG_DELAY) doFlush()