From b8c3703c2f20a53e616d5b71e2fe62f9089d1b4b Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 14 Aug 2021 22:19:07 +0200 Subject: [PATCH 1/2] Fix endless loop on shutdown when DB can't be reached The shutdown handler calls `checkAllNotesRevision` on a 100 ms interval. If the database connection is broken, this will return an error. Previously, this error was effectively ignored and resulted in an endless loop printing out the error message every 100 ms. This improves the error handling by terminating the process with a nonzero exit code when an error was encountered 30 times. The loop interval is also increased to 200 ms, giving the database 6 seconds total time to recover in case of intermittent issues. Signed-off-by: David Mehren --- app.js | 17 ++++++++++++++--- public/docs/release-notes.md | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/app.js b/app.js index d37a883c8..21efd6798 100644 --- a/app.js +++ b/app.js @@ -331,17 +331,28 @@ function handleTermSignals () { } }) } + const maxCleanTries = 30 + let currentCleanTry = 1 const checkCleanTimer = setInterval(function () { if (realtime.isReady()) { models.Revision.checkAllNotesRevision(function (err, notes) { - if (err) return logger.error(err) + if (err) { + logger.error('Error while saving note revisions: ' + err) + if (currentCleanTry <= maxCleanTries) { + logger.warn(`Trying again. Try ${currentCleanTry} of ${maxCleanTries}`) + currentCleanTry++ + return null + } + logger.error(`Could not save note revisions after ${maxCleanTries} tries! Exiting.`) + process.exit(1) + } if (!notes || notes.length <= 0) { clearInterval(checkCleanTimer) - return process.exit(0) + process.exit(0) } }) } - }, 100) + }, 200) } process.on('SIGINT', handleTermSignals) process.on('SIGTERM', handleTermSignals) diff --git a/public/docs/release-notes.md b/public/docs/release-notes.md index 58b945fea..7134a3d92 100644 --- a/public/docs/release-notes.md +++ b/public/docs/release-notes.md @@ -11,6 +11,7 @@ ### Bugfixes - Fix crash when trying to read the current Git commit on startup +- Fix endless loop on shutdown when HedgeDoc can't connect to the database ## 1.8.2 2021-05-11 From b8bb40b9b621cca54274e5b3f3eb22c98eb6c6de Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 14 Aug 2021 23:59:37 +0200 Subject: [PATCH 2/2] Forcefully exit on second term signal Signed-off-by: David Mehren --- app.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app.js b/app.js index 21efd6798..45e70c8b5 100644 --- a/app.js +++ b/app.js @@ -309,9 +309,15 @@ process.on('uncaughtException', function (err) { process.exit(1) }) +let alreadyHandlingTermSignals = false // install exit handler function handleTermSignals () { + if (alreadyHandlingTermSignals) { + logger.info('Forcefully exiting.') + process.exit(1) + } logger.info('HedgeDoc has been killed by signal, try to exit gracefully...') + alreadyHandlingTermSignals = true realtime.maintenance = true // disconnect all socket.io clients Object.keys(io.sockets.sockets).forEach(function (key) {