Merge pull request #16554 from overleaf/bg-clsi-timeout-filestore-graceful-shutdown

Add graceful shutdown to filestore

GitOrigin-RevId: a940fe1c68a32aa281b4be866e350e3e6b184c16
This commit is contained in:
Brian Gough 2024-01-17 11:30:33 +00:00 committed by Copybot
parent 56a72690d5
commit f5442626f4
2 changed files with 36 additions and 1 deletions

View file

@ -39,6 +39,19 @@ app.use(function (req, res, next) {
next() next()
}) })
// Handle requests that come in after we've started shutting down
app.use((req, res, next) => {
if (settings.shuttingDown) {
logger.warn(
{ req, timeSinceShutdown: Date.now() - settings.shutDownTime },
'request received after shutting down'
)
// We don't want keep-alive connections to be kept open when the server is shutting down.
res.set('Connection', 'close')
}
next()
})
Metrics.injectMetricsRoute(app) Metrics.injectMetricsRoute(app)
app.head( app.head(
@ -134,7 +147,11 @@ app.get(
) )
app.get('/status', function (req, res) { app.get('/status', function (req, res) {
res.send('filestore sharelatex up') if (settings.shuttingDown) {
res.sendStatus(503) // Service unavailable
} else {
res.send('filestore is up')
}
}) })
app.get('/health_check', healthCheckController.check) app.get('/health_check', healthCheckController.check)
@ -164,4 +181,20 @@ process
process.exit(1) process.exit(1)
}) })
function handleShutdownSignal(signal) {
logger.info({ signal }, 'received interrupt, cleaning up')
if (settings.shuttingDown) {
logger.warn({ signal }, 'already shutting down, ignoring interrupt')
return
}
settings.shuttingDown = true
settings.shutDownTime = Date.now()
setTimeout(() => {
logger.info({ signal }, 'shutting down')
process.exit()
}, settings.delayShutdownMs)
}
process.on('SIGTERM', handleShutdownSignal)
module.exports = app module.exports = app

View file

@ -112,6 +112,8 @@ const settings = {
sentry: { sentry: {
dsn: process.env.SENTRY_DSN, dsn: process.env.SENTRY_DSN,
}, },
delayShutdownMs: parseInt(process.env.DELAY_SHUTDOWN_MS || '30000', 10),
} }
// Filestore health check // Filestore health check