Merge pull request #11733 from overleaf/msm-close-site-sp-shutdwn

[CE/SP] close site on shutdown before flushing

GitOrigin-RevId: e36c80f2f4c6f10eda3a536319a2fcc5dfda4fa4
This commit is contained in:
Tim Down 2023-02-08 15:19:18 +00:00 committed by Copybot
parent 1a4011fb61
commit 7c7f8c1573
5 changed files with 109 additions and 0 deletions

View file

@ -70,6 +70,11 @@ COPY server-ce/config/production.json /overleaf/services/history-v1/config/produ
ADD server-ce/bin/grunt /usr/local/bin/grunt
RUN chmod +x /usr/local/bin/grunt
# File that controls open|closed status of the site
# -------------------------------------------------
ENV SITE_MAINTENANCE_FILE "/etc/sharelatex/site_status"
RUN touch $SITE_MAINTENANCE_FILE
# Set Environment Variables
# --------------------------------
ENV SHARELATEX_CONFIG /etc/sharelatex/settings.js

View file

@ -0,0 +1,23 @@
#!/bin/sh
SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown
mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}"
echo "closed" > "${SITE_MAINTENANCE_FILE}"
# status file is polled every 5 seconds
sleep 5
# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/sharelatex/web.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
then
echo "scripts/disconnect_all_users.js failed with exit code $EXIT_CODE"
fi
# wait for disconnection
sleep 5
exit 0

View file

@ -0,0 +1,12 @@
#!/bin/bash
set -e
# pre-shutdown scripts close the site by overriding the content of SITE_MAINTENANCE_FILE,
# this script restores the original value on container restart
SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown"
if [ -f "${SITE_MAINTENANCE_FILE_BAK}" ]; then
mv -f "${SITE_MAINTENANCE_FILE_BAK}" "${SITE_MAINTENANCE_FILE}"
rm -f "${SITE_MAINTENANCE_FILE_BAK}"
fi

View file

@ -14,6 +14,7 @@ metrics.initialize(process.env.METRICS_APP_NAME || 'web')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const PlansLocator = require('./app/src/Features/Subscription/PlansLocator')
const SiteAdminHandler = require('./app/src/infrastructure/SiteAdminHandler')
logger.initialize(process.env.METRICS_APP_NAME || 'web')
logger.logger.serializers.user =
require('./app/src/infrastructure/LoggerSerializers').user
@ -78,6 +79,9 @@ if (!module.parent) {
})
}
// monitor site maintenance file
SiteAdminHandler.initialise()
// handle SIGTERM for graceful shutdown in kubernetes
process.on('SIGTERM', function (signal) {
triggerGracefulShutdown(Server.server, signal)

View file

@ -0,0 +1,65 @@
const logger = require('@overleaf/logger')
const settings = require('@overleaf/settings')
const fs = require('fs')
const {
addOptionalCleanupHandlerAfterDrainingConnections,
} = require('./GracefulShutdown')
// Monitor a site maintenance file (e.g. /etc/site_status) periodically and
// close the site if the file contents contain the string "closed".
const STATUS_FILE_CHECK_INTERVAL = 5000
const statusFile = settings.siteMaintenanceFile
function updateSiteMaintenanceStatus(fileContent) {
const isClosed = !settings.siteIsOpen
const shouldBeClosed = fileContent && fileContent.indexOf('closed') >= 0
if (!isClosed && shouldBeClosed) {
settings.siteIsOpen = false
logger.warn({ fileContent }, 'putting site into maintenance mode')
} else if (isClosed && !shouldBeClosed) {
settings.siteIsOpen = true
logger.warn({ fileContent }, 'taking site out of maintenance mode')
}
}
function pollSiteMaintenanceFile() {
fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => {
if (err) {
logger.error(
{ file: statusFile, fsErr: err },
'error reading site maintenance file'
)
return
}
updateSiteMaintenanceStatus(fileContent)
})
}
function checkSiteMaintenanceFileSync() {
// crash on start up if file does not exist
const content = fs.readFileSync(statusFile, { encoding: 'utf8' })
updateSiteMaintenanceStatus(content)
}
module.exports = {
initialise() {
if (settings.enabledServices.includes('web') && statusFile) {
logger.debug(
{ statusFile, interval: STATUS_FILE_CHECK_INTERVAL },
'monitoring site maintenance file'
)
checkSiteMaintenanceFileSync() // perform an initial synchronous check at start up
const intervalHandle = setInterval(
pollSiteMaintenanceFile,
STATUS_FILE_CHECK_INTERVAL
) // continue checking periodically
addOptionalCleanupHandlerAfterDrainingConnections(
'poll site maintenance file',
() => {
clearInterval(intervalHandle)
}
)
}
},
}