Merge pull request #19098 from overleaf/jpa-graceful-shutdown-tests

[server-ce] add test for graceful shutdown

GitOrigin-RevId: 5e72e0073169009d2e3ece5a79cbd62051f6ad5b
This commit is contained in:
Jakob Ackermann 2024-06-25 13:03:05 +02:00 committed by Copybot
parent 956aad7e43
commit ce9b531892
6 changed files with 135 additions and 8 deletions

View file

@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
sleep 5
# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/overleaf/web.log 2>&1
cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
@ -21,9 +21,14 @@ then
exit 1
fi
sleep 10 &
GIVE_EDITOR_10_SECONDS_TO_RELOAD=$!
# wait for disconnection
while ! sv stop real-time-overleaf; do
sleep 1
done
wait $GIVE_EDITOR_10_SECONDS_TO_RELOAD
exit 0

View file

@ -0,0 +1,103 @@
import {
ensureUserExists,
login,
resetCreatedUsersCache,
} from './helpers/login'
import { STARTUP_TIMEOUT, startWith } from './helpers/config'
import {
dockerCompose,
getRedisKeys,
resetData,
} from './helpers/hostAdminClient'
import { createProject } from './helpers/project'
import { throttledRecompile } from './helpers/compile'
const USER = 'user@example.com'
const PROJECT_NAME = 'Old Project'
function bringServerProBackUp() {
cy.log('bring server pro back up')
cy.then({ timeout: STARTUP_TIMEOUT }, async () => {
await dockerCompose('up', '--detach', '--wait', 'sharelatex')
})
}
describe('GracefulShutdown', function () {
before(async () => {
resetCreatedUsersCache()
await resetData()
})
startWith({
pro: true,
withDataDir: true,
})
ensureUserExists({ email: USER })
let projectId: string
it('should display banner and flush changes out of redis', () => {
bringServerProBackUp()
login(USER)
cy.visit('/project')
createProject(PROJECT_NAME).then(id => {
projectId = id
})
const recompile = throttledRecompile()
cy.log('add additional content')
cy.findByText('\\maketitle').parent().click()
cy.findByText('\\maketitle').parent().type(`\n\\section{{}New Section}`)
recompile()
cy.log(
'check flush from frontend to backend: should include new section in PDF'
)
cy.get('.pdf-viewer').should('contain.text', 'New Section')
cy.log('should have unflushed content in redis before shutdown')
cy.then(async () => {
const keys = await getRedisKeys()
expect(keys).to.contain(`DocsIn:${projectId}`)
expect(keys).to.contain(`ProjectHistory:Ops:{${projectId}}`)
})
cy.log('trigger graceful shutdown')
let pendingShutdown: Promise<any>
cy.then(() => {
pendingShutdown = dockerCompose('stop', '--timeout=60', 'sharelatex')
})
cy.log('wait for banner')
cy.findByText(/performing maintenance/)
cy.log('wait for page reload')
cy.findByText(/is currently down for maintenance/)
cy.log('wait for shutdown to complete')
cy.then({ timeout: 60 * 1000 }, async () => {
await pendingShutdown
})
cy.log('should not have any unflushed content in redis after shutdown')
cy.then(async () => {
const keys = await getRedisKeys()
expect(keys).to.not.contain(`DocsIn:${projectId}`)
expect(keys).to.not.contain(`ProjectHistory:Ops:{${projectId}}`)
})
bringServerProBackUp()
cy.then(() => {
cy.visit(`/project/${projectId}?trick-cypress-into-page-reload=true`)
})
cy.log('check loading doc from mongo')
cy.findByText('New Section')
cy.log('check PDF')
cy.get('.pdf-viewer').should('contain.text', 'New Section')
cy.log('check history')
cy.findByText('History').click()
cy.findByText(/\\section\{New Section}/)
})
})

View file

@ -1,5 +1,8 @@
import { reconfigure } from './hostAdminClient'
export const STARTUP_TIMEOUT =
parseInt(Cypress.env('STARTUP_TIMEOUT'), 10) || 120_000
let lastConfig: string
export function startWith({
@ -14,7 +17,7 @@ export function startWith({
const cfg = JSON.stringify({ pro, version, vars, withDataDir })
if (lastConfig === cfg) return
this.timeout(100 * 1000)
this.timeout(STARTUP_TIMEOUT)
await reconfigure({ pro, version, vars, withDataDir })
lastConfig = cfg
})

View file

@ -76,3 +76,10 @@ export async function runScript({
}),
})
}
export async function getRedisKeys() {
const { stdout } = await fetchJSON(`${hostAdminUrl}/redis/keys`, {
method: 'GET',
})
return stdout.split('\n')
}

View file

@ -197,8 +197,7 @@ app.post(
'--detach',
'--wait',
'--volumes',
'--timeout',
'0',
'--timeout=60',
'sharelatex',
'git-bridge',
'mongo',
@ -315,6 +314,16 @@ app.post('/reset/data', (req, res) => {
)
})
app.get('/redis/keys', (req, res) => {
runDockerCompose(
'exec',
['redis', 'redis-cli', 'KEYS', '*'],
(error, stdout, stderr) => {
res.json({ error, stdout, stderr })
}
)
})
app.use(handleValidationErrors())
purgeDataDir()

View file

@ -186,17 +186,17 @@ server.listen(port, host, function (error) {
// Stop huge stack traces in logs from all the socket.io parsing steps.
Error.stackTraceLimit = 10
function shutdownCleanly(signal) {
function shutdownAfterAllClientsHaveDisconnected() {
const connectedClients = io.sockets.clients().length
if (connectedClients === 0) {
logger.info('no clients connected, exiting')
logger.info({}, 'no clients connected, exiting')
process.exit()
} else {
logger.info(
{ connectedClients },
'clients still connected, not shutting down yet'
)
setTimeout(() => shutdownCleanly(signal), 30 * 1000)
setTimeout(() => shutdownAfterAllClientsHaveDisconnected(), 5_000)
}
}
@ -218,6 +218,7 @@ function drainAndShutdown(signal) {
`received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`
)
DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow, () => {
shutdownAfterAllClientsHaveDisconnected()
setTimeout(() => {
const staleClients = io.sockets.clients()
if (staleClients.length !== 0) {
@ -233,7 +234,6 @@ function drainAndShutdown(signal) {
Settings.shutDownComplete = true
}, Settings.gracefulReconnectTimeoutMs)
})
shutdownCleanly(signal)
}, statusCheckInterval)
}
}