From ce9b531892a882a23c3be7c8c03aa3bf97c810bc Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 25 Jun 2024 13:03:05 +0200 Subject: [PATCH] Merge pull request #19098 from overleaf/jpa-graceful-shutdown-tests [server-ce] add test for graceful shutdown GitOrigin-RevId: 5e72e0073169009d2e3ece5a79cbd62051f6ad5b --- .../init_preshutdown_scripts/00_close_site | 7 +- server-ce/test/graceful-shutdown.spec.ts | 103 ++++++++++++++++++ server-ce/test/helpers/config.ts | 5 +- server-ce/test/helpers/hostAdminClient.ts | 7 ++ server-ce/test/host-admin.js | 13 ++- services/real-time/app.js | 8 +- 6 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 server-ce/test/graceful-shutdown.spec.ts diff --git a/server-ce/init_preshutdown_scripts/00_close_site b/server-ce/init_preshutdown_scripts/00_close_site index fafc426b3f..0afb9e26e3 100755 --- a/server-ce/init_preshutdown_scripts/00_close_site +++ b/server-ce/init_preshutdown_scripts/00_close_site @@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}" sleep 5 # giving a grace period of 5 seconds for users before disconnecting them and start shutting down -cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/overleaf/web.log 2>&1 +cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1 EXIT_CODE="$?" if [ $EXIT_CODE -ne 0 ] @@ -21,9 +21,14 @@ then exit 1 fi +sleep 10 & +GIVE_EDITOR_10_SECONDS_TO_RELOAD=$! + # wait for disconnection while ! sv stop real-time-overleaf; do sleep 1 done +wait $GIVE_EDITOR_10_SECONDS_TO_RELOAD + exit 0 diff --git a/server-ce/test/graceful-shutdown.spec.ts b/server-ce/test/graceful-shutdown.spec.ts new file mode 100644 index 0000000000..61dc3f69e1 --- /dev/null +++ b/server-ce/test/graceful-shutdown.spec.ts @@ -0,0 +1,103 @@ +import { + ensureUserExists, + login, + resetCreatedUsersCache, +} from './helpers/login' +import { STARTUP_TIMEOUT, startWith } from './helpers/config' +import { + dockerCompose, + getRedisKeys, + resetData, +} from './helpers/hostAdminClient' +import { createProject } from './helpers/project' +import { throttledRecompile } from './helpers/compile' + +const USER = 'user@example.com' +const PROJECT_NAME = 'Old Project' + +function bringServerProBackUp() { + cy.log('bring server pro back up') + cy.then({ timeout: STARTUP_TIMEOUT }, async () => { + await dockerCompose('up', '--detach', '--wait', 'sharelatex') + }) +} + +describe('GracefulShutdown', function () { + before(async () => { + resetCreatedUsersCache() + await resetData() + }) + startWith({ + pro: true, + withDataDir: true, + }) + ensureUserExists({ email: USER }) + + let projectId: string + it('should display banner and flush changes out of redis', () => { + bringServerProBackUp() + login(USER) + + cy.visit('/project') + createProject(PROJECT_NAME).then(id => { + projectId = id + }) + const recompile = throttledRecompile() + + cy.log('add additional content') + cy.findByText('\\maketitle').parent().click() + cy.findByText('\\maketitle').parent().type(`\n\\section{{}New Section}`) + recompile() + + cy.log( + 'check flush from frontend to backend: should include new section in PDF' + ) + cy.get('.pdf-viewer').should('contain.text', 'New Section') + + cy.log('should have unflushed content in redis before shutdown') + cy.then(async () => { + const keys = await getRedisKeys() + expect(keys).to.contain(`DocsIn:${projectId}`) + expect(keys).to.contain(`ProjectHistory:Ops:{${projectId}}`) + }) + + cy.log('trigger graceful shutdown') + let pendingShutdown: Promise + cy.then(() => { + pendingShutdown = dockerCompose('stop', '--timeout=60', 'sharelatex') + }) + + cy.log('wait for banner') + cy.findByText(/performing maintenance/) + cy.log('wait for page reload') + cy.findByText(/is currently down for maintenance/) + + cy.log('wait for shutdown to complete') + cy.then({ timeout: 60 * 1000 }, async () => { + await pendingShutdown + }) + + cy.log('should not have any unflushed content in redis after shutdown') + cy.then(async () => { + const keys = await getRedisKeys() + expect(keys).to.not.contain(`DocsIn:${projectId}`) + expect(keys).to.not.contain(`ProjectHistory:Ops:{${projectId}}`) + }) + + bringServerProBackUp() + + cy.then(() => { + cy.visit(`/project/${projectId}?trick-cypress-into-page-reload=true`) + }) + + cy.log('check loading doc from mongo') + cy.findByText('New Section') + + cy.log('check PDF') + cy.get('.pdf-viewer').should('contain.text', 'New Section') + + cy.log('check history') + cy.findByText('History').click() + cy.findByText(/\\section\{New Section}/) + }) +}) diff --git a/server-ce/test/helpers/config.ts b/server-ce/test/helpers/config.ts index fa0b5ecaeb..23d20bb82e 100644 --- a/server-ce/test/helpers/config.ts +++ b/server-ce/test/helpers/config.ts @@ -1,5 +1,8 @@ import { reconfigure } from './hostAdminClient' +export const STARTUP_TIMEOUT = + parseInt(Cypress.env('STARTUP_TIMEOUT'), 10) || 120_000 + let lastConfig: string export function startWith({ @@ -14,7 +17,7 @@ export function startWith({ const cfg = JSON.stringify({ pro, version, vars, withDataDir }) if (lastConfig === cfg) return - this.timeout(100 * 1000) + this.timeout(STARTUP_TIMEOUT) await reconfigure({ pro, version, vars, withDataDir }) lastConfig = cfg }) diff --git a/server-ce/test/helpers/hostAdminClient.ts b/server-ce/test/helpers/hostAdminClient.ts index 9ed0088a83..e5fcdd0eb9 100644 --- a/server-ce/test/helpers/hostAdminClient.ts +++ b/server-ce/test/helpers/hostAdminClient.ts @@ -76,3 +76,10 @@ export async function runScript({ }), }) } + +export async function getRedisKeys() { + const { stdout } = await fetchJSON(`${hostAdminUrl}/redis/keys`, { + method: 'GET', + }) + return stdout.split('\n') +} diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js index 03b7521d37..41365af551 100644 --- a/server-ce/test/host-admin.js +++ b/server-ce/test/host-admin.js @@ -197,8 +197,7 @@ app.post( '--detach', '--wait', '--volumes', - '--timeout', - '0', + '--timeout=60', 'sharelatex', 'git-bridge', 'mongo', @@ -315,6 +314,16 @@ app.post('/reset/data', (req, res) => { ) }) +app.get('/redis/keys', (req, res) => { + runDockerCompose( + 'exec', + ['redis', 'redis-cli', 'KEYS', '*'], + (error, stdout, stderr) => { + res.json({ error, stdout, stderr }) + } + ) +}) + app.use(handleValidationErrors()) purgeDataDir() diff --git a/services/real-time/app.js b/services/real-time/app.js index d1c043bdd8..2679e143f0 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -186,17 +186,17 @@ server.listen(port, host, function (error) { // Stop huge stack traces in logs from all the socket.io parsing steps. Error.stackTraceLimit = 10 -function shutdownCleanly(signal) { +function shutdownAfterAllClientsHaveDisconnected() { const connectedClients = io.sockets.clients().length if (connectedClients === 0) { - logger.info('no clients connected, exiting') + logger.info({}, 'no clients connected, exiting') process.exit() } else { logger.info( { connectedClients }, 'clients still connected, not shutting down yet' ) - setTimeout(() => shutdownCleanly(signal), 30 * 1000) + setTimeout(() => shutdownAfterAllClientsHaveDisconnected(), 5_000) } } @@ -218,6 +218,7 @@ function drainAndShutdown(signal) { `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins` ) DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow, () => { + shutdownAfterAllClientsHaveDisconnected() setTimeout(() => { const staleClients = io.sockets.clients() if (staleClients.length !== 0) { @@ -233,7 +234,6 @@ function drainAndShutdown(signal) { Settings.shutDownComplete = true }, Settings.gracefulReconnectTimeoutMs) }) - shutdownCleanly(signal) }, statusCheckInterval) } }