Merge pull request #19098 from overleaf/jpa-graceful-shutdown-tests

[server-ce] add test for graceful shutdown

GitOrigin-RevId: 5e72e0073169009d2e3ece5a79cbd62051f6ad5b

server-ce/init_preshutdown_scripts/00_close_site

@@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
 sleep 5
 
 # giving a grace period of 5 seconds for users before disconnecting them and start shutting down
-cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/overleaf/web.log 2>&1
+cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
 
 EXIT_CODE="$?"
 if [ $EXIT_CODE -ne 0 ]
@@ -21,9 +21,14 @@ then
   exit 1
 fi
 
+sleep 10 &
+GIVE_EDITOR_10_SECONDS_TO_RELOAD=$!
+
 # wait for disconnection
 while ! sv stop real-time-overleaf; do
   sleep 1
 done
 
+wait $GIVE_EDITOR_10_SECONDS_TO_RELOAD
+
 exit 0

server-ce/test/graceful-shutdown.spec.ts

@@ -0,0 +1,103 @@
+import {
+  ensureUserExists,
+  login,
+  resetCreatedUsersCache,
+} from './helpers/login'
+import { STARTUP_TIMEOUT, startWith } from './helpers/config'
+import {
+  dockerCompose,
+  getRedisKeys,
+  resetData,
+} from './helpers/hostAdminClient'
+import { createProject } from './helpers/project'
+import { throttledRecompile } from './helpers/compile'
+
+const USER = 'user@example.com'
+const PROJECT_NAME = 'Old Project'
+
+function bringServerProBackUp() {
+  cy.log('bring server pro back up')
+  cy.then({ timeout: STARTUP_TIMEOUT }, async () => {
+    await dockerCompose('up', '--detach', '--wait', 'sharelatex')
+  })
+}
+
+describe('GracefulShutdown', function () {
+  before(async () => {
+    resetCreatedUsersCache()
+    await resetData()
+  })
+  startWith({
+    pro: true,
+    withDataDir: true,
+  })
+  ensureUserExists({ email: USER })
+
+  let projectId: string
+  it('should display banner and flush changes out of redis', () => {
+    bringServerProBackUp()
+    login(USER)
+
+    cy.visit('/project')
+    createProject(PROJECT_NAME).then(id => {
+      projectId = id
+    })
+    const recompile = throttledRecompile()
+
+    cy.log('add additional content')
+    cy.findByText('\\maketitle').parent().click()
+    cy.findByText('\\maketitle').parent().type(`\n\\section{{}New Section}`)
+    recompile()
+
+    cy.log(
+      'check flush from frontend to backend: should include new section in PDF'
+    )
+    cy.get('.pdf-viewer').should('contain.text', 'New Section')
+
+    cy.log('should have unflushed content in redis before shutdown')
+    cy.then(async () => {
+      const keys = await getRedisKeys()
+      expect(keys).to.contain(`DocsIn:${projectId}`)
+      expect(keys).to.contain(`ProjectHistory:Ops:{${projectId}}`)
+    })
+
+    cy.log('trigger graceful shutdown')
+    let pendingShutdown: Promise<any>
+    cy.then(() => {
+      pendingShutdown = dockerCompose('stop', '--timeout=60', 'sharelatex')
+    })
+
+    cy.log('wait for banner')
+    cy.findByText(/performing maintenance/)
+    cy.log('wait for page reload')
+    cy.findByText(/is currently down for maintenance/)
+
+    cy.log('wait for shutdown to complete')
+    cy.then({ timeout: 60 * 1000 }, async () => {
+      await pendingShutdown
+    })
+
+    cy.log('should not have any unflushed content in redis after shutdown')
+    cy.then(async () => {
+      const keys = await getRedisKeys()
+      expect(keys).to.not.contain(`DocsIn:${projectId}`)
+      expect(keys).to.not.contain(`ProjectHistory:Ops:{${projectId}}`)
+    })
+
+    bringServerProBackUp()
+
+    cy.then(() => {
+      cy.visit(`/project/${projectId}?trick-cypress-into-page-reload=true`)
+    })
+
+    cy.log('check loading doc from mongo')
+    cy.findByText('New Section')
+
+    cy.log('check PDF')
+    cy.get('.pdf-viewer').should('contain.text', 'New Section')
+
+    cy.log('check history')
+    cy.findByText('History').click()
+    cy.findByText(/\\section\{New Section}/)
+  })
+})

server-ce/test/helpers/config.ts

@@ -1,5 +1,8 @@
 import { reconfigure } from './hostAdminClient'
 
+export const STARTUP_TIMEOUT =
+  parseInt(Cypress.env('STARTUP_TIMEOUT'), 10) || 120_000
+
 let lastConfig: string
 
 export function startWith({
@@ -14,7 +17,7 @@ export function startWith({
     const cfg = JSON.stringify({ pro, version, vars, withDataDir })
     if (lastConfig === cfg) return
 
-    this.timeout(100 * 1000)
+    this.timeout(STARTUP_TIMEOUT)
     await reconfigure({ pro, version, vars, withDataDir })
     lastConfig = cfg
   })

server-ce/test/helpers/hostAdminClient.ts

@@ -76,3 +76,10 @@ export async function runScript({
     }),
   })
 }
+
+export async function getRedisKeys() {
+  const { stdout } = await fetchJSON(`${hostAdminUrl}/redis/keys`, {
+    method: 'GET',
+  })
+  return stdout.split('\n')
+}

server-ce/test/host-admin.js

@@ -197,8 +197,7 @@ app.post(
           '--detach',
           '--wait',
           '--volumes',
-          '--timeout',
+          '--timeout=60',
-          '0',
           'sharelatex',
           'git-bridge',
           'mongo',
@@ -315,6 +314,16 @@ app.post('/reset/data', (req, res) => {
   )
 })
 
+app.get('/redis/keys', (req, res) => {
+  runDockerCompose(
+    'exec',
+    ['redis', 'redis-cli', 'KEYS', '*'],
+    (error, stdout, stderr) => {
+      res.json({ error, stdout, stderr })
+    }
+  )
+})
+
 app.use(handleValidationErrors())
 
 purgeDataDir()

services/real-time/app.js

@@ -186,17 +186,17 @@ server.listen(port, host, function (error) {
 // Stop huge stack traces in logs from all the socket.io parsing steps.
 Error.stackTraceLimit = 10
 
-function shutdownCleanly(signal) {
+function shutdownAfterAllClientsHaveDisconnected() {
   const connectedClients = io.sockets.clients().length
   if (connectedClients === 0) {
-    logger.info('no clients connected, exiting')
+    logger.info({}, 'no clients connected, exiting')
     process.exit()
   } else {
     logger.info(
       { connectedClients },
       'clients still connected, not shutting down yet'
     )
-    setTimeout(() => shutdownCleanly(signal), 30 * 1000)
+    setTimeout(() => shutdownAfterAllClientsHaveDisconnected(), 5_000)
   }
 }
 
@@ -218,6 +218,7 @@ function drainAndShutdown(signal) {
         `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`
       )
       DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow, () => {
+        shutdownAfterAllClientsHaveDisconnected()
         setTimeout(() => {
           const staleClients = io.sockets.clients()
           if (staleClients.length !== 0) {
@@ -233,7 +234,6 @@ function drainAndShutdown(signal) {
           Settings.shutDownComplete = true
         }, Settings.gracefulReconnectTimeoutMs)
       })
-      shutdownCleanly(signal)
     }, statusCheckInterval)
   }
 }