mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #19098 from overleaf/jpa-graceful-shutdown-tests
[server-ce] add test for graceful shutdown GitOrigin-RevId: 5e72e0073169009d2e3ece5a79cbd62051f6ad5b
This commit is contained in:
parent
956aad7e43
commit
ce9b531892
6 changed files with 135 additions and 8 deletions
|
@ -12,7 +12,7 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
|
|||
sleep 5
|
||||
|
||||
# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
|
||||
cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/overleaf/web.log 2>&1
|
||||
cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
|
||||
|
||||
EXIT_CODE="$?"
|
||||
if [ $EXIT_CODE -ne 0 ]
|
||||
|
@ -21,9 +21,14 @@ then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
sleep 10 &
|
||||
GIVE_EDITOR_10_SECONDS_TO_RELOAD=$!
|
||||
|
||||
# wait for disconnection
|
||||
while ! sv stop real-time-overleaf; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
wait $GIVE_EDITOR_10_SECONDS_TO_RELOAD
|
||||
|
||||
exit 0
|
||||
|
|
103
server-ce/test/graceful-shutdown.spec.ts
Normal file
103
server-ce/test/graceful-shutdown.spec.ts
Normal file
|
@ -0,0 +1,103 @@
|
|||
import {
|
||||
ensureUserExists,
|
||||
login,
|
||||
resetCreatedUsersCache,
|
||||
} from './helpers/login'
|
||||
import { STARTUP_TIMEOUT, startWith } from './helpers/config'
|
||||
import {
|
||||
dockerCompose,
|
||||
getRedisKeys,
|
||||
resetData,
|
||||
} from './helpers/hostAdminClient'
|
||||
import { createProject } from './helpers/project'
|
||||
import { throttledRecompile } from './helpers/compile'
|
||||
|
||||
const USER = 'user@example.com'
|
||||
const PROJECT_NAME = 'Old Project'
|
||||
|
||||
function bringServerProBackUp() {
|
||||
cy.log('bring server pro back up')
|
||||
cy.then({ timeout: STARTUP_TIMEOUT }, async () => {
|
||||
await dockerCompose('up', '--detach', '--wait', 'sharelatex')
|
||||
})
|
||||
}
|
||||
|
||||
describe('GracefulShutdown', function () {
|
||||
before(async () => {
|
||||
resetCreatedUsersCache()
|
||||
await resetData()
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
})
|
||||
ensureUserExists({ email: USER })
|
||||
|
||||
let projectId: string
|
||||
it('should display banner and flush changes out of redis', () => {
|
||||
bringServerProBackUp()
|
||||
login(USER)
|
||||
|
||||
cy.visit('/project')
|
||||
createProject(PROJECT_NAME).then(id => {
|
||||
projectId = id
|
||||
})
|
||||
const recompile = throttledRecompile()
|
||||
|
||||
cy.log('add additional content')
|
||||
cy.findByText('\\maketitle').parent().click()
|
||||
cy.findByText('\\maketitle').parent().type(`\n\\section{{}New Section}`)
|
||||
recompile()
|
||||
|
||||
cy.log(
|
||||
'check flush from frontend to backend: should include new section in PDF'
|
||||
)
|
||||
cy.get('.pdf-viewer').should('contain.text', 'New Section')
|
||||
|
||||
cy.log('should have unflushed content in redis before shutdown')
|
||||
cy.then(async () => {
|
||||
const keys = await getRedisKeys()
|
||||
expect(keys).to.contain(`DocsIn:${projectId}`)
|
||||
expect(keys).to.contain(`ProjectHistory:Ops:{${projectId}}`)
|
||||
})
|
||||
|
||||
cy.log('trigger graceful shutdown')
|
||||
let pendingShutdown: Promise<any>
|
||||
cy.then(() => {
|
||||
pendingShutdown = dockerCompose('stop', '--timeout=60', 'sharelatex')
|
||||
})
|
||||
|
||||
cy.log('wait for banner')
|
||||
cy.findByText(/performing maintenance/)
|
||||
cy.log('wait for page reload')
|
||||
cy.findByText(/is currently down for maintenance/)
|
||||
|
||||
cy.log('wait for shutdown to complete')
|
||||
cy.then({ timeout: 60 * 1000 }, async () => {
|
||||
await pendingShutdown
|
||||
})
|
||||
|
||||
cy.log('should not have any unflushed content in redis after shutdown')
|
||||
cy.then(async () => {
|
||||
const keys = await getRedisKeys()
|
||||
expect(keys).to.not.contain(`DocsIn:${projectId}`)
|
||||
expect(keys).to.not.contain(`ProjectHistory:Ops:{${projectId}}`)
|
||||
})
|
||||
|
||||
bringServerProBackUp()
|
||||
|
||||
cy.then(() => {
|
||||
cy.visit(`/project/${projectId}?trick-cypress-into-page-reload=true`)
|
||||
})
|
||||
|
||||
cy.log('check loading doc from mongo')
|
||||
cy.findByText('New Section')
|
||||
|
||||
cy.log('check PDF')
|
||||
cy.get('.pdf-viewer').should('contain.text', 'New Section')
|
||||
|
||||
cy.log('check history')
|
||||
cy.findByText('History').click()
|
||||
cy.findByText(/\\section\{New Section}/)
|
||||
})
|
||||
})
|
|
@ -1,5 +1,8 @@
|
|||
import { reconfigure } from './hostAdminClient'
|
||||
|
||||
export const STARTUP_TIMEOUT =
|
||||
parseInt(Cypress.env('STARTUP_TIMEOUT'), 10) || 120_000
|
||||
|
||||
let lastConfig: string
|
||||
|
||||
export function startWith({
|
||||
|
@ -14,7 +17,7 @@ export function startWith({
|
|||
const cfg = JSON.stringify({ pro, version, vars, withDataDir })
|
||||
if (lastConfig === cfg) return
|
||||
|
||||
this.timeout(100 * 1000)
|
||||
this.timeout(STARTUP_TIMEOUT)
|
||||
await reconfigure({ pro, version, vars, withDataDir })
|
||||
lastConfig = cfg
|
||||
})
|
||||
|
|
|
@ -76,3 +76,10 @@ export async function runScript({
|
|||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function getRedisKeys() {
|
||||
const { stdout } = await fetchJSON(`${hostAdminUrl}/redis/keys`, {
|
||||
method: 'GET',
|
||||
})
|
||||
return stdout.split('\n')
|
||||
}
|
||||
|
|
|
@ -197,8 +197,7 @@ app.post(
|
|||
'--detach',
|
||||
'--wait',
|
||||
'--volumes',
|
||||
'--timeout',
|
||||
'0',
|
||||
'--timeout=60',
|
||||
'sharelatex',
|
||||
'git-bridge',
|
||||
'mongo',
|
||||
|
@ -315,6 +314,16 @@ app.post('/reset/data', (req, res) => {
|
|||
)
|
||||
})
|
||||
|
||||
app.get('/redis/keys', (req, res) => {
|
||||
runDockerCompose(
|
||||
'exec',
|
||||
['redis', 'redis-cli', 'KEYS', '*'],
|
||||
(error, stdout, stderr) => {
|
||||
res.json({ error, stdout, stderr })
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
app.use(handleValidationErrors())
|
||||
|
||||
purgeDataDir()
|
||||
|
|
|
@ -186,17 +186,17 @@ server.listen(port, host, function (error) {
|
|||
// Stop huge stack traces in logs from all the socket.io parsing steps.
|
||||
Error.stackTraceLimit = 10
|
||||
|
||||
function shutdownCleanly(signal) {
|
||||
function shutdownAfterAllClientsHaveDisconnected() {
|
||||
const connectedClients = io.sockets.clients().length
|
||||
if (connectedClients === 0) {
|
||||
logger.info('no clients connected, exiting')
|
||||
logger.info({}, 'no clients connected, exiting')
|
||||
process.exit()
|
||||
} else {
|
||||
logger.info(
|
||||
{ connectedClients },
|
||||
'clients still connected, not shutting down yet'
|
||||
)
|
||||
setTimeout(() => shutdownCleanly(signal), 30 * 1000)
|
||||
setTimeout(() => shutdownAfterAllClientsHaveDisconnected(), 5_000)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -218,6 +218,7 @@ function drainAndShutdown(signal) {
|
|||
`received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`
|
||||
)
|
||||
DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow, () => {
|
||||
shutdownAfterAllClientsHaveDisconnected()
|
||||
setTimeout(() => {
|
||||
const staleClients = io.sockets.clients()
|
||||
if (staleClients.length !== 0) {
|
||||
|
@ -233,7 +234,6 @@ function drainAndShutdown(signal) {
|
|||
Settings.shutDownComplete = true
|
||||
}, Settings.gracefulReconnectTimeoutMs)
|
||||
})
|
||||
shutdownCleanly(signal)
|
||||
}, statusCheckInterval)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue