github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-20 07:23:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #24269 from overleaf/bg-backup-replication-smoothing

Browse source 
extend backup scheduler to delay jobs for load levelling

GitOrigin-RevId: be62c458d3be464bf80ca590fcc107c2760aa061
This commit is contained in:
Brian Gough 2025-03-12 14:14:37 +00:00 • committed by Copybot
parent a178c0f400
commit 2d48c86e61
1 changed files with 26 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
services/history-v1/storage/scripts/backup_scheduler.mjs
View file

@ -46,6 +46,12 @@ const optionDefinitions = [
type: Number,
description: 'Limit the number of jobs to be added',
},
{
name: 'interval',
type: Number,
description: 'Time in seconds to spread jobs over (default: 300)',
defaultValue: 300,
},
{
name: 'backoff-delay',
type: Number,
@ -190,6 +196,7 @@ async function processPendingProjects(
showOnly,
limit,
verbose,
jobInterval,
jobOpts = {}
) {
const timeIntervalMs = age * 1000
@ -212,10 +219,11 @@ async function processPendingProjects(
`Project: ${projectId} (pending since: ${formatPendingTime(pendingAt)})`
)
} else if (!showOnly) {
const delay = Math.floor(Math.random() * jobInterval * 1000) // add random delay to avoid all jobs running simultaneously
const { job, added } = await addJobWithCheck(
backupQueue,
{ projectId, pendingChangeAt: pendingAt.getTime() },
{ ...jobOpts, jobId: projectId }
{ ...jobOpts, delay, jobId: projectId }
)
if (added) {
@ -304,13 +312,20 @@ async function run() {
setupMonitoring()
} else if (options['queue-pending'] !== undefined) {
const age = validatePendingTime('queue-pending', options['queue-pending'])
await processPendingProjects(age, false, options.limit, options.verbose, {
attempts: options.attempts,
backoff: {
type: 'exponential',
delay: options['backoff-delay'],
},
})
await processPendingProjects(
age,
false,
options.limit,
options.verbose,
options.interval,
{
attempts: options.attempts,
backoff: {
type: 'exponential',
delay: options['backoff-delay'],
},
}
)
} else if (options['show-pending'] !== undefined) {
const age = validatePendingTime('show-pending', options['show-pending'])
await processPendingProjects(age, true, options.limit, options.verbose)
@ -330,6 +345,9 @@ async function run() {
' --show-pending TIME Show count of pending projects older than TIME seconds'
)
console.log(' --limit N Limit the number of jobs to be added')
console.log(
' --interval TIME Time interval in seconds to spread jobs over'
)
console.log(
' --backoff-delay TIME Backoff delay in milliseconds for failed jobs (default: 1000)'
)