diff --git a/services/history-v1/storage/scripts/backup_worker.mjs b/services/history-v1/storage/scripts/backup_worker.mjs index a61931350d..9a28059dc2 100644 --- a/services/history-v1/storage/scripts/backup_worker.mjs +++ b/services/history-v1/storage/scripts/backup_worker.mjs @@ -19,7 +19,9 @@ configureBackup({ concurrency: 50, batchConcurrency: 3, useSecondary: true }) const backupQueue = new Queue('backup', { redis: redisOptions, settings: { - stalledInterval: 0, // don't check for stalled jobs + lockDuration: 15 * 60 * 1000, // 15 minutes + lockRenewTime: 60 * 1000, // 1 minute + maxStalledCount: 0, // mark stalled jobs as failed }, }) @@ -46,6 +48,22 @@ backupQueue.on('error', error => { logger.error({ error }, 'queue error') }) +backupQueue.on('stalled', job => { + logger.error({ job }, 'job has stalled') +}) + +backupQueue.on('lock-extension-failed', (job, err) => { + logger.error({ job, err }, 'lock extension failed') +}) + +backupQueue.on('paused', () => { + logger.info('queue paused') +}) + +backupQueue.on('resumed', () => { + logger.info('queue resumed') +}) + // Process jobs backupQueue.process(CONCURRENCY, async job => { const { projectId, startDate, endDate } = job.data