2020-11-04 08:35:37 -05:00
|
|
|
const Queue = require('bull')
|
2021-07-07 05:38:56 -04:00
|
|
|
const Settings = require('@overleaf/settings')
|
2020-11-04 08:35:37 -05:00
|
|
|
|
2021-01-28 08:40:18 -05:00
|
|
|
// Bull will keep a fixed number of the most recently completed jobs. This is
|
|
|
|
// useful to inspect recently completed jobs. The bull prometheus exporter also
|
|
|
|
// uses the completed job records to report on job duration.
|
|
|
|
const MAX_COMPLETED_JOBS_RETAINED = 10000
|
2021-02-17 05:05:04 -05:00
|
|
|
const MAX_FAILED_JOBS_RETAINED = 50000
|
2021-01-28 08:40:18 -05:00
|
|
|
|
2021-10-25 08:07:45 -04:00
|
|
|
const QUEUES_JOB_OPTIONS = {
|
|
|
|
'analytics-events': {},
|
|
|
|
'analytics-editing-sessions': {},
|
|
|
|
'analytics-user-properties': {},
|
|
|
|
'refresh-features': {
|
|
|
|
attempts: 3,
|
|
|
|
},
|
|
|
|
'emails-onboarding': {},
|
|
|
|
'post-registration-analytics': {},
|
|
|
|
'scheduled-jobs': {
|
|
|
|
attempts: 1,
|
|
|
|
},
|
2021-05-19 04:29:39 -04:00
|
|
|
}
|
|
|
|
|
2022-03-31 07:59:17 -04:00
|
|
|
const ANALYTICS_QUEUES = [
|
|
|
|
'analytics-events',
|
|
|
|
'analytics-editing-sessions',
|
|
|
|
'analytics-user-properties',
|
|
|
|
'post-registration-analytics',
|
|
|
|
]
|
|
|
|
|
2021-10-25 08:07:45 -04:00
|
|
|
const queues = {}
|
2021-06-10 04:13:06 -04:00
|
|
|
|
2021-10-25 08:07:45 -04:00
|
|
|
function getQueue(queueName) {
|
2021-03-31 05:24:39 -04:00
|
|
|
if (!queues[queueName]) {
|
2022-03-31 07:59:17 -04:00
|
|
|
const redisOptions = ANALYTICS_QUEUES.includes(queueName)
|
|
|
|
? Settings.redis.analyticsQueues
|
|
|
|
: Settings.redis.queues
|
2021-10-25 08:07:45 -04:00
|
|
|
const jobOptions = QUEUES_JOB_OPTIONS[queueName] || {}
|
2021-03-31 05:24:39 -04:00
|
|
|
queues[queueName] = new Queue(queueName, {
|
2021-10-12 10:00:43 -04:00
|
|
|
// this configuration is duplicated in /services/analytics/app/js/Queues.js
|
|
|
|
// and needs to be manually kept in sync whenever modified
|
2022-03-31 07:59:17 -04:00
|
|
|
redis: redisOptions,
|
2021-03-31 05:24:39 -04:00
|
|
|
defaultJobOptions: {
|
|
|
|
removeOnComplete: MAX_COMPLETED_JOBS_RETAINED,
|
|
|
|
removeOnFail: MAX_FAILED_JOBS_RETAINED,
|
|
|
|
attempts: 11,
|
|
|
|
backoff: {
|
|
|
|
type: 'exponential',
|
2021-04-27 03:52:58 -04:00
|
|
|
delay: 3000,
|
|
|
|
},
|
2021-09-20 08:20:33 -04:00
|
|
|
...jobOptions,
|
2021-04-27 03:52:58 -04:00
|
|
|
},
|
2021-03-31 05:24:39 -04:00
|
|
|
})
|
|
|
|
}
|
|
|
|
return queues[queueName]
|
2020-11-04 08:35:37 -05:00
|
|
|
}
|
|
|
|
|
2021-10-25 08:07:45 -04:00
|
|
|
async function createScheduledJob(queueName, { name, data, options }, delay) {
|
|
|
|
await getQueue('scheduled-jobs').add(
|
|
|
|
{ queueName, name, data, options },
|
|
|
|
{
|
|
|
|
delay,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2021-03-31 05:24:39 -04:00
|
|
|
module.exports = {
|
2021-10-25 08:07:45 -04:00
|
|
|
getQueue,
|
|
|
|
createScheduledJob,
|
2021-03-31 05:24:39 -04:00
|
|
|
}
|