From 10ef61a02b6e0772ec72f17560a8dccf2bdd1b7f Mon Sep 17 00:00:00 2001 From: Alexandre Bourdin Date: Tue, 13 Aug 2024 17:01:15 +0200 Subject: [PATCH] Merge pull request #19901 from overleaf/ac-confirm-institution-domain-queue-in-institution-module [web] Add `start` function to modules. Move `confirm-institution-domain` queue to institution module GitOrigin-RevId: 0d36ff3dcc529e77c76d72d0a67bbdb9310b42fd --- services/web/app.js | 9 +-- .../web/app/src/infrastructure/Modules.js | 7 +++ .../app/src/infrastructure/QueueWorkers.js | 62 ++++++++----------- services/web/docker-compose.common.env | 1 + .../test/acceptance/src/helpers/InitApp.js | 4 +- services/web/types/web-module.ts | 1 + 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/services/web/app.js b/services/web/app.js index 4c82e69c9e..5aef45a96f 100644 --- a/services/web/app.js +++ b/services/web/app.js @@ -6,6 +6,8 @@ const Settings = require('@overleaf/settings') const logger = require('@overleaf/logger') const PlansLocator = require('./app/src/Features/Subscription/PlansLocator') const SiteAdminHandler = require('./app/src/infrastructure/SiteAdminHandler') +const Modules = require('./app/src/infrastructure/Modules') + logger.initialize(process.env.METRICS_APP_NAME || 'web') logger.logger.serializers.user = require('./app/src/infrastructure/LoggerSerializers').user @@ -60,7 +62,7 @@ if (!module.parent) { PlansLocator.ensurePlansAreSetupCorrectly() Promise.all([mongodb.waitForDb(), mongoose.connectionPromise]) - .then(() => { + .then(async () => { Server.server.listen(port, host, function () { logger.debug(`web starting up, listening on ${host}:${port}`) logger.debug( @@ -69,9 +71,8 @@ if (!module.parent) { // wait until the process is ready before monitoring the event loop metrics.event_loop.monitor(logger) }) - if (process.env.QUEUE_PROCESSING_ENABLED === 'true') { - QueueWorkers.start() - } + QueueWorkers.start() + await Modules.start() }) .catch(err => { logger.fatal({ err }, 'Cannot connect to mongo. Exiting.') diff --git a/services/web/app/src/infrastructure/Modules.js b/services/web/app/src/infrastructure/Modules.js index b5daf62b06..5dc4e5b217 100644 --- a/services/web/app/src/infrastructure/Modules.js +++ b/services/web/app/src/infrastructure/Modules.js @@ -79,6 +79,12 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) { } } +async function start() { + for (const module of modules()) { + await module.start?.() + } +} + function loadViewIncludes(app) { _viewIncludes = Views.compileViewIncludes(app) } @@ -183,6 +189,7 @@ module.exports = { moduleIncludes, moduleIncludesAvailable, applyMiddleware, + start, hooks: { attach: attachHook, fire: fireHook, diff --git a/services/web/app/src/infrastructure/QueueWorkers.js b/services/web/app/src/infrastructure/QueueWorkers.js index 0c9ed4f1cb..caff895bf5 100644 --- a/services/web/app/src/infrastructure/QueueWorkers.js +++ b/services/web/app/src/infrastructure/QueueWorkers.js @@ -3,7 +3,6 @@ const Queues = require('./Queues') const UserOnboardingEmailManager = require('../Features/User/UserOnboardingEmailManager') const UserPostRegistrationAnalyticsManager = require('../Features/User/UserPostRegistrationAnalyticsManager') const FeaturesUpdater = require('../Features/Subscription/FeaturesUpdater') -const InstitutionsManager = require('../Features/Institutions/InstitutionsManager') const { addOptionalCleanupHandlerBeforeStoppingTraffic, addRequiredCleanupHandlerBeforeDrainingConnections, @@ -13,13 +12,30 @@ const logger = require('@overleaf/logger') const OError = require('@overleaf/o-error') const Modules = require('./Modules') +/** + * @typedef {{ + * data: {queueName: string,name?: string,data?: any}, + * }} BullJob + */ + +/** + * @param {string} queueName + * @param {(job: BullJob) => Promise} handler + */ +function registerQueue(queueName, handler) { + if (process.env.QUEUE_PROCESSING_ENABLED === 'true') { + const queue = Queues.getQueue(queueName) + queue.process(handler) + registerCleanup(queue) + } +} + function start() { if (!Features.hasFeature('saas')) { return } - const scheduledJobsQueue = Queues.getQueue('scheduled-jobs') - scheduledJobsQueue.process(async job => { + registerQueue('scheduled-jobs', async job => { const { queueName, name, data, options } = job.data const queue = Queues.getQueue(queueName) if (name) { @@ -28,33 +44,23 @@ function start() { await queue.add(data || {}, options || {}) } }) - registerCleanup(scheduledJobsQueue) - const onboardingEmailsQueue = Queues.getQueue('emails-onboarding') - onboardingEmailsQueue.process(async job => { + registerQueue('emails-onboarding', async job => { const { userId } = job.data await UserOnboardingEmailManager.sendOnboardingEmail(userId) }) - registerCleanup(onboardingEmailsQueue) - const postRegistrationAnalyticsQueue = Queues.getQueue( - 'post-registration-analytics' - ) - postRegistrationAnalyticsQueue.process(async job => { + registerQueue('post-registration-analytics', async job => { const { userId } = job.data await UserPostRegistrationAnalyticsManager.postRegistrationAnalytics(userId) }) - registerCleanup(postRegistrationAnalyticsQueue) - const refreshFeaturesQueue = Queues.getQueue('refresh-features') - refreshFeaturesQueue.process(async job => { + registerQueue('refresh-features', async job => { const { userId, reason } = job.data await FeaturesUpdater.promises.refreshFeatures(userId, reason) }) - registerCleanup(refreshFeaturesQueue) - const deferredEmailsQueue = Queues.getQueue('deferred-emails') - deferredEmailsQueue.process(async job => { + registerQueue('deferred-emails', async job => { const { emailType, opts } = job.data try { await EmailHandler.promises.sendEmail(emailType, opts) @@ -64,25 +70,8 @@ function start() { throw error } }) - registerCleanup(deferredEmailsQueue) - const confirmInstitutionDomainQueue = Queues.getQueue( - 'confirm-institution-domain' - ) - confirmInstitutionDomainQueue.process(async job => { - const { hostname } = job.data - try { - await InstitutionsManager.promises.affiliateUsers(hostname) - } catch (e) { - const error = OError.tag(e, 'failed to confirm university domain') - logger.warn(error) - throw error - } - }) - registerCleanup(confirmInstitutionDomainQueue) - - const groupSSOReminderQueue = Queues.getQueue('group-sso-reminder') - groupSSOReminderQueue.process(async job => { + registerQueue('group-sso-reminder', async job => { const { userId, subscriptionId } = job.data try { await Modules.promises.hooks.fire( @@ -99,7 +88,6 @@ function start() { throw error } }) - registerCleanup(groupSSOReminderQueue) } function registerCleanup(queue) { @@ -119,4 +107,4 @@ function registerCleanup(queue) { // Disconnect from redis is scheduled in queue setup. } -module.exports = { start } +module.exports = { start, registerQueue } diff --git a/services/web/docker-compose.common.env b/services/web/docker-compose.common.env index 666857d0df..21c242efb5 100644 --- a/services/web/docker-compose.common.env +++ b/services/web/docker-compose.common.env @@ -43,3 +43,4 @@ OVERLEAF_SAML_CERT=MIIDXTCCAkWgAwIBAgIJAOvOeQ4xFTzsMA0GCSqGSIb3DQEBCwUAMEUxCzAJB # NOTE: crypto.generateKeySync was added in v15, v16 is the next LTS release. # $ docker run --rm node:18.20.2 --print 'require("crypto").generateKeySync("aes", { length: 256 }).export().toString("hex")' DEVICE_HISTORY_SECRET=1b46e6cdf72db02845da06c9517c9cfbbfa0d87357479f4e1df3ce160bd54807 +QUEUE_PROCESSING_ENABLED=true diff --git a/services/web/test/acceptance/src/helpers/InitApp.js b/services/web/test/acceptance/src/helpers/InitApp.js index 4feb103f5a..1f5314d525 100644 --- a/services/web/test/acceptance/src/helpers/InitApp.js +++ b/services/web/test/acceptance/src/helpers/InitApp.js @@ -12,6 +12,7 @@ const { app } = require('../../../../app/src/infrastructure/Server') const { injectRouteAfter } = require('./injectRoute') const SplitTestHandler = require('../../../../app/src/Features/SplitTests/SplitTestHandler') const SplitTestSessionHandler = require('../../../../app/src/Features/SplitTests/SplitTestSessionHandler') +const Modules = require('../../../../app/src/infrastructure/Modules') logger.logger.level('error') @@ -104,8 +105,9 @@ before('start main app', function (done) { server = App.listen(23000, '127.0.0.1', done) }) -before('start queue workers', function () { +before('start queue workers', async function () { QueueWorkers.start() + await Modules.start() }) after('stop main app', async function () { diff --git a/services/web/types/web-module.ts b/services/web/types/web-module.ts index 11b1ca080f..406d03d85b 100644 --- a/services/web/types/web-module.ts +++ b/services/web/types/web-module.ts @@ -52,6 +52,7 @@ export type WebModule = { [name: string]: (req: any, res: any, next: any) => void } sessionMiddleware?: (webRouter: any, options: any) => void + start?: () => Promise appMiddleware?: (app: any) => void linkedFileAgents?: { [name: string]: () => LinkedFileAgent