Merge pull request #19901 from overleaf/ac-confirm-institution-domain-queue-in-institution-module

[web] Add `start` function to modules. Move `confirm-institution-domain` queue to institution module

GitOrigin-RevId: 0d36ff3dcc529e77c76d72d0a67bbdb9310b42fd

services/web/app.js

@@ -6,6 +6,8 @@ const Settings = require('@overleaf/settings')
 const logger = require('@overleaf/logger')
 const PlansLocator = require('./app/src/Features/Subscription/PlansLocator')
 const SiteAdminHandler = require('./app/src/infrastructure/SiteAdminHandler')
+const Modules = require('./app/src/infrastructure/Modules')
+
 logger.initialize(process.env.METRICS_APP_NAME || 'web')
 logger.logger.serializers.user =
   require('./app/src/infrastructure/LoggerSerializers').user
@@ -60,7 +62,7 @@ if (!module.parent) {
   PlansLocator.ensurePlansAreSetupCorrectly()
 
   Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
-    .then(() => {
+    .then(async () => {
       Server.server.listen(port, host, function () {
         logger.debug(`web starting up, listening on ${host}:${port}`)
         logger.debug(
@@ -69,9 +71,8 @@ if (!module.parent) {
         // wait until the process is ready before monitoring the event loop
         metrics.event_loop.monitor(logger)
       })
-      if (process.env.QUEUE_PROCESSING_ENABLED === 'true') {
       QueueWorkers.start()
-      }
+      await Modules.start()
     })
     .catch(err => {
       logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')

services/web/app/src/infrastructure/Modules.js

@@ -79,6 +79,12 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
   }
 }
 
+async function start() {
+  for (const module of modules()) {
+    await module.start?.()
+  }
+}
+
 function loadViewIncludes(app) {
   _viewIncludes = Views.compileViewIncludes(app)
 }
@@ -183,6 +189,7 @@ module.exports = {
   moduleIncludes,
   moduleIncludesAvailable,
   applyMiddleware,
+  start,
   hooks: {
     attach: attachHook,
     fire: fireHook,

services/web/app/src/infrastructure/QueueWorkers.js

@@ -3,7 +3,6 @@ const Queues = require('./Queues')
 const UserOnboardingEmailManager = require('../Features/User/UserOnboardingEmailManager')
 const UserPostRegistrationAnalyticsManager = require('../Features/User/UserPostRegistrationAnalyticsManager')
 const FeaturesUpdater = require('../Features/Subscription/FeaturesUpdater')
-const InstitutionsManager = require('../Features/Institutions/InstitutionsManager')
 const {
   addOptionalCleanupHandlerBeforeStoppingTraffic,
   addRequiredCleanupHandlerBeforeDrainingConnections,
@@ -13,13 +12,30 @@ const logger = require('@overleaf/logger')
 const OError = require('@overleaf/o-error')
 const Modules = require('./Modules')
 
+/**
+ * @typedef {{
+ *   data: {queueName: string,name?: string,data?: any},
+ * }} BullJob
+ */
+
+/**
+ * @param {string} queueName
+ * @param {(job: BullJob) => Promise<void>} handler
+ */
+function registerQueue(queueName, handler) {
+  if (process.env.QUEUE_PROCESSING_ENABLED === 'true') {
+    const queue = Queues.getQueue(queueName)
+    queue.process(handler)
+    registerCleanup(queue)
+  }
+}
+
 function start() {
   if (!Features.hasFeature('saas')) {
     return
   }
 
-  const scheduledJobsQueue = Queues.getQueue('scheduled-jobs')
+  registerQueue('scheduled-jobs', async job => {
-  scheduledJobsQueue.process(async job => {
     const { queueName, name, data, options } = job.data
     const queue = Queues.getQueue(queueName)
     if (name) {
@@ -28,33 +44,23 @@ function start() {
       await queue.add(data || {}, options || {})
     }
   })
-  registerCleanup(scheduledJobsQueue)
 
-  const onboardingEmailsQueue = Queues.getQueue('emails-onboarding')
+  registerQueue('emails-onboarding', async job => {
-  onboardingEmailsQueue.process(async job => {
     const { userId } = job.data
     await UserOnboardingEmailManager.sendOnboardingEmail(userId)
   })
-  registerCleanup(onboardingEmailsQueue)
 
-  const postRegistrationAnalyticsQueue = Queues.getQueue(
+  registerQueue('post-registration-analytics', async job => {
-    'post-registration-analytics'
-  )
-  postRegistrationAnalyticsQueue.process(async job => {
     const { userId } = job.data
     await UserPostRegistrationAnalyticsManager.postRegistrationAnalytics(userId)
   })
-  registerCleanup(postRegistrationAnalyticsQueue)
 
-  const refreshFeaturesQueue = Queues.getQueue('refresh-features')
+  registerQueue('refresh-features', async job => {
-  refreshFeaturesQueue.process(async job => {
     const { userId, reason } = job.data
     await FeaturesUpdater.promises.refreshFeatures(userId, reason)
   })
-  registerCleanup(refreshFeaturesQueue)
 
-  const deferredEmailsQueue = Queues.getQueue('deferred-emails')
+  registerQueue('deferred-emails', async job => {
-  deferredEmailsQueue.process(async job => {
     const { emailType, opts } = job.data
     try {
       await EmailHandler.promises.sendEmail(emailType, opts)
@@ -64,25 +70,8 @@ function start() {
       throw error
     }
   })
-  registerCleanup(deferredEmailsQueue)
 
-  const confirmInstitutionDomainQueue = Queues.getQueue(
+  registerQueue('group-sso-reminder', async job => {
-    'confirm-institution-domain'
-  )
-  confirmInstitutionDomainQueue.process(async job => {
-    const { hostname } = job.data
-    try {
-      await InstitutionsManager.promises.affiliateUsers(hostname)
-    } catch (e) {
-      const error = OError.tag(e, 'failed to confirm university domain')
-      logger.warn(error)
-      throw error
-    }
-  })
-  registerCleanup(confirmInstitutionDomainQueue)
-
-  const groupSSOReminderQueue = Queues.getQueue('group-sso-reminder')
-  groupSSOReminderQueue.process(async job => {
     const { userId, subscriptionId } = job.data
     try {
       await Modules.promises.hooks.fire(
@@ -99,7 +88,6 @@ function start() {
       throw error
     }
   })
-  registerCleanup(groupSSOReminderQueue)
 }
 
 function registerCleanup(queue) {
@@ -119,4 +107,4 @@ function registerCleanup(queue) {
   // Disconnect from redis is scheduled in queue setup.
 }
 
-module.exports = { start }
+module.exports = { start, registerQueue }

services/web/docker-compose.common.env

@@ -43,3 +43,4 @@ OVERLEAF_SAML_CERT=MIIDXTCCAkWgAwIBAgIJAOvOeQ4xFTzsMA0GCSqGSIb3DQEBCwUAMEUxCzAJB
 # NOTE: crypto.generateKeySync was added in v15, v16 is the next LTS release.
 # $ docker run --rm node:18.20.2 --print 'require("crypto").generateKeySync("aes", { length: 256 }).export().toString("hex")'
 DEVICE_HISTORY_SECRET=1b46e6cdf72db02845da06c9517c9cfbbfa0d87357479f4e1df3ce160bd54807
+QUEUE_PROCESSING_ENABLED=true

services/web/test/acceptance/src/helpers/InitApp.js

@@ -12,6 +12,7 @@ const { app } = require('../../../../app/src/infrastructure/Server')
 const { injectRouteAfter } = require('./injectRoute')
 const SplitTestHandler = require('../../../../app/src/Features/SplitTests/SplitTestHandler')
 const SplitTestSessionHandler = require('../../../../app/src/Features/SplitTests/SplitTestSessionHandler')
+const Modules = require('../../../../app/src/infrastructure/Modules')
 
 logger.logger.level('error')
 
@@ -104,8 +105,9 @@ before('start main app', function (done) {
   server = App.listen(23000, '127.0.0.1', done)
 })
 
-before('start queue workers', function () {
+before('start queue workers', async function () {
   QueueWorkers.start()
+  await Modules.start()
 })
 
 after('stop main app', async function () {

services/web/types/web-module.ts

@@ -52,6 +52,7 @@ export type WebModule = {
     [name: string]: (req: any, res: any, next: any) => void
   }
   sessionMiddleware?: (webRouter: any, options: any) => void
+  start?: () => Promise<void>
   appMiddleware?: (app: any) => void
   linkedFileAgents?: {
     [name: string]: () => LinkedFileAgent