Merge pull request #14687 from overleaf/bg-queue-affiliation-confirmation

add rate limited bull queue for institutional domain confirmation

GitOrigin-RevId: 6dd38b586b7023e017d6480567a3f5faff74cbd5

services/web/app/src/Features/Institutions/InstitutionsController.js

@@ -1,9 +1,9 @@
-const { affiliateUsers } = require('./InstitutionsManager')
+const InstitutionsManager = require('./InstitutionsManager')
 
 module.exports = {
   confirmDomain(req, res, next) {
     const { hostname } = req.body
-    affiliateUsers(hostname, function (error) {
+    InstitutionsManager.confirmDomain(hostname, function (error) {
       if (error) {
         return next(error)
       }

services/web/app/src/Features/Institutions/InstitutionsManager.js

@@ -18,6 +18,7 @@ const NotificationsHandler = require('../Notifications/NotificationsHandler')
 const SubscriptionLocator = require('../Subscription/SubscriptionLocator')
 const { Institution } = require('../../models/Institution')
 const { Subscription } = require('../../models/Subscription')
+const Queues = require('../../infrastructure/Queues')
 const OError = require('@overleaf/o-error')
 
 const ASYNC_LIMIT = parseInt(process.env.ASYNC_LIMIT, 10) || 5
@@ -270,6 +271,8 @@ const InstitutionsManager = {
       )
     })
   },
+
+  confirmDomain: callbackify(confirmDomain),
 }
 
 const fetchInstitutionAndAffiliations = (institutionId, callback) =>
@@ -380,6 +383,14 @@ async function fetchV1Data(institution) {
   }
 }
 
+/**
+ * Enqueue a job for adding affiliations for when a domain is confirmed
+ */
+async function confirmDomain(hostname) {
+  const queue = Queues.getQueue('confirm-institution-domain')
+  await queue.add({ hostname })
+}
+
 function affiliateUserByReversedHostname(user, reversedHostname, callback) {
   const matchingEmails = user.emails.filter(
     email => email.reversedHostname === reversedHostname
@@ -422,6 +433,7 @@ function affiliateUserByReversedHostname(user, reversedHostname, callback) {
 
 InstitutionsManager.promises = {
   affiliateUsers: promisify(InstitutionsManager.affiliateUsers),
+  confirmDomain,
   checkInstitutionUsers,
   clearInstitutionNotifications: promisify(
     InstitutionsManager.clearInstitutionNotifications

services/web/app/src/infrastructure/QueueWorkers.js

@@ -3,6 +3,7 @@ const Queues = require('./Queues')
 const UserOnboardingEmailManager = require('../Features/User/UserOnboardingEmailManager')
 const UserPostRegistrationAnalyticsManager = require('../Features/User/UserPostRegistrationAnalyticsManager')
 const FeaturesUpdater = require('../Features/Subscription/FeaturesUpdater')
+const InstitutionsManager = require('../Features/Institutions/InstitutionsManager')
 const {
   addOptionalCleanupHandlerBeforeStoppingTraffic,
   addRequiredCleanupHandlerBeforeDrainingConnections,
@@ -63,6 +64,21 @@ function start() {
     }
   })
   registerCleanup(deferredEmailsQueue)
+
+  const confirmInstitutionDomainQueue = Queues.getQueue(
+    'confirm-institution-domain'
+  )
+  confirmInstitutionDomainQueue.process(async job => {
+    const { hostname } = job.data
+    try {
+      await InstitutionsManager.promises.affiliateUsers(hostname)
+    } catch (e) {
+      const error = OError.tag(e, 'failed to confirm university domain')
+      logger.warn(error)
+      throw error
+    }
+  })
+  registerCleanup(confirmInstitutionDomainQueue)
 }
 
 function registerCleanup(queue) {

services/web/app/src/infrastructure/Queues.js

@@ -33,6 +33,19 @@ const QUEUES_JOB_OPTIONS = {
     removeOnFail: MAX_FAILED_JOBS_RETAINED,
     attempts: 1,
   },
+  'confirm-institution-domain': {
+    removeOnFail: MAX_FAILED_JOBS_RETAINED,
+    attempts: 3,
+  },
+}
+
+const QUEUE_OPTIONS = {
+  'confirm-institution-domain': {
+    limiter: {
+      max: 1,
+      duration: 60 * 1000,
+    },
+  },
 }
 
 const ANALYTICS_QUEUES = [
@@ -49,11 +62,13 @@ function getQueue(queueName) {
     const redisOptions = ANALYTICS_QUEUES.includes(queueName)
       ? Settings.redis.analyticsQueues
       : Settings.redis.queues
+    const queueOptions = QUEUE_OPTIONS[queueName] || {}
     const jobOptions = QUEUES_JOB_OPTIONS[queueName] || {}
     queues[queueName] = new Queue(queueName, {
       // this configuration is duplicated in /services/analytics/app/js/Queues.js
       // and needs to be manually kept in sync whenever modified
       redis: redisOptions,
+      ...queueOptions,
       defaultJobOptions: {
         removeOnComplete: MAX_COMPLETED_JOBS_RETAINED,
         attempts: 11,

services/web/app/src/router.js

@@ -99,10 +99,6 @@ const rateLimiters = {
     points: 10,
     duration: 60,
   }),
-  confirmUniversityDomain: new RateLimiter('confirm-university-domain', {
-    points: 1,
-    duration: 60,
-  }),
   createProject: new RateLimiter('create-project', {
     points: 20,
     duration: 60,
@@ -1141,7 +1137,6 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) {
   )
   publicApiRouter.post(
     '/api/institutions/confirm_university_domain',
-    RateLimiterMiddleware.rateLimit(rateLimiters.confirmUniversityDomain),
     AuthenticationController.requirePrivateApiAuth(),
     InstitutionsController.confirmDomain
   )