Merge pull request #3302 from overleaf/em-analytics-queues

Send analytics events through a queue

GitOrigin-RevId: b9eb12e469faf16e32aba5fae665c5f85dfbc52c

services/web/app/src/Features/Analytics/AnalyticsController.js

@@ -1,5 +1,5 @@
+const metrics = require('@overleaf/metrics')
 const AnalyticsManager = require('./AnalyticsManager')
-const Errors = require('../Errors/Errors')
 const AuthenticationController = require('../Authentication/AuthenticationController')
 const InstitutionsAPI = require('../Institutions/InstitutionsAPI')
 const GeoIpLookup = require('../../infrastructure/GeoIpLookup')
@@ -8,7 +8,7 @@ const Features = require('../../infrastructure/Features')
 module.exports = {
   updateEditingSession(req, res, next) {
     if (!Features.hasFeature('analytics')) {
-      return res.sendStatus(204)
+      return res.sendStatus(202)
     }
     const userId = AuthenticationController.getLoggedInUserId(req)
     const { projectId } = req.params
@@ -16,30 +16,25 @@ module.exports = {
 
     if (userId) {
       GeoIpLookup.getDetails(req.ip, function(err, geoDetails) {
-        if (!err && geoDetails && geoDetails.country_code) {
+        if (err) {
+          metrics.inc('analytics_geo_ip_lookup_errors')
+        } else if (geoDetails && geoDetails.country_code) {
           countryCode = geoDetails.country_code
         }
-        AnalyticsManager.updateEditingSession(
-          userId,
-          projectId,
-          countryCode,
-          error => respondWith(error, res, next)
-        )
+        AnalyticsManager.updateEditingSession(userId, projectId, countryCode)
       })
-    } else {
-      res.sendStatus(204)
     }
+    res.sendStatus(202)
   },
 
   recordEvent(req, res, next) {
     if (!Features.hasFeature('analytics')) {
-      return res.sendStatus(204)
+      return res.sendStatus(202)
     }
     const userId =
       AuthenticationController.getLoggedInUserId(req) || req.sessionID
-    AnalyticsManager.recordEvent(userId, req.params.event, req.body, error =>
-      respondWith(error, res, next)
-    )
+    AnalyticsManager.recordEvent(userId, req.params.event, req.body)
+    res.sendStatus(202)
   },
 
   licences(req, res, next) {
@@ -72,14 +67,3 @@ module.exports = {
     )
   }
 }
-
-var respondWith = function(error, res, next) {
-  if (error instanceof Errors.ServiceNotConfiguredError) {
-    // ignore, no-op
-    res.sendStatus(204)
-  } else if (error) {
-    next(error)
-  } else {
-    res.sendStatus(204)
-  }
-}

services/web/app/src/Features/Analytics/AnalyticsManager.js

@@ -1,128 +1,72 @@
-const settings = require('settings-sharelatex')
-const FaultTolerantRequest = require('../../infrastructure/FaultTolerantRequest')
-const Errors = require('../Errors/Errors')
+const Settings = require('settings-sharelatex')
+const Metrics = require('../../infrastructure/Metrics')
+const Queues = require('../../infrastructure/Queues')
 
-// check that the request should be made: ignore smoke test user and ensure the
-// analytics service is configured
-const checkAnalyticsRequest = function(userId) {
-  if (
-    settings.smokeTest &&
-    settings.smokeTest.userId &&
-    settings.smokeTest.userId.toString() === userId.toString()
-  ) {
-    // ignore smoke test user
-    return { error: null, skip: true }
+function identifyUser(userId, oldUserId) {
+  if (isAnalyticsDisabled() || isSmokeTestUser(userId)) {
+    return
   }
-
-  if (!settings.apis.analytics) {
-    return {
-      error: new Errors.ServiceNotConfiguredError(
-        'Analytics service not configured'
-      ),
-      skip: true
-    }
-  }
-
-  return { error: null, skip: false }
+  Metrics.analyticsQueue.inc({ status: 'adding', event_type: 'identify' })
+  Queues.analytics.events
+    .add('identify', { userId, oldUserId })
+    .then(() => {
+      Metrics.analyticsQueue.inc({ status: 'added', event_type: 'identify' })
+    })
+    .catch(() => {
+      Metrics.analyticsQueue.inc({ status: 'error', event_type: 'identify' })
+    })
 }
 
-// prepare the request: set `fromv2` param and full URL
-const prepareAnalyticsRequest = function(options) {
-  if (settings.overleaf != null) {
-    options.qs = Object.assign({}, options.qs, { fromV2: 1 })
+function recordEvent(userId, event, segmentation) {
+  if (isAnalyticsDisabled() || isSmokeTestUser(userId)) {
+    return
   }
-
-  const urlPath = options.url
-  options.url = `${settings.apis.analytics.url}${urlPath}`
-
-  options.timeout = options.timeout || 30000
-
-  return options
+  Metrics.analyticsQueue.inc({ status: 'adding', event_type: 'event' })
+  Queues.analytics.events
+    .add('event', { userId, event, segmentation })
+    .then(() => {
+      Metrics.analyticsQueue.inc({ status: 'added', event_type: 'event' })
+    })
+    .catch(() => {
+      Metrics.analyticsQueue.inc({ status: 'error', event_type: 'event' })
+    })
 }
 
-// make the request to analytics after checking and preparing it.
-// request happens asynchronously in the background and will be retried on error
-const makeAnalyticsBackgroundRequest = function(userId, options, callback) {
-  let { error, skip } = checkAnalyticsRequest(userId)
-  if (error || skip) {
-    return callback(error)
+function updateEditingSession(userId, projectId, countryCode) {
+  if (isAnalyticsDisabled() || isSmokeTestUser(userId)) {
+    return
   }
-  prepareAnalyticsRequest(options)
+  Metrics.analyticsQueue.inc({
+    status: 'adding',
+    event_type: 'editing-session'
+  })
+  Queues.analytics.editingSessions
+    .add({ userId, projectId, countryCode })
+    .then(() => {
+      Metrics.analyticsQueue.inc({
+        status: 'added',
+        event_type: 'editing-session'
+      })
+    })
+    .catch(() => {
+      Metrics.analyticsQueue.inc({
+        status: 'error',
+        event_type: 'editing-session'
+      })
+    })
+}
 
-  // With the tweaked parameter values (BACKOFF_BASE=3000, BACKOFF_MULTIPLIER=3):
-  // - the 6th attempt (maxAttempts=6) will run after 5.5 to 11.5 minutes
-  // - the 9th attempt (maxAttempts=9) will run after 86 to 250 minutes
-  options.maxAttempts = options.maxAttempts || 9
-  options.backoffBase = options.backoffBase || 3000
-  options.backoffMultiplier = options.backoffMultiplier || 3
+function isSmokeTestUser(userId) {
+  const smokeTestUserId = Settings.smokeTest && Settings.smokeTest.userId
+  return smokeTestUserId != null && userId.toString() === smokeTestUserId
+}
 
-  FaultTolerantRequest.backgroundRequest(options, callback)
+function isAnalyticsDisabled() {
+  return !(Settings.analytics && Settings.analytics.enabled)
 }
 
 module.exports = {
-  identifyUser(userId, oldUserId, callback) {
-    if (!callback) {
-      // callback is optional
-      callback = () => {}
-    }
-
-    const opts = {
-      body: {
-        old_user_id: oldUserId
-      },
-      json: true,
-      method: 'POST',
-      url: `/user/${userId}/identify`
-    }
-    makeAnalyticsBackgroundRequest(userId, opts, callback)
-  },
-
-  recordEvent(userId, event, segmentation, callback) {
-    if (segmentation == null) {
-      // segmentation is optional
-      segmentation = {}
-    }
-    if (!callback) {
-      // callback is optional
-      callback = () => {}
-    }
-
-    const opts = {
-      body: {
-        event,
-        segmentation
-      },
-      json: true,
-      method: 'POST',
-      url: `/user/${userId}/event`
-    }
-
-    makeAnalyticsBackgroundRequest(userId, opts, callback)
-  },
-
-  updateEditingSession(userId, projectId, countryCode, callback) {
-    if (!callback) {
-      // callback is optional
-      callback = () => {}
-    }
-
-    const query = {
-      userId,
-      projectId
-    }
-
-    if (countryCode) {
-      query.countryCode = countryCode
-    }
-
-    const opts = {
-      method: 'PUT',
-      url: '/editingSession',
-      qs: query,
-      maxAttempts: 6 // dont retry for too long as session ping timestamp are
-      // recorded when the request is received on the analytics
-    }
-
-    makeAnalyticsBackgroundRequest(userId, opts, callback)
-  }
+  identifyUser,
+  recordEvent,
+  updateEditingSession
 }

services/web/app/src/Features/Collaborators/CollaboratorsInviteController.js

@@ -24,7 +24,7 @@ const Settings = require('settings-sharelatex')
 const EmailHelper = require('../Helpers/EmailHelper')
 const EditorRealTimeController = require('../Editor/EditorRealTimeController')
 const NotificationsBuilder = require('../Notifications/NotificationsBuilder')
-const AnalyticsManger = require('../Analytics/AnalyticsManager')
+const AnalyticsManager = require('../Analytics/AnalyticsManager')
 const AuthenticationController = require('../Authentication/AuthenticationController')
 const rateLimiter = require('../../infrastructure/RateLimiter')
 const request = require('request')
@@ -377,7 +377,7 @@ module.exports = CollaboratorsInviteController = {
           'project:membership:changed',
           { invites: true, members: true }
         )
-        AnalyticsManger.recordEvent(currentUser._id, 'project-invite-accept', {
+        AnalyticsManager.recordEvent(currentUser._id, 'project-invite-accept', {
           projectId,
           userId: currentUser._id
         })

services/web/app/src/Features/Project/ProjectCreationHandler.js

@@ -28,7 +28,7 @@ const fs = require('fs')
 const Path = require('path')
 const { promisify } = require('util')
 const _ = require('underscore')
-const AnalyticsManger = require('../Analytics/AnalyticsManager')
+const AnalyticsManager = require('../Analytics/AnalyticsManager')
 
 const ProjectCreationHandler = {
   createBlankProject(owner_id, projectName, attributes, callback) {
@@ -56,7 +56,7 @@ const ProjectCreationHandler = {
             if (error != null) {
               return callback(error)
             }
-            AnalyticsManger.recordEvent(owner_id, 'project-imported', {
+            AnalyticsManager.recordEvent(owner_id, 'project-imported', {
               projectId: project._id,
               attributes
             })
@@ -79,7 +79,7 @@ const ProjectCreationHandler = {
               if (error != null) {
                 return callback(error)
               }
-              AnalyticsManger.recordEvent(owner_id, 'project-created', {
+              AnalyticsManager.recordEvent(owner_id, 'project-created', {
                 projectId: project._id,
                 attributes
               })

services/web/app/src/infrastructure/ExpressLocals.js

@@ -245,7 +245,8 @@ module.exports = function(webRouter, privateApiRouter, publicApiRouter) {
   })
 
   webRouter.use(function(req, res, next) {
-    res.locals.gaToken = Settings.analytics && Settings.analytics.ga.token
+    res.locals.gaToken =
+      Settings.analytics && Settings.analytics.ga && Settings.analytics.ga.token
     res.locals.gaOptimizeId = _.get(Settings, ['analytics', 'gaOptimize', 'id'])
     next()
   })

services/web/app/src/infrastructure/FaultTolerantRequest.js

@@ -1,85 +0,0 @@
-const logger = require('logger-sharelatex')
-const request = require('requestretry')
-
-const isProduction =
-  (process.env['NODE_ENV'] || '').toLowerCase() === 'production'
-const isTest = process.env['MOCHA_GREP'] !== undefined
-
-const BACKOFF_MAX_TRIES = 3
-const BACKOFF_BASE = 500
-const BACKOFF_MULTIPLIER = 1.5
-const BACKOFF_RANDOM_FACTOR = 0.5
-
-//
-// Use an exponential backoff to retry requests
-//
-// This is based on what the Google HTTP client does:
-// https://developers.google.com/api-client-library/java/google-http-java-client/reference/1.20.0/com/google/api/client/util/ExponentialBackOff
-//
-let FaultTolerantRequest
-module.exports = FaultTolerantRequest = {
-  request: function(options, callback) {
-    options = Object.assign(
-      {
-        maxAttempts: BACKOFF_MAX_TRIES,
-        backoffBase: BACKOFF_BASE,
-        backoffMultiplier: BACKOFF_MULTIPLIER,
-        backoffRandomFactor: BACKOFF_RANDOM_FACTOR
-      },
-      options
-    )
-
-    options.delayStrategy = FaultTolerantRequest.exponentialDelayStrategy(
-      options.backoffBase,
-      options.backoffMultiplier,
-      options.backoffRandomFactor
-    )
-
-    request(options, callback)
-  },
-
-  backgroundRequest: function(options, callback) {
-    FaultTolerantRequest.request(options, function(err) {
-      if (err) {
-        return logger.err(
-          { err, url: options.url, query: options.qs, body: options.body },
-          'Background request failed'
-        )
-      }
-    })
-
-    callback() // Do not wait for all the attempts
-  },
-
-  exponentialDelayStrategy: function(
-    backoffBase,
-    backoffMultiplier,
-    backoffRandomFactor
-  ) {
-    let backoff = backoffBase
-
-    return function() {
-      const delay = exponentialDelay(backoff, backoffRandomFactor)
-      backoff *= backoffMultiplier
-      return delay
-    }
-  }
-}
-
-function exponentialDelay(backoff, backoffRandomFactor) {
-  // set delay to `backoff` initially
-  let delay = backoff
-
-  // adds randomness
-  delay *= 1 - backoffRandomFactor + 2 * Math.random() * backoffRandomFactor
-
-  // round value as it's already in milliseconds
-  delay = Math.round(delay)
-
-  // log retries in production
-  if (isProduction && !isTest) {
-    logger.warn(`Background request failed. Will try again in ${delay}ms`)
-  }
-
-  return delay
-}

services/web/app/src/infrastructure/Metrics.js

@@ -0,0 +1,7 @@
+const Metrics = require('@overleaf/metrics')
+
+exports.analyticsQueue = new Metrics.prom.Counter({
+  name: 'analytics_queue',
+  help: 'Number of events sent to the analytics queue',
+  labelNames: ['status', 'event_type']
+})

services/web/app/src/infrastructure/Queues.js

@@ -0,0 +1,23 @@
+const Queue = require('bull')
+const Settings = require('settings-sharelatex')
+
+function createQueue(queueName, defaultJobOptions) {
+  return new Queue(queueName, {
+    redis: Settings.redis.queues,
+    defaultJobOptions: {
+      removeOnComplete: true,
+      attempts: 11,
+      backoff: {
+        type: 'exponential',
+        delay: 3000
+      }
+    }
+  })
+}
+
+module.exports = {
+  analytics: {
+    events: createQueue('analytics-events'),
+    editingSessions: createQueue('analytics-editing-sessions')
+  }
+}

services/web/config/settings.defaults.coffee

@@ -93,6 +93,11 @@ module.exports = settings =
 			password: process.env["REDIS_PASSWORD"] or ""
 			maxRetriesPerRequest: parseInt(process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || '20')
 
+		queues:
+			host: process.env['QUEUES_REDIS_HOST'] || process.env['REDIS_HOST'] || 'localhost'
+			port: process.env['QUEUES_REDIS_PORT'] || process.env['REDIS_PORT'] || '6379'
+			password: process.env['QUEUES_REDIS_PASSWORD'] || process.env['REDIS_PASSWORD'] || ''
+
 	# Service locations
 	# -----------------
 
@@ -616,6 +621,9 @@ module.exports = settings =
 			everyone: process.env['RATE_LIMIT_AUTO_COMPILE_EVERYONE'] or 100
 			standard: process.env['RATE_LIMIT_AUTO_COMPILE_STANDARD'] or 25
 
+	analytics:
+		enabled: process.env['ANALYTICS_ENABLED'] == 'true'
+
 	# currentImage: "texlive-full:2017.1"
 	# imageRoot: "<DOCKER REPOSITORY ROOT>" # without any trailing slash
 

services/web/package-lock.json

@@ -7,7 +7,7 @@
     "@auth0/thumbprint": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/@auth0/thumbprint/-/thumbprint-0.0.6.tgz",
-      "integrity": "sha1-yrEGLGwEZizmxZLUgVfsQmiuhRg=",
+      "integrity": "sha512-+YciWHxNUOE78T+xoXI1fMI6G1WdyyAay8ioaMZhvGOJ+lReYzj0b7mpfNr5WtjGrmtWPvPOOxh0TO+5Y2M/Hw==",
       "dev": true
     },
     "@babel/cli": {
@@ -2889,9 +2889,9 @@
       },
       "dependencies": {
         "@types/node": {
-          "version": "13.13.29",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.29.tgz",
-          "integrity": "sha512-WPGpyEDx4/F4Rx1p1Zar8m+JsMxpSY/wNFPlyNXWV+UzJwkYt3LQg2be/qJgpsLdVJsfxTR5ipY6rv2579jStQ=="
+          "version": "13.13.30",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.30.tgz",
+          "integrity": "sha512-HmqFpNzp3TSELxU/bUuRK+xzarVOAsR00hzcvM0TXrMlt/+wcSLa5q6YhTb6/cA6wqDCZLDcfd8fSL95x5h7AA=="
         },
         "protobufjs": {
           "version": "6.10.1",
@@ -3817,9 +3817,9 @@
       }
     },
     "@overleaf/metrics": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.3.0.tgz",
-      "integrity": "sha512-y+t0ZUW3WC90PrDH6KTDig4S4wRtL/0Uay+DFG2iuxFTaxYvf0CA1ebZidj2QyRZm6Yf0FNOJnSRp55fcrAadA==",
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.4.0.tgz",
+      "integrity": "sha512-MsD+5d7gpoz2VyJ5AnEP1VHGyHsfya7bfkdnMgi93kS+FPcYvdpcooBXPq4jmGemEVxXhdyP9lBTJDwcdjZeiQ==",
       "requires": {
         "@google-cloud/debug-agent": "^5.1.2",
         "@google-cloud/profiler": "^4.0.3",
@@ -12154,6 +12154,175 @@
       "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==",
       "dev": true
     },
+    "bull": {
+      "version": "3.18.0",
+      "resolved": "https://registry.npmjs.org/bull/-/bull-3.18.0.tgz",
+      "integrity": "sha512-nE/BKlg1dnJ/AcOy5D1nzthcmpAKqpUVXzQ43mJfnVC8ZM7mi4ZzP3spN7745UuikzmGGsbTe9px2TbEKhR+DQ==",
+      "requires": {
+        "cron-parser": "^2.13.0",
+        "debuglog": "^1.0.0",
+        "get-port": "^5.1.1",
+        "ioredis": "^4.14.1",
+        "lodash": "^4.17.19",
+        "p-timeout": "^3.2.0",
+        "promise.prototype.finally": "^3.1.2",
+        "semver": "^7.3.2",
+        "util.promisify": "^1.0.1",
+        "uuid": "^8.3.0"
+      },
+      "dependencies": {
+        "define-properties": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
+          "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
+          "requires": {
+            "object-keys": "^1.0.12"
+          }
+        },
+        "es-abstract": {
+          "version": "1.17.7",
+          "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz",
+          "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==",
+          "requires": {
+            "es-to-primitive": "^1.2.1",
+            "function-bind": "^1.1.1",
+            "has": "^1.0.3",
+            "has-symbols": "^1.0.1",
+            "is-callable": "^1.2.2",
+            "is-regex": "^1.1.1",
+            "object-inspect": "^1.8.0",
+            "object-keys": "^1.1.1",
+            "object.assign": "^4.1.1",
+            "string.prototype.trimend": "^1.0.1",
+            "string.prototype.trimstart": "^1.0.1"
+          }
+        },
+        "es-to-primitive": {
+          "version": "1.2.1",
+          "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
+          "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
+          "requires": {
+            "is-callable": "^1.1.4",
+            "is-date-object": "^1.0.1",
+            "is-symbol": "^1.0.2"
+          }
+        },
+        "has": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
+          "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+          "requires": {
+            "function-bind": "^1.1.1"
+          }
+        },
+        "has-symbols": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
+          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg=="
+        },
+        "is-callable": {
+          "version": "1.2.2",
+          "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz",
+          "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA=="
+        },
+        "is-regex": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
+          "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==",
+          "requires": {
+            "has-symbols": "^1.0.1"
+          }
+        },
+        "is-symbol": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz",
+          "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==",
+          "requires": {
+            "has-symbols": "^1.0.1"
+          }
+        },
+        "object-inspect": {
+          "version": "1.8.0",
+          "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz",
+          "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA=="
+        },
+        "object-keys": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
+        },
+        "object.assign": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.1.tgz",
+          "integrity": "sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA==",
+          "requires": {
+            "define-properties": "^1.1.3",
+            "es-abstract": "^1.18.0-next.0",
+            "has-symbols": "^1.0.1",
+            "object-keys": "^1.1.1"
+          },
+          "dependencies": {
+            "es-abstract": {
+              "version": "1.18.0-next.1",
+              "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz",
+              "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==",
+              "requires": {
+                "es-to-primitive": "^1.2.1",
+                "function-bind": "^1.1.1",
+                "has": "^1.0.3",
+                "has-symbols": "^1.0.1",
+                "is-callable": "^1.2.2",
+                "is-negative-zero": "^2.0.0",
+                "is-regex": "^1.1.1",
+                "object-inspect": "^1.8.0",
+                "object-keys": "^1.1.1",
+                "object.assign": "^4.1.1",
+                "string.prototype.trimend": "^1.0.1",
+                "string.prototype.trimstart": "^1.0.1"
+              }
+            }
+          }
+        },
+        "object.getownpropertydescriptors": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz",
+          "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==",
+          "requires": {
+            "define-properties": "^1.1.3",
+            "es-abstract": "^1.17.0-next.1"
+          }
+        },
+        "p-timeout": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
+          "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
+          "requires": {
+            "p-finally": "^1.0.0"
+          }
+        },
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ=="
+        },
+        "util.promisify": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.1.tgz",
+          "integrity": "sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA==",
+          "requires": {
+            "define-properties": "^1.1.3",
+            "es-abstract": "^1.17.2",
+            "has-symbols": "^1.0.1",
+            "object.getownpropertydescriptors": "^2.1.0"
+          }
+        },
+        "uuid": {
+          "version": "8.3.1",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz",
+          "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg=="
+        }
+      }
+    },
     "bunyan": {
       "version": "1.8.14",
       "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz",
@@ -13897,6 +14066,15 @@
         }
       }
     },
+    "cron-parser": {
+      "version": "2.16.3",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.16.3.tgz",
+      "integrity": "sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==",
+      "requires": {
+        "is-nan": "^1.3.0",
+        "moment-timezone": "^0.5.31"
+      }
+    },
     "cross-spawn": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
@@ -14493,6 +14671,11 @@
         }
       }
     },
+    "debuglog": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
+      "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI="
+    },
     "decamelize": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
@@ -17375,7 +17558,7 @@
     "flowstate": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/flowstate/-/flowstate-0.4.1.tgz",
-      "integrity": "sha1-tfu4t/wte9xbVL5GyYMJ73NvTsA=",
+      "integrity": "sha512-U67AgveyMwXFIiDgs6Yz/PrUNrZGLJUUMDwJ9Q0fDFTQSzyDg8Jj9YDyZIUnFZKggQZONVueK9+grp/Gxa/scw==",
       "dev": true,
       "requires": {
         "clone": "^1.0.2",
@@ -18378,11 +18561,11 @@
       }
     },
     "gcp-metadata": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.0.tgz",
-      "integrity": "sha512-vQZD57cQkqIA6YPGXM/zc+PIZfNRFdukWGsGZ5+LcJzesi5xp6Gn7a02wRJi4eXPyArNMIYpPET4QMxGqtlk6Q==",
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz",
+      "integrity": "sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw==",
       "requires": {
-        "gaxios": "^3.0.0",
+        "gaxios": "^4.0.0",
         "json-bigint": "^1.0.0"
       },
       "dependencies": {
@@ -18391,23 +18574,6 @@
           "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz",
           "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA=="
         },
-        "gaxios": {
-          "version": "3.2.0",
-          "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.2.0.tgz",
-          "integrity": "sha512-+6WPeVzPvOshftpxJwRi2Ozez80tn/hdtOUag7+gajDHRJvAblKxTFSSMPtr2hmnLy7p0mvYz0rMXLBl8pSO7Q==",
-          "requires": {
-            "abort-controller": "^3.0.0",
-            "extend": "^3.0.2",
-            "https-proxy-agent": "^5.0.0",
-            "is-stream": "^2.0.0",
-            "node-fetch": "^2.3.0"
-          }
-        },
-        "is-stream": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
-          "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
-        },
         "json-bigint": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
@@ -18441,6 +18607,11 @@
       "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
       "dev": true
     },
+    "get-port": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
+      "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ=="
+    },
     "get-stream": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
@@ -20527,6 +20698,34 @@
       "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==",
       "dev": true
     },
+    "is-nan": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.0.tgz",
+      "integrity": "sha512-z7bbREymOqt2CCaZVly8aC4ML3Xhfi0ekuOnjO2L8vKdl+CttdVoGZQhd4adMFAsxQ5VeRVwORs4tU8RH+HFtQ==",
+      "requires": {
+        "define-properties": "^1.1.3"
+      },
+      "dependencies": {
+        "define-properties": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
+          "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
+          "requires": {
+            "object-keys": "^1.0.12"
+          }
+        },
+        "object-keys": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
+        }
+      }
+    },
+    "is-negative-zero": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz",
+      "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE="
+    },
     "is-npm": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz",
@@ -23660,7 +23859,7 @@
     },
     "mkdirp": {
       "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "resolved": "",
       "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==",
       "requires": {
         "minimist": "0.0.8"
@@ -23835,6 +24034,14 @@
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
       "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
     },
+    "moment-timezone": {
+      "version": "0.5.31",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz",
+      "integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==",
+      "requires": {
+        "moment": ">= 2.9.0"
+      }
+    },
     "mongodb": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.2.tgz",
@@ -25283,8 +25490,7 @@
     "object-inspect": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz",
-      "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==",
-      "dev": true
+      "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw=="
     },
     "object-is": {
       "version": "1.1.2",
@@ -27802,7 +28008,6 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz",
       "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==",
-      "dev": true,
       "requires": {
         "define-properties": "^1.1.3",
         "es-abstract": "^1.17.0-next.0",
@@ -27813,7 +28018,6 @@
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
           "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
-          "dev": true,
           "requires": {
             "object-keys": "^1.0.12"
           }
@@ -27822,7 +28026,6 @@
           "version": "1.17.6",
           "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
           "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==",
-          "dev": true,
           "requires": {
             "es-to-primitive": "^1.2.1",
             "function-bind": "^1.1.1",
@@ -27841,7 +28044,6 @@
           "version": "1.2.1",
           "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
           "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
-          "dev": true,
           "requires": {
             "is-callable": "^1.1.4",
             "is-date-object": "^1.0.1",
@@ -27852,7 +28054,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
           "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-          "dev": true,
           "requires": {
             "function-bind": "^1.1.1"
           }
@@ -27860,20 +28061,17 @@
         "has-symbols": {
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
-          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
-          "dev": true
+          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg=="
         },
         "is-callable": {
           "version": "1.2.1",
           "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.1.tgz",
-          "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg==",
-          "dev": true
+          "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg=="
         },
         "is-regex": {
           "version": "1.1.1",
           "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
           "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==",
-          "dev": true,
           "requires": {
             "has-symbols": "^1.0.1"
           }
@@ -27882,7 +28080,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz",
           "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==",
-          "dev": true,
           "requires": {
             "has-symbols": "^1.0.1"
           }
@@ -27890,8 +28087,7 @@
         "object-keys": {
           "version": "1.1.1",
           "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
-          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
-          "dev": true
+          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
         }
       }
     },
@@ -33366,7 +33562,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz",
       "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==",
-      "dev": true,
       "requires": {
         "define-properties": "^1.1.3",
         "es-abstract": "^1.17.5"
@@ -33376,7 +33571,6 @@
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
           "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
-          "dev": true,
           "requires": {
             "object-keys": "^1.0.12"
           }
@@ -33385,7 +33579,6 @@
           "version": "1.17.5",
           "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz",
           "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==",
-          "dev": true,
           "requires": {
             "es-to-primitive": "^1.2.1",
             "function-bind": "^1.1.1",
@@ -33404,7 +33597,6 @@
           "version": "1.2.1",
           "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
           "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
-          "dev": true,
           "requires": {
             "is-callable": "^1.1.4",
             "is-date-object": "^1.0.1",
@@ -33415,7 +33607,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
           "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-          "dev": true,
           "requires": {
             "function-bind": "^1.1.1"
           }
@@ -33423,20 +33614,17 @@
         "has-symbols": {
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
-          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
-          "dev": true
+          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg=="
         },
         "is-callable": {
           "version": "1.1.5",
           "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz",
-          "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==",
-          "dev": true
+          "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q=="
         },
         "is-regex": {
           "version": "1.0.5",
           "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz",
           "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==",
-          "dev": true,
           "requires": {
             "has": "^1.0.3"
           }
@@ -33445,7 +33633,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz",
           "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==",
-          "dev": true,
           "requires": {
             "has-symbols": "^1.0.1"
           }
@@ -33453,14 +33640,12 @@
         "object-keys": {
           "version": "1.1.1",
           "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
-          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
-          "dev": true
+          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
         },
         "string.prototype.trimleft": {
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz",
           "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==",
-          "dev": true,
           "requires": {
             "define-properties": "^1.1.3",
             "es-abstract": "^1.17.5",
@@ -33471,7 +33656,6 @@
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz",
           "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==",
-          "dev": true,
           "requires": {
             "define-properties": "^1.1.3",
             "es-abstract": "^1.17.5",
@@ -33538,7 +33722,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz",
       "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==",
-      "dev": true,
       "requires": {
         "define-properties": "^1.1.3",
         "es-abstract": "^1.17.5"
@@ -33548,7 +33731,6 @@
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
           "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
-          "dev": true,
           "requires": {
             "object-keys": "^1.0.12"
           }
@@ -33557,7 +33739,6 @@
           "version": "1.17.5",
           "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz",
           "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==",
-          "dev": true,
           "requires": {
             "es-to-primitive": "^1.2.1",
             "function-bind": "^1.1.1",
@@ -33576,7 +33757,6 @@
           "version": "1.2.1",
           "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
           "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
-          "dev": true,
           "requires": {
             "is-callable": "^1.1.4",
             "is-date-object": "^1.0.1",
@@ -33587,7 +33767,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
           "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-          "dev": true,
           "requires": {
             "function-bind": "^1.1.1"
           }
@@ -33595,20 +33774,17 @@
         "has-symbols": {
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
-          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
-          "dev": true
+          "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg=="
         },
         "is-callable": {
           "version": "1.1.5",
           "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz",
-          "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==",
-          "dev": true
+          "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q=="
         },
         "is-regex": {
           "version": "1.0.5",
           "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz",
           "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==",
-          "dev": true,
           "requires": {
             "has": "^1.0.3"
           }
@@ -33617,7 +33793,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz",
           "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==",
-          "dev": true,
           "requires": {
             "has-symbols": "^1.0.1"
           }
@@ -33625,14 +33800,12 @@
         "object-keys": {
           "version": "1.1.1",
           "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
-          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
-          "dev": true
+          "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
         },
         "string.prototype.trimleft": {
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz",
           "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==",
-          "dev": true,
           "requires": {
             "define-properties": "^1.1.3",
             "es-abstract": "^1.17.5",
@@ -33643,7 +33816,6 @@
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz",
           "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==",
-          "dev": true,
           "requires": {
             "define-properties": "^1.1.3",
             "es-abstract": "^1.17.5",
@@ -35690,7 +35862,7 @@
     "utils-flatten": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/utils-flatten/-/utils-flatten-1.0.0.tgz",
-      "integrity": "sha1-AfMNMZO+RkxAsxdV5nQNDbDO8kM=",
+      "integrity": "sha512-s21PUgUZ+XPvH8Wi8aj2FEqzZWeNEdemP7LB4u8u5wTDRO4xB+7czAYd3FY2O2rnu89U//khR0Ce8ka3//6M0w==",
       "dev": true
     },
     "utils-merge": {
@@ -37651,7 +37823,7 @@
     "xml-name-validator": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-2.0.1.tgz",
-      "integrity": "sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=",
+      "integrity": "sha512-jRKe/iQYMyVJpzPH+3HL97Lgu5HrCfii+qSo+TfjKHtOnvbnvdVfMYrn9Q34YV81M2e5sviJlI6Ko9y+nByzvA==",
       "dev": true
     },
     "xml2js": {

services/web/package.json

@@ -41,7 +41,7 @@
     "@babel/core": "^7.9.0",
     "@babel/preset-env": "^7.9.5",
     "@babel/preset-react": "^7.9.4",
-    "@overleaf/metrics": "^3.3.0",
+    "@overleaf/metrics": "^3.4.0",
     "@overleaf/o-error": "^3.0.0",
     "@pollyjs/adapter-node-http": "^4.2.1",
     "@pollyjs/core": "^4.2.1",
@@ -58,6 +58,7 @@
     "bcrypt": "^5.0.0",
     "body-parser": "^1.19.0",
     "bufferedstream": "1.6.0",
+    "bull": "^3.18.0",
     "celebrate": "^10.0.1",
     "classnames": "^2.2.6",
     "codemirror": "^5.33.0",

services/web/test/unit/src/Analytics/AnalyticsControllerTests.js

@@ -11,8 +11,8 @@ describe('AnalyticsController', function() {
     this.AuthenticationController = { getLoggedInUserId: sinon.stub() }
 
     this.AnalyticsManager = {
-      updateEditingSession: sinon.stub().callsArgWith(3),
-      recordEvent: sinon.stub().callsArgWith(3)
+      updateEditingSession: sinon.stub(),
+      recordEvent: sinon.stub()
     }
 
     this.InstitutionsAPI = {

services/web/test/unit/src/Analytics/AnalyticsManagerTests.js

@@ -1,106 +1,96 @@
 const SandboxedModule = require('sandboxed-module')
 const path = require('path')
-const modulePath = path.join(
+const sinon = require('sinon')
+
+const MODULE_PATH = path.join(
   __dirname,
   '../../../../app/src/Features/Analytics/AnalyticsManager'
 )
-const sinon = require('sinon')
-const { expect } = require('chai')
-const Errors = require('../../../../app/src/Features/Errors/Errors')
 
 describe('AnalyticsManager', function() {
   beforeEach(function() {
     this.fakeUserId = '123abc'
-    this.settings = {
-      overleaf: true,
-      apis: { analytics: { url: 'analytics.test' } }
+    this.Settings = {
+      analytics: { enabled: true }
+    }
+
+    this.Queues = {
+      analytics: {
+        events: {
+          add: sinon.stub().resolves()
+        },
+        editingSessions: {
+          add: sinon.stub().resolves()
+        }
+      }
     }
     this.backgroundRequest = sinon.stub().yields()
     this.request = sinon.stub().yields()
-    this.AnalyticsManager = SandboxedModule.require(modulePath, {
+    this.AnalyticsManager = SandboxedModule.require(MODULE_PATH, {
       globals: {
         console: console
       },
       requires: {
-        'settings-sharelatex': this.settings,
-        '../../infrastructure/FaultTolerantRequest': {
-          backgroundRequest: this.backgroundRequest
-        },
-        '../Errors/Errors': Errors,
-        request: this.request,
+        'settings-sharelatex': this.Settings,
         'logger-sharelatex': {
-          log() {}
-        }
+          warn() {}
+        },
+        '../../infrastructure/Queues': this.Queues
       }
     })
   })
 
-  describe('checkAnalyticsRequest', function() {
-    it('ignores smoke test user', function(done) {
-      this.settings.smokeTest = { userId: this.fakeUserId }
-      this.AnalyticsManager.identifyUser(this.fakeUserId, '', error => {
-        expect(error).to.not.exist
-        sinon.assert.notCalled(this.request)
-        done()
-      })
+  describe('ignores when', function() {
+    it('user is smoke test user', function() {
+      this.Settings.smokeTest = { userId: this.fakeUserId }
+      this.AnalyticsManager.identifyUser(this.fakeUserId, '')
+      sinon.assert.notCalled(this.Queues.analytics.events.add)
     })
 
-    it('return error if analytics service is not configured', function(done) {
-      this.settings.apis.analytics = null
-      this.AnalyticsManager.identifyUser(this.fakeUserId, '', error => {
-        expect(error).to.be.instanceof(Errors.ServiceNotConfiguredError)
-        sinon.assert.notCalled(this.request)
-        done()
-      })
+    it('analytics service is disabled', function() {
+      this.Settings.analytics.enabled = false
+      this.AnalyticsManager.identifyUser(this.fakeUserId, '')
+      sinon.assert.notCalled(this.Queues.analytics.events.add)
     })
   })
 
-  describe('makes correct request to analytics', function() {
-    it('identifyUser', function(done) {
+  describe('queues the appropriate message for', function() {
+    it('identifyUser', function() {
       const oldUserId = '456def'
-      this.AnalyticsManager.identifyUser(this.fakeUserId, oldUserId, error => {
-        expect(error).to.not.exist
-        sinon.assert.calledWithMatch(this.backgroundRequest, {
-          body: { old_user_id: oldUserId },
-          url: 'analytics.test/user/123abc/identify'
-        })
-        done()
-      })
+      this.AnalyticsManager.identifyUser(this.fakeUserId, oldUserId)
+      sinon.assert.calledWithMatch(
+        this.Queues.analytics.events.add,
+        'identify',
+        {
+          userId: this.fakeUserId,
+          oldUserId
+        }
+      )
     })
 
-    it('recordEvent', function(done) {
+    it('recordEvent', function() {
       const event = 'fake-event'
-      this.AnalyticsManager.recordEvent(this.fakeUserId, event, null, error => {
-        expect(error).to.not.exist
-        sinon.assert.calledWithMatch(this.backgroundRequest, {
-          body: { event },
-          qs: { fromV2: 1 },
-          url: 'analytics.test/user/123abc/event',
-          timeout: 30000,
-          maxAttempts: 9,
-          backoffBase: 3000,
-          backoffMultiplier: 3
-        })
-        done()
+      this.AnalyticsManager.recordEvent(this.fakeUserId, event, null)
+      sinon.assert.calledWithMatch(this.Queues.analytics.events.add, 'event', {
+        event,
+        userId: this.fakeUserId,
+        segmentation: null
       })
     })
 
-    it('updateEditingSession', function(done) {
+    it('updateEditingSession', function() {
       const projectId = '789ghi'
       const countryCode = 'fr'
       this.AnalyticsManager.updateEditingSession(
         this.fakeUserId,
         projectId,
-        countryCode,
-        error => {
-          expect(error).to.not.exist
-          sinon.assert.calledWithMatch(this.backgroundRequest, {
-            qs: { userId: this.fakeUserId, projectId, countryCode, fromV2: 1 },
-            url: 'analytics.test/editingSession'
-          })
-          done()
-        }
+        countryCode
       )
+      sinon.assert.calledWithMatch(this.Queues.analytics.editingSessions.add, {
+        userId: this.fakeUserId,
+        projectId,
+        countryCode
+      })
     })
   })
 })

services/web/test/unit/src/infrastructure/FaultTolerantRequestTests.js

@@ -1,121 +0,0 @@
-const SandboxedModule = require('sandboxed-module')
-const path = require('path')
-const modulePath = path.join(
-  __dirname,
-  '../../../../app/src/infrastructure/FaultTolerantRequest'
-)
-const sinon = require('sinon')
-const { expect } = require('chai')
-
-describe('FaultTolerantRequest', function() {
-  beforeEach(function() {
-    this.request = sinon.stub().yields()
-    this.logger = {
-      err: sinon.stub()
-    }
-    this.FaultTolerantRequest = SandboxedModule.require(modulePath, {
-      globals: {
-        console: console
-      },
-      requires: {
-        requestretry: this.request,
-        'logger-sharelatex': this.logger
-      }
-    })
-  })
-
-  describe('exponentialBackoffStrategy', function() {
-    it('returns delays within expected range with default options', function() {
-      this.FaultTolerantRequest.request({}, () => {})
-      const delayStrategy = this.request.lastCall.args[0].delayStrategy
-      expect(delayStrategy()).to.be.closeTo(500, 250) // attempt 1
-      expect(delayStrategy()).to.be.closeTo(750, 375) // attempt 2
-      expect(delayStrategy()).to.be.closeTo(1125, 563) // attempt 3
-      expect(delayStrategy()).to.be.closeTo(1688, 844) // attempt 4
-      expect(delayStrategy()).to.be.closeTo(2531, 1266) // attempt 5
-      expect(delayStrategy()).to.be.closeTo(3797, 1899) // attempt 6
-      expect(delayStrategy()).to.be.closeTo(5695, 2848) // attempt 7
-      expect(delayStrategy()).to.be.closeTo(8543, 4272) // attempt 8
-      expect(delayStrategy()).to.be.closeTo(12814, 6408) // attempt 9
-      expect(delayStrategy()).to.be.closeTo(19222, 9610) // attempt 10
-    })
-
-    it('returns delays within expected range with custom options', function() {
-      const delayStrategy = this.FaultTolerantRequest.exponentialDelayStrategy(
-        3000,
-        3,
-        0.5
-      )
-      expect(delayStrategy()).to.be.closeTo(3000, 1500) // attempt 1
-      expect(delayStrategy()).to.be.closeTo(9000, 4500) // attempt 2
-      expect(delayStrategy()).to.be.closeTo(27000, 13500) // attempt 3
-      expect(delayStrategy()).to.be.closeTo(81000, 40500) // attempt 4
-      expect(delayStrategy()).to.be.closeTo(243000, 121500) // attempt 5
-      expect(delayStrategy()).to.be.closeTo(729000, 364500) // attempt 6
-      expect(delayStrategy()).to.be.closeTo(2187000, 1093500) // attempt 7
-      expect(delayStrategy()).to.be.closeTo(6561000, 3280500) // attempt 8
-      expect(delayStrategy()).to.be.closeTo(19683000, 9841500) // attempt 9
-      expect(delayStrategy()).to.be.closeTo(59049000, 29524500) // attempt 10
-    })
-  })
-
-  describe('request', function() {
-    it('sets retry options', function(done) {
-      this.FaultTolerantRequest.request({}, error => {
-        expect(error).to.not.exist
-        sinon.assert.calledOnce(this.request)
-
-        const { delayStrategy, maxAttempts } = this.request.lastCall.args[0]
-        expect(delayStrategy).to.be.a('function')
-        expect(maxAttempts).to.be.a('number')
-        done()
-      })
-    })
-
-    it("don't overwrite retry options", function(done) {
-      const customMaxAttempts = Math.random()
-      const customBase = Math.random()
-      const customMultiplier = Math.random()
-      const customRandomFactor = Math.random()
-      this.FaultTolerantRequest.request(
-        {
-          maxAttempts: customMaxAttempts,
-          backoffBase: customBase,
-          backoffMultiplier: customMultiplier,
-          backoffRandomFactor: customRandomFactor
-        },
-        error => {
-          expect(error).to.not.exist
-
-          const {
-            maxAttempts,
-            backoffBase,
-            backoffMultiplier,
-            backoffRandomFactor
-          } = this.request.lastCall.args[0]
-          expect(maxAttempts).to.equal(customMaxAttempts)
-          expect(backoffBase).to.equal(customBase)
-          expect(backoffMultiplier).to.equal(customMultiplier)
-          expect(backoffRandomFactor).to.equal(customRandomFactor)
-          done()
-        }
-      )
-    })
-  })
-
-  describe('backgroundRequest', function() {
-    it('logs error in the background', function(done) {
-      this.request.yields(new Error('Nope'))
-      this.logger.err = (options, message) => {
-        expect(options.url).to.equal('test.url')
-        done()
-      }
-      this.FaultTolerantRequest.backgroundRequest(
-        { url: 'test.url' },
-        error => {
-          expect(error).to.not.exist
-        }
-      )
-    })
-  })
-})