Queues.js 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121
  1. const Queue = require('bull')
  2. const Settings = require('@overleaf/settings')
  3. const Features = require('../infrastructure/Features')
  4. const { addConnectionDrainer } = require('./GracefulShutdown')
  5. // Bull will keep a fixed number of the most recently completed jobs. This is
  6. // useful to inspect recently completed jobs. The bull prometheus exporter also
  7. // uses the completed job records to report on job duration.
  8. const MAX_COMPLETED_JOBS_RETAINED = 10000
  9. const MAX_FAILED_JOBS_RETAINED = 50000
  10. const MAX_FAILED_JOBS_RETAINED_ANALYTICS = 3000000
  11. const QUEUES_JOB_OPTIONS = {
  12. 'analytics-events': {
  13. removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
  14. },
  15. 'analytics-editing-sessions': {
  16. removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
  17. },
  18. 'analytics-account-mapping': {
  19. removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
  20. },
  21. 'analytics-user-properties': {
  22. removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
  23. },
  24. 'refresh-features': {
  25. removeOnFail: MAX_FAILED_JOBS_RETAINED,
  26. attempts: 3,
  27. },
  28. 'emails-onboarding': {
  29. removeOnFail: MAX_FAILED_JOBS_RETAINED,
  30. },
  31. 'post-registration-analytics': {
  32. removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
  33. },
  34. 'scheduled-jobs': {
  35. removeOnFail: MAX_FAILED_JOBS_RETAINED,
  36. attempts: 1,
  37. },
  38. 'confirm-institution-domain': {
  39. removeOnFail: MAX_FAILED_JOBS_RETAINED,
  40. attempts: 3,
  41. },
  42. 'group-sso-reminder': {
  43. removeOnFail: MAX_FAILED_JOBS_RETAINED,
  44. attempts: 3,
  45. },
  46. }
  47. const QUEUE_OPTIONS = {
  48. 'confirm-institution-domain': {
  49. limiter: {
  50. max: 1,
  51. duration: 60 * 1000,
  52. },
  53. },
  54. }
  55. const ANALYTICS_QUEUES = [
  56. 'analytics-account-mapping',
  57. 'analytics-events',
  58. 'analytics-editing-sessions',
  59. 'analytics-user-properties',
  60. 'post-registration-analytics',
  61. ]
  62. const queues = {}
  63. function getQueue(queueName) {
  64. if (!Features.hasFeature('saas')) {
  65. // Disable bull queue handling for Server Pro/CE by providing a stub interface.
  66. return {
  67. async add() {},
  68. process() {},
  69. }
  70. }
  71. if (!queues[queueName]) {
  72. const redisOptions = ANALYTICS_QUEUES.includes(queueName)
  73. ? Settings.redis.analyticsQueues
  74. : Settings.redis.queues
  75. const queueOptions = QUEUE_OPTIONS[queueName] || {}
  76. const jobOptions = QUEUES_JOB_OPTIONS[queueName] || {}
  77. queues[queueName] = new Queue(queueName, {
  78. // this configuration is duplicated in /services/analytics/app/js/Queues.js
  79. // and needs to be manually kept in sync whenever modified
  80. redis: redisOptions,
  81. ...queueOptions,
  82. defaultJobOptions: {
  83. removeOnComplete: MAX_COMPLETED_JOBS_RETAINED,
  84. attempts: 11,
  85. backoff: {
  86. type: 'exponential',
  87. delay: 3000,
  88. },
  89. ...jobOptions,
  90. },
  91. })
  92. // Disconnect from redis eventually.
  93. addConnectionDrainer(`bull queue ${queueName}`, async () => {
  94. await queues[queueName].disconnect()
  95. })
  96. }
  97. return queues[queueName]
  98. }
  99. async function createScheduledJob(queueName, { name, data, options }, delay) {
  100. await getQueue('scheduled-jobs').add(
  101. { queueName, name, data, options },
  102. {
  103. delay,
  104. }
  105. )
  106. }
  107. module.exports = {
  108. getQueue,
  109. createScheduledJob,
  110. }