FlushManager.js 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. /* eslint-disable
  2. camelcase,
  3. */
  4. // TODO: This file was created by bulk-decaffeinate.
  5. // Fix any style issues and re-enable lint.
  6. /*
  7. * decaffeinate suggestions:
  8. * DS101: Remove unnecessary use of Array.from
  9. * DS102: Remove unnecessary code created because of implicit returns
  10. * DS207: Consider shorter variations of null checks
  11. * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  12. */
  13. import async from 'async'
  14. import logger from '@overleaf/logger'
  15. import OError from '@overleaf/o-error'
  16. import _ from 'lodash'
  17. import * as RedisManager from './RedisManager.js'
  18. import * as UpdatesProcessor from './UpdatesProcessor.js'
  19. import * as ErrorRecorder from './ErrorRecorder.js'
  20. export function flushIfOld(project_id, cutoffTime, callback) {
  21. if (callback == null) {
  22. callback = function () {}
  23. }
  24. return RedisManager.getFirstOpTimestamp(
  25. project_id,
  26. function (err, firstOpTimestamp) {
  27. if (err != null) {
  28. return callback(OError.tag(err))
  29. }
  30. // In the normal case, the flush marker will be set with the
  31. // timestamp of the oldest operation in the queue by docupdater.
  32. // If the marker is not set for any reason, we flush it anyway
  33. // for safety.
  34. if (!firstOpTimestamp || firstOpTimestamp < cutoffTime) {
  35. logger.debug(
  36. { project_id, firstOpTimestamp, cutoffTime },
  37. 'flushing old project'
  38. )
  39. return UpdatesProcessor.processUpdatesForProject(project_id, callback)
  40. } else {
  41. return callback()
  42. }
  43. }
  44. )
  45. }
  46. export function flushOldOps(options, callback) {
  47. if (callback == null) {
  48. callback = function () {}
  49. }
  50. logger.debug({ options }, 'starting flush of old ops')
  51. // allow running flush in background for cron jobs
  52. if (options.background) {
  53. // return immediate response to client, then discard callback
  54. callback(null, { message: 'running flush in background' })
  55. callback = function () {}
  56. }
  57. return RedisManager.getProjectIdsWithHistoryOps(
  58. null,
  59. function (error, projectIds) {
  60. if (error != null) {
  61. return callback(OError.tag(error))
  62. }
  63. return ErrorRecorder.getFailedProjects(function (
  64. error,
  65. projectHistoryFailures
  66. ) {
  67. if (error != null) {
  68. return callback(OError.tag(error))
  69. }
  70. // exclude failed projects already in projectHistoryFailures
  71. const failedProjects = new Set()
  72. for (const entry of Array.from(projectHistoryFailures)) {
  73. failedProjects.add(entry.project_id)
  74. }
  75. // randomise order so we get different projects if there is a limit
  76. projectIds = _.shuffle(projectIds)
  77. const maxAge = options.maxAge || 6 * 3600 // default to 6 hours
  78. const cutoffTime = new Date(Date.now() - maxAge * 1000)
  79. const startTime = new Date()
  80. let count = 0
  81. const jobs = projectIds.map(
  82. project_id =>
  83. function (cb) {
  84. const timeTaken = new Date() - startTime
  85. count++
  86. if (
  87. (options != null ? options.timeout : undefined) &&
  88. timeTaken > options.timeout
  89. ) {
  90. // finish early due to timeout, return an error to bail out of the async iteration
  91. logger.debug('background retries timed out')
  92. return cb(new OError('retries timed out'))
  93. }
  94. if (
  95. (options != null ? options.limit : undefined) &&
  96. count > options.limit
  97. ) {
  98. // finish early due to reaching limit, return an error to bail out of the async iteration
  99. logger.debug({ count }, 'background retries hit limit')
  100. return cb(new OError('hit limit'))
  101. }
  102. if (failedProjects.has(project_id)) {
  103. // skip failed projects
  104. return setTimeout(cb, options.queueDelay || 100) // pause between flushes
  105. }
  106. return flushIfOld(project_id, cutoffTime, function (err) {
  107. if (err != null) {
  108. logger.warn(
  109. { project_id, flushErr: err },
  110. 'error flushing old project'
  111. )
  112. }
  113. return setTimeout(cb, options.queueDelay || 100)
  114. })
  115. }
  116. ) // pause between flushes
  117. return async.series(async.reflectAll(jobs), function (error, results) {
  118. const success = []
  119. const failure = []
  120. results.forEach((result, i) => {
  121. if (
  122. result.error != null &&
  123. !['retries timed out', 'hit limit'].includes(
  124. result?.error?.message
  125. )
  126. ) {
  127. // ignore expected errors
  128. return failure.push(projectIds[i])
  129. } else {
  130. return success.push(projectIds[i])
  131. }
  132. })
  133. return callback(error, { success, failure, failedProjects })
  134. })
  135. })
  136. }
  137. )
  138. }