RetryManagerTests.js 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. import sinon from 'sinon'
  2. import { expect } from 'chai'
  3. import mongodb from 'mongodb-legacy'
  4. import { strict as esmock } from 'esmock'
  5. const { ObjectId } = mongodb
  6. const MODULE_PATH = '../../../../app/js/RetryManager.js'
  7. describe('RetryManager', function () {
  8. beforeEach(async function () {
  9. this.projectId1 = new ObjectId().toString()
  10. this.projectId2 = new ObjectId().toString()
  11. this.projectId3 = new ObjectId().toString()
  12. this.projectId4 = new ObjectId().toString()
  13. this.projectId5 = new ObjectId().toString()
  14. this.projectId6 = new ObjectId().toString()
  15. this.historyId = 12345
  16. this.WebApiManager = {
  17. promises: {
  18. getHistoryId: sinon.stub().resolves(this.historyId),
  19. },
  20. }
  21. this.RedisManager = {
  22. promises: {
  23. countUnprocessedUpdates: sinon.stub().resolves(0),
  24. },
  25. }
  26. this.ErrorRecorder = {
  27. promises: {
  28. getFailedProjects: sinon.stub().resolves([
  29. {
  30. project_id: this.projectId1,
  31. error: 'Error: Timeout',
  32. attempts: 1,
  33. },
  34. {
  35. project_id: this.projectId2,
  36. error: 'Error: Timeout',
  37. attempts: 25,
  38. },
  39. {
  40. project_id: this.projectId3,
  41. error: 'OpsOutOfOrderError: doc version out of order',
  42. attempts: 5,
  43. resyncAttempts: 1,
  44. },
  45. {
  46. project_id: this.projectId4,
  47. error: 'OpsOutOfOrderError: doc version out of order',
  48. attempts: 5,
  49. resyncAttempts: 2,
  50. },
  51. {
  52. project_id: this.projectId5,
  53. error: 'OError: sync ongoing',
  54. attempts: 10,
  55. resyncAttempts: 1,
  56. },
  57. {
  58. project_id: this.projectId6,
  59. error: 'OError: sync ongoing',
  60. attempts: 10,
  61. resyncAttempts: 2,
  62. },
  63. ]),
  64. getFailureRecord: sinon.stub().resolves(),
  65. },
  66. }
  67. this.SyncManager = {
  68. promises: {
  69. startResync: sinon.stub().resolves(),
  70. startHardResync: sinon.stub().resolves(),
  71. },
  72. }
  73. this.UpdatesProcessor = {
  74. promises: {
  75. processUpdatesForProject: sinon.stub().resolves(),
  76. },
  77. }
  78. this.settings = {
  79. redis: {
  80. lock: {
  81. key_schema: {
  82. projectHistoryLock({ projectId }) {
  83. return `ProjectHistoryLock:${projectId}`
  84. },
  85. },
  86. },
  87. },
  88. }
  89. this.request = {}
  90. this.RetryManager = await esmock(MODULE_PATH, {
  91. '../../../../app/js/WebApiManager.js': this.WebApiManager,
  92. '../../../../app/js/RedisManager.js': this.RedisManager,
  93. '../../../../app/js/ErrorRecorder.js': this.ErrorRecorder,
  94. '../../../../app/js/SyncManager.js': this.SyncManager,
  95. '../../../../app/js/UpdatesProcessor.js': this.UpdatesProcessor,
  96. '@overleaf/settings': this.settings,
  97. request: this.request,
  98. })
  99. })
  100. describe('RetryManager', function () {
  101. describe('for a soft failure', function () {
  102. beforeEach(async function () {
  103. await this.RetryManager.promises.retryFailures({ failureType: 'soft' })
  104. })
  105. it('should flush the queue', function () {
  106. expect(
  107. this.UpdatesProcessor.promises.processUpdatesForProject
  108. ).to.have.been.calledWith(this.projectId1)
  109. })
  110. })
  111. describe('for a hard failure', function () {
  112. beforeEach(async function () {
  113. await this.RetryManager.promises.retryFailures({ failureType: 'hard' })
  114. })
  115. it('should check the overleaf project id', function () {
  116. expect(
  117. this.WebApiManager.promises.getHistoryId
  118. ).to.have.been.calledWith(this.projectId2)
  119. })
  120. it("should start a soft resync when a resync hasn't been tried yet", function () {
  121. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  122. this.projectId2
  123. )
  124. })
  125. it('should start a hard resync when a resync has already been tried', function () {
  126. expect(
  127. this.SyncManager.promises.startHardResync
  128. ).to.have.been.calledWith(this.projectId3)
  129. })
  130. it("shouldn't try a resync after a hard resync attempt failed", function () {
  131. expect(
  132. this.SyncManager.promises.startHardResync
  133. ).not.to.have.been.calledWith(this.projectId4)
  134. })
  135. it('should use soft resync for sync ongoing failures regardless of resyncAttempts', function () {
  136. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  137. this.projectId5
  138. )
  139. expect(
  140. this.SyncManager.promises.startHardResync
  141. ).not.to.have.been.calledWith(this.projectId5)
  142. })
  143. it('should retry stuck sync ongoing failures via soft resync', function () {
  144. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  145. this.projectId6
  146. )
  147. expect(
  148. this.SyncManager.promises.startHardResync
  149. ).not.to.have.been.calledWith(this.projectId6)
  150. })
  151. it('should count the unprocessed updates', function () {
  152. expect(
  153. this.RedisManager.promises.countUnprocessedUpdates
  154. ).to.have.been.calledWith(this.projectId2)
  155. })
  156. it('should check the failure record', function () {
  157. expect(
  158. this.ErrorRecorder.promises.getFailureRecord
  159. ).to.have.been.calledWith(this.projectId2)
  160. })
  161. })
  162. })
  163. })