RetryManagerTests.js 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. import sinon from 'sinon'
  2. import { expect } from 'chai'
  3. import mongodb from 'mongodb-legacy'
  4. import { strict as esmock } from 'esmock'
  5. const { ObjectId } = mongodb
  6. const MODULE_PATH = '../../../../app/js/RetryManager.js'
  7. describe('RetryManager', function () {
  8. beforeEach(async function () {
  9. this.projectId1 = new ObjectId().toString()
  10. this.projectId2 = new ObjectId().toString()
  11. this.projectId3 = new ObjectId().toString()
  12. this.projectId4 = new ObjectId().toString()
  13. this.projectId5 = new ObjectId().toString()
  14. this.projectId6 = new ObjectId().toString()
  15. this.historyId = 12345
  16. this.WebApiManager = {
  17. promises: {
  18. getHistoryId: sinon.stub().resolves(this.historyId),
  19. },
  20. }
  21. this.RedisManager = {
  22. promises: {
  23. countUnprocessedUpdates: sinon.stub().resolves(0),
  24. },
  25. }
  26. this.ErrorRecorder = {
  27. promises: {
  28. getFailedProjects: sinon.stub().resolves([
  29. {
  30. project_id: this.projectId1,
  31. error: 'Error: Timeout',
  32. attempts: 1,
  33. },
  34. {
  35. project_id: this.projectId2,
  36. error: 'Error: Timeout',
  37. attempts: 25,
  38. },
  39. {
  40. project_id: this.projectId3,
  41. error: 'OpsOutOfOrderError: doc version out of order',
  42. attempts: 5,
  43. resyncAttempts: 1,
  44. },
  45. {
  46. project_id: this.projectId4,
  47. error: 'OpsOutOfOrderError: doc version out of order',
  48. attempts: 5,
  49. resyncAttempts: 2,
  50. },
  51. {
  52. project_id: this.projectId5,
  53. error: 'OError: sync ongoing',
  54. attempts: 10,
  55. resyncAttempts: 1,
  56. },
  57. {
  58. project_id: this.projectId6,
  59. error: 'OError: sync ongoing',
  60. attempts: 10,
  61. resyncAttempts: 2,
  62. },
  63. ]),
  64. getFailureRecord: sinon.stub().resolves(),
  65. },
  66. }
  67. this.SyncManager = {
  68. promises: {
  69. startResync: sinon.stub().resolves(),
  70. startHardResync: sinon.stub().resolves(),
  71. },
  72. }
  73. this.UpdatesProcessor = {
  74. promises: {
  75. processUpdatesForProject: sinon.stub().resolves(),
  76. },
  77. }
  78. this.settings = {
  79. redis: {
  80. lock: {
  81. key_schema: {
  82. projectHistoryLock({ projectId }) {
  83. return `ProjectHistoryLock:${projectId}`
  84. },
  85. },
  86. },
  87. },
  88. }
  89. this.RetryManager = await esmock(MODULE_PATH, {
  90. '../../../../app/js/WebApiManager.js': this.WebApiManager,
  91. '../../../../app/js/RedisManager.js': this.RedisManager,
  92. '../../../../app/js/ErrorRecorder.js': this.ErrorRecorder,
  93. '../../../../app/js/SyncManager.js': this.SyncManager,
  94. '../../../../app/js/UpdatesProcessor.js': this.UpdatesProcessor,
  95. '@overleaf/settings': this.settings,
  96. })
  97. })
  98. describe('RetryManager', function () {
  99. describe('for a soft failure', function () {
  100. beforeEach(async function () {
  101. await this.RetryManager.promises.retryFailures({ failureType: 'soft' })
  102. })
  103. it('should flush the queue', function () {
  104. expect(
  105. this.UpdatesProcessor.promises.processUpdatesForProject
  106. ).to.have.been.calledWith(this.projectId1)
  107. })
  108. })
  109. describe('for a hard failure', function () {
  110. beforeEach(async function () {
  111. await this.RetryManager.promises.retryFailures({ failureType: 'hard' })
  112. })
  113. it('should check the overleaf project id', function () {
  114. expect(
  115. this.WebApiManager.promises.getHistoryId
  116. ).to.have.been.calledWith(this.projectId2)
  117. })
  118. it("should start a soft resync when a resync hasn't been tried yet", function () {
  119. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  120. this.projectId2
  121. )
  122. })
  123. it('should start a hard resync when a resync has already been tried', function () {
  124. expect(
  125. this.SyncManager.promises.startHardResync
  126. ).to.have.been.calledWith(this.projectId3)
  127. })
  128. it("shouldn't try a resync after a hard resync attempt failed", function () {
  129. expect(
  130. this.SyncManager.promises.startHardResync
  131. ).not.to.have.been.calledWith(this.projectId4)
  132. })
  133. it('should use soft resync for sync ongoing failures regardless of resyncAttempts', function () {
  134. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  135. this.projectId5
  136. )
  137. expect(
  138. this.SyncManager.promises.startHardResync
  139. ).not.to.have.been.calledWith(this.projectId5)
  140. })
  141. it('should retry stuck sync ongoing failures via soft resync', function () {
  142. expect(this.SyncManager.promises.startResync).to.have.been.calledWith(
  143. this.projectId6
  144. )
  145. expect(
  146. this.SyncManager.promises.startHardResync
  147. ).not.to.have.been.calledWith(this.projectId6)
  148. })
  149. it('should count the unprocessed updates', function () {
  150. expect(
  151. this.RedisManager.promises.countUnprocessedUpdates
  152. ).to.have.been.calledWith(this.projectId2)
  153. })
  154. it('should check the failure record', function () {
  155. expect(
  156. this.ErrorRecorder.promises.getFailureRecord
  157. ).to.have.been.calledWith(this.projectId2)
  158. })
  159. })
  160. })
  161. })