PackManagerTests.coffee 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. sinon = require('sinon')
  2. chai = require('chai')
  3. assert = require('chai').assert
  4. should = chai.should()
  5. expect = chai.expect
  6. modulePath = "../../../../app/js/PackManager.js"
  7. SandboxedModule = require('sandboxed-module')
  8. {ObjectId} = require("mongojs")
  9. bson = require("bson")
  10. BSON = new bson.BSONPure()
  11. _ = require("underscore")
  12. tk = require "timekeeper"
  13. describe "PackManager", ->
  14. beforeEach ->
  15. tk.freeze(new Date())
  16. @PackManager = SandboxedModule.require modulePath, requires:
  17. "./mongojs" : { db: @db = {}, ObjectId: ObjectId, BSON: BSON }
  18. "./LockManager" : {}
  19. "./MongoAWS": {}
  20. "logger-sharelatex": { log: sinon.stub(), error: sinon.stub() }
  21. 'metrics-sharelatex': {inc: ()->}
  22. @callback = sinon.stub()
  23. @doc_id = ObjectId().toString()
  24. @project_id = ObjectId().toString()
  25. @PackManager.MAX_COUNT = 512
  26. afterEach ->
  27. tk.reset()
  28. describe "insertCompressedUpdates", ->
  29. beforeEach ->
  30. @lastUpdate = {
  31. _id: "12345"
  32. pack: [
  33. { op: "op-1", meta: "meta-1", v: 1},
  34. { op: "op-2", meta: "meta-2", v: 2}
  35. ]
  36. n : 2
  37. sz : 100
  38. }
  39. @newUpdates = [
  40. { op: "op-3", meta: "meta-3", v: 3},
  41. { op: "op-4", meta: "meta-4", v: 4}
  42. ]
  43. @db.docHistory =
  44. save: sinon.stub().callsArg(1)
  45. insert: sinon.stub().callsArg(1)
  46. findAndModify: sinon.stub().callsArg(1)
  47. describe "with no last update", ->
  48. beforeEach ->
  49. @PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
  50. @PackManager.insertCompressedUpdates @project_id, @doc_id, null, @newUpdates, true, @callback
  51. describe "for a small update", ->
  52. it "should insert the update into a new pack", ->
  53. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates, true).should.equal true
  54. it "should call the callback", ->
  55. @callback.called.should.equal true
  56. describe "for many small updates", ->
  57. beforeEach ->
  58. @newUpdates = ({ op: "op-#{i}", meta: "meta-#{i}", v: i} for i in [0..2048])
  59. @PackManager.insertCompressedUpdates @project_id, @doc_id, null, @newUpdates, false, @callback
  60. it "should append the initial updates to the existing pack", ->
  61. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[0...512], false).should.equal true
  62. it "should insert the first set remaining updates as a new pack", ->
  63. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[512...1024], false).should.equal true
  64. it "should insert the second set of remaining updates as a new pack", ->
  65. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1024...1536], false).should.equal true
  66. it "should insert the third set of remaining updates as a new pack", ->
  67. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1536...2048], false).should.equal true
  68. it "should insert the final set of remaining updates as a new pack", ->
  69. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2048..2048], false).should.equal true
  70. it "should call the callback", ->
  71. @callback.called.should.equal true
  72. describe "with an existing pack as the last update", ->
  73. beforeEach ->
  74. @PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5)
  75. @PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
  76. @PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
  77. describe "for a small update", ->
  78. it "should append the update to the existing pack", ->
  79. @PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates, false).should.equal true
  80. it "should not insert any new packs", ->
  81. @PackManager.insertUpdatesIntoNewPack.called.should.equal false
  82. it "should call the callback", ->
  83. @callback.called.should.equal true
  84. describe "for many small updates", ->
  85. beforeEach ->
  86. @newUpdates = ({ op: "op-#{i}", meta: "meta-#{i}", v: i} for i in [0..2048])
  87. @PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
  88. it "should append the initial updates to the existing pack", ->
  89. @PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates[0...510], false).should.equal true
  90. it "should insert the first set remaining updates as a new pack", ->
  91. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[510...1022], false).should.equal true
  92. it "should insert the second set of remaining updates as a new pack", ->
  93. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1022...1534], false).should.equal true
  94. it "should insert the third set of remaining updates as a new pack", ->
  95. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1534...2046], false).should.equal true
  96. it "should insert the final set of remaining updates as a new pack", ->
  97. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2046..2048], false).should.equal true
  98. it "should call the callback", ->
  99. @callback.called.should.equal true
  100. describe "for many big updates", ->
  101. beforeEach ->
  102. longString = ("a" for [0 .. (0.75*@PackManager.MAX_SIZE)]).join("")
  103. @newUpdates = ({ op: "op-#{i}-#{longString}", meta: "meta-#{i}", v: i} for i in [0..4])
  104. @PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
  105. it "should append the initial updates to the existing pack", ->
  106. @PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates[0..0], false).should.equal true
  107. it "should insert the first set remaining updates as a new pack", ->
  108. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1..1], false).should.equal true
  109. it "should insert the second set of remaining updates as a new pack", ->
  110. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2..2], false).should.equal true
  111. it "should insert the third set of remaining updates as a new pack", ->
  112. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[3..3], false).should.equal true
  113. it "should insert the final set of remaining updates as a new pack", ->
  114. @PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[4..4], false).should.equal true
  115. it "should call the callback", ->
  116. @callback.called.should.equal true
  117. describe "flushCompressedUpdates", ->
  118. describe "when there is no previous update", ->
  119. beforeEach ->
  120. @PackManager.flushCompressedUpdates @project_id, @doc_id, null, @newUpdates, true, @callback
  121. describe "for a small update that will expire", ->
  122. it "should insert the update into mongo", ->
  123. @db.docHistory.save.calledWithMatch({
  124. pack: @newUpdates,
  125. project_id: ObjectId(@project_id),
  126. doc_id: ObjectId(@doc_id)
  127. n: @newUpdates.length
  128. v: @newUpdates[0].v
  129. v_end: @newUpdates[@newUpdates.length-1].v
  130. }).should.equal true
  131. it "should set an expiry time in the future", ->
  132. @db.docHistory.save.calledWithMatch({
  133. expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
  134. }).should.equal true
  135. it "should call the callback", ->
  136. @callback.called.should.equal true
  137. describe "when there is a recent previous update in mongo that expires", ->
  138. beforeEach ->
  139. @lastUpdate = {
  140. _id: "12345"
  141. pack: [
  142. { op: "op-1", meta: "meta-1", v: 1},
  143. { op: "op-2", meta: "meta-2", v: 2}
  144. ]
  145. n : 2
  146. sz : 100
  147. meta: {start_ts: Date.now() - 6 * 3600 * 1000}
  148. expiresAt: new Date(Date.now())
  149. }
  150. @PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, true, @callback
  151. describe "for a small update that will expire", ->
  152. it "should append the update in mongo", ->
  153. @db.docHistory.findAndModify.calledWithMatch({
  154. query: {_id: @lastUpdate._id}
  155. update: { $push: {"pack" : {$each: @newUpdates}}, $set: {v_end: @newUpdates[@newUpdates.length-1].v}}
  156. }).should.equal true
  157. it "should set an expiry time in the future", ->
  158. @db.docHistory.findAndModify.calledWithMatch({
  159. update: {$set: {expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)}}
  160. }).should.equal true
  161. it "should call the callback", ->
  162. @callback.called.should.equal true
  163. describe "when there is a recent previous update in mongo that expires", ->
  164. beforeEach ->
  165. @PackManager.updateIndex = sinon.stub().callsArg(2)
  166. @lastUpdate = {
  167. _id: "12345"
  168. pack: [
  169. { op: "op-1", meta: "meta-1", v: 1},
  170. { op: "op-2", meta: "meta-2", v: 2}
  171. ]
  172. n : 2
  173. sz : 100
  174. meta: {start_ts: Date.now() - 6 * 3600 * 1000}
  175. expiresAt: new Date(Date.now())
  176. }
  177. @PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
  178. describe "for a small update that will not expire", ->
  179. it "should insert the update into mongo", ->
  180. @db.docHistory.save.calledWithMatch({
  181. pack: @newUpdates,
  182. project_id: ObjectId(@project_id),
  183. doc_id: ObjectId(@doc_id)
  184. n: @newUpdates.length
  185. v: @newUpdates[0].v
  186. v_end: @newUpdates[@newUpdates.length-1].v
  187. }).should.equal true
  188. it "should not set any expiry time", ->
  189. @db.docHistory.save.neverCalledWithMatch(sinon.match.has("expiresAt")).should.equal true
  190. it "should call the callback", ->
  191. @callback.called.should.equal true
  192. describe "when there is an old previous update in mongo", ->
  193. beforeEach ->
  194. @lastUpdate = {
  195. _id: "12345"
  196. pack: [
  197. { op: "op-1", meta: "meta-1", v: 1},
  198. { op: "op-2", meta: "meta-2", v: 2}
  199. ]
  200. n : 2
  201. sz : 100
  202. meta: {start_ts: Date.now() - 30 * 24 * 3600 * 1000}
  203. expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000)
  204. }
  205. @PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, true, @callback
  206. describe "for a small update that will expire", ->
  207. it "should insert the update into mongo", ->
  208. @db.docHistory.save.calledWithMatch({
  209. pack: @newUpdates,
  210. project_id: ObjectId(@project_id),
  211. doc_id: ObjectId(@doc_id)
  212. n: @newUpdates.length
  213. v: @newUpdates[0].v
  214. v_end: @newUpdates[@newUpdates.length-1].v
  215. }).should.equal true
  216. it "should set an expiry time in the future", ->
  217. @db.docHistory.save.calledWithMatch({
  218. expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
  219. }).should.equal true
  220. it "should call the callback", ->
  221. @callback.called.should.equal true
  222. describe "getOpsByVersionRange", ->
  223. describe "loadPacksByVersionRange", ->
  224. describe "fetchPacksIfNeeded", ->
  225. describe "makeProjectIterator", ->
  226. describe "getPackById", ->
  227. describe "increaseTTL", ->
  228. describe "getIndex", ->
  229. describe "getPackFromIndex", ->
  230. # getLastPackFromIndex:
  231. # getIndexWithKeys
  232. # initialiseIndex
  233. # updateIndex
  234. # findCompletedPacks
  235. # findUnindexedPacks
  236. # insertPacksIntoIndexWithLock
  237. # _insertPacksIntoIndex
  238. # archivePack
  239. # checkArchivedPack
  240. # processOldPack
  241. # updateIndexIfNeeded
  242. # findUnarchivedPacks
  243. describe "checkArchiveNotInProgress", ->
  244. describe "when an archive is in progress", ->
  245. beforeEach ->
  246. @db.docHistoryIndex =
  247. findOne: sinon.stub().callsArgWith(2, null, {inS3:false})
  248. @PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
  249. it "should call the callback", ->
  250. @callback.called.should.equal true
  251. it "should return an error", ->
  252. @callback.calledWith(new Error()).should.equal true
  253. describe "when an archive is completed", ->
  254. beforeEach ->
  255. @db.docHistoryIndex =
  256. findOne: sinon.stub().callsArgWith(2, null, {inS3:true})
  257. @PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
  258. it "should call the callback", ->
  259. @callback.called.should.equal true
  260. it "should return an error", ->
  261. @callback.calledWith(new Error()).should.equal true
  262. describe "when the archive has not started or completed", ->
  263. beforeEach ->
  264. @db.docHistoryIndex =
  265. findOne: sinon.stub().callsArgWith(2, null, {})
  266. @PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
  267. it "should call the callback", ->
  268. @callback.called.should.equal true
  269. it "should return with no error", ->
  270. @callback.calledWith(undefined).should.equal true
  271. # describe "setTTLOnArchivedPack", ->
  272. # beforeEach ->
  273. # @pack_id = "somepackid"
  274. # @onedayinms = 86400000
  275. # @db.docHistory =
  276. # findAndModify : sinon.stub().callsArgWith(1)
  277. # it "should set expires to 1 day", (done)->
  278. # #@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
  279. # @PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
  280. # args = @db.docHistory.findAndModify.args[0][0]
  281. # args.query._id.should.equal @pack_id
  282. # args.update['$set'].expiresAt.should.equal @onedayinms
  283. # done()
  284. # describe "_getOneDayInFutureWithRandomDelay", ->
  285. # beforeEach ->
  286. # @onedayinms = 86400000
  287. # @thirtyMins = 1000 * 60 * 30
  288. # it "should give 1 day + 30 mins random time", (done)->
  289. # loops = 10000
  290. # while --loops > 0
  291. # randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
  292. # randomDelay.should.be.above(0)
  293. # randomDelay.should.be.below(@thirtyMins + 1)
  294. # done()