PackManagerTests.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470
  1. /* eslint-disable
  2. mocha/no-identical-title,
  3. no-return-assign,
  4. no-unused-vars,
  5. */
  6. // TODO: This file was created by bulk-decaffeinate.
  7. // Fix any style issues and re-enable lint.
  8. /*
  9. * decaffeinate suggestions:
  10. * DS102: Remove unnecessary code created because of implicit returns
  11. * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  12. */
  13. const sinon = require('sinon');
  14. const chai = require('chai');
  15. const { assert } = require('chai');
  16. const should = chai.should();
  17. const { expect } = chai;
  18. const modulePath = "../../../../app/js/PackManager.js";
  19. const SandboxedModule = require('sandboxed-module');
  20. const {ObjectId} = require("mongojs");
  21. const bson = require("bson");
  22. const BSON = new bson.BSONPure();
  23. const _ = require("underscore");
  24. const tk = require("timekeeper");
  25. describe("PackManager", function() {
  26. beforeEach(function() {
  27. tk.freeze(new Date());
  28. this.PackManager = SandboxedModule.require(modulePath, { requires: {
  29. "./mongojs" : { db: (this.db = {}), ObjectId, BSON },
  30. "./LockManager" : {},
  31. "./MongoAWS": {},
  32. "logger-sharelatex": { log: sinon.stub(), error: sinon.stub() },
  33. 'metrics-sharelatex': {inc(){}},
  34. "./ProjectIterator": require("../../../../app/js/ProjectIterator.js"), // Cache for speed
  35. "settings-sharelatex": {
  36. redis: {lock: {key_schema: {}}}
  37. }
  38. }
  39. });
  40. this.callback = sinon.stub();
  41. this.doc_id = ObjectId().toString();
  42. this.project_id = ObjectId().toString();
  43. return this.PackManager.MAX_COUNT = 512;
  44. });
  45. afterEach(function() { return tk.reset(); });
  46. describe("insertCompressedUpdates", function() {
  47. beforeEach(function() {
  48. this.lastUpdate = {
  49. _id: "12345",
  50. pack: [
  51. { op: "op-1", meta: "meta-1", v: 1},
  52. { op: "op-2", meta: "meta-2", v: 2}
  53. ],
  54. n : 2,
  55. sz : 100
  56. };
  57. this.newUpdates = [
  58. { op: "op-3", meta: "meta-3", v: 3},
  59. { op: "op-4", meta: "meta-4", v: 4}
  60. ];
  61. return this.db.docHistory = {
  62. save: sinon.stub().callsArg(1),
  63. insert: sinon.stub().callsArg(1),
  64. findAndModify: sinon.stub().callsArg(1)
  65. };
  66. });
  67. describe("with no last update", function() {
  68. beforeEach(function() {
  69. this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4);
  70. return this.PackManager.insertCompressedUpdates(this.project_id, this.doc_id, null, this.newUpdates, true, this.callback);
  71. });
  72. describe("for a small update", function() {
  73. it("should insert the update into a new pack", function() {
  74. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates, true).should.equal(true);
  75. });
  76. return it("should call the callback", function() {
  77. return this.callback.called.should.equal(true);
  78. });
  79. });
  80. return describe("for many small updates", function() {
  81. beforeEach(function() {
  82. this.newUpdates = (__range__(0, 2048, true).map((i) => ({ op: `op-${i}`, meta: `meta-${i}`, v: i})));
  83. return this.PackManager.insertCompressedUpdates(this.project_id, this.doc_id, null, this.newUpdates, false, this.callback);
  84. });
  85. it("should append the initial updates to the existing pack", function() {
  86. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(0, 512), false).should.equal(true);
  87. });
  88. it("should insert the first set remaining updates as a new pack", function() {
  89. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(512, 1024), false).should.equal(true);
  90. });
  91. it("should insert the second set of remaining updates as a new pack", function() {
  92. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(1024, 1536), false).should.equal(true);
  93. });
  94. it("should insert the third set of remaining updates as a new pack", function() {
  95. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(1536, 2048), false).should.equal(true);
  96. });
  97. it("should insert the final set of remaining updates as a new pack", function() {
  98. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(2048, 2049), false).should.equal(true);
  99. });
  100. return it("should call the callback", function() {
  101. return this.callback.called.should.equal(true);
  102. });
  103. });
  104. });
  105. describe("with an existing pack as the last update", function() {
  106. beforeEach(function() {
  107. this.PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5);
  108. this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4);
  109. return this.PackManager.insertCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback);
  110. });
  111. describe("for a small update", function() {
  112. it("should append the update to the existing pack", function() {
  113. return this.PackManager.appendUpdatesToExistingPack.calledWith(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false).should.equal(true);
  114. });
  115. it("should not insert any new packs", function() {
  116. return this.PackManager.insertUpdatesIntoNewPack.called.should.equal(false);
  117. });
  118. return it("should call the callback", function() {
  119. return this.callback.called.should.equal(true);
  120. });
  121. });
  122. describe("for many small updates", function() {
  123. beforeEach(function() {
  124. this.newUpdates = (__range__(0, 2048, true).map((i) => ({ op: `op-${i}`, meta: `meta-${i}`, v: i})));
  125. return this.PackManager.insertCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback);
  126. });
  127. it("should append the initial updates to the existing pack", function() {
  128. return this.PackManager.appendUpdatesToExistingPack.calledWith(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates.slice(0, 510), false).should.equal(true);
  129. });
  130. it("should insert the first set remaining updates as a new pack", function() {
  131. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(510, 1022), false).should.equal(true);
  132. });
  133. it("should insert the second set of remaining updates as a new pack", function() {
  134. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(1022, 1534), false).should.equal(true);
  135. });
  136. it("should insert the third set of remaining updates as a new pack", function() {
  137. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(1534, 2046), false).should.equal(true);
  138. });
  139. it("should insert the final set of remaining updates as a new pack", function() {
  140. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(2046, 2049), false).should.equal(true);
  141. });
  142. return it("should call the callback", function() {
  143. return this.callback.called.should.equal(true);
  144. });
  145. });
  146. return describe("for many big updates", function() {
  147. beforeEach(function() {
  148. const longString = (__range__(0, (0.75*this.PackManager.MAX_SIZE), true).map((j) => "a")).join("");
  149. this.newUpdates = ([0, 1, 2, 3, 4].map((i) => ({ op: `op-${i}-${longString}`, meta: `meta-${i}`, v: i})));
  150. return this.PackManager.insertCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback);
  151. });
  152. it("should append the initial updates to the existing pack", function() {
  153. return this.PackManager.appendUpdatesToExistingPack.calledWith(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates.slice(0, 1), false).should.equal(true);
  154. });
  155. it("should insert the first set remaining updates as a new pack", function() {
  156. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(1, 2), false).should.equal(true);
  157. });
  158. it("should insert the second set of remaining updates as a new pack", function() {
  159. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(2, 3), false).should.equal(true);
  160. });
  161. it("should insert the third set of remaining updates as a new pack", function() {
  162. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(3, 4), false).should.equal(true);
  163. });
  164. it("should insert the final set of remaining updates as a new pack", function() {
  165. return this.PackManager.insertUpdatesIntoNewPack.calledWith(this.project_id, this.doc_id, this.newUpdates.slice(4, 5), false).should.equal(true);
  166. });
  167. return it("should call the callback", function() {
  168. return this.callback.called.should.equal(true);
  169. });
  170. });
  171. });
  172. describe("flushCompressedUpdates", function() { return describe("when there is no previous update", function() {
  173. beforeEach(function() {
  174. return this.PackManager.flushCompressedUpdates(this.project_id, this.doc_id, null, this.newUpdates, true, this.callback);
  175. });
  176. return describe("for a small update that will expire", function() {
  177. it("should insert the update into mongo", function() {
  178. return this.db.docHistory.save.calledWithMatch({
  179. pack: this.newUpdates,
  180. project_id: ObjectId(this.project_id),
  181. doc_id: ObjectId(this.doc_id),
  182. n: this.newUpdates.length,
  183. v: this.newUpdates[0].v,
  184. v_end: this.newUpdates[this.newUpdates.length-1].v
  185. }).should.equal(true);
  186. });
  187. it("should set an expiry time in the future", function() {
  188. return this.db.docHistory.save.calledWithMatch({
  189. expiresAt: new Date(Date.now() + (7 * 24 * 3600 * 1000))
  190. }).should.equal(true);
  191. });
  192. return it("should call the callback", function() {
  193. return this.callback.called.should.equal(true);
  194. });
  195. });
  196. }); }
  197. );
  198. describe("when there is a recent previous update in mongo that expires", function() {
  199. beforeEach(function() {
  200. this.lastUpdate = {
  201. _id: "12345",
  202. pack: [
  203. { op: "op-1", meta: "meta-1", v: 1},
  204. { op: "op-2", meta: "meta-2", v: 2}
  205. ],
  206. n : 2,
  207. sz : 100,
  208. meta: {start_ts: Date.now() - (6 * 3600 * 1000)},
  209. expiresAt: new Date(Date.now())
  210. };
  211. return this.PackManager.flushCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, true, this.callback);
  212. });
  213. return describe("for a small update that will expire", function() {
  214. it("should append the update in mongo", function() {
  215. return this.db.docHistory.findAndModify.calledWithMatch({
  216. query: {_id: this.lastUpdate._id},
  217. update: { $push: {"pack" : {$each: this.newUpdates}}, $set: {v_end: this.newUpdates[this.newUpdates.length-1].v}}
  218. }).should.equal(true);
  219. });
  220. it("should set an expiry time in the future", function() {
  221. return this.db.docHistory.findAndModify.calledWithMatch({
  222. update: {$set: {expiresAt: new Date(Date.now() + (7 * 24 * 3600 * 1000))}}
  223. }).should.equal(true);
  224. });
  225. return it("should call the callback", function() {
  226. return this.callback.called.should.equal(true);
  227. });
  228. });
  229. });
  230. describe("when there is a recent previous update in mongo that expires", function() {
  231. beforeEach(function() {
  232. this.PackManager.updateIndex = sinon.stub().callsArg(2);
  233. this.lastUpdate = {
  234. _id: "12345",
  235. pack: [
  236. { op: "op-1", meta: "meta-1", v: 1},
  237. { op: "op-2", meta: "meta-2", v: 2}
  238. ],
  239. n : 2,
  240. sz : 100,
  241. meta: {start_ts: Date.now() - (6 * 3600 * 1000)},
  242. expiresAt: new Date(Date.now())
  243. };
  244. return this.PackManager.flushCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback);
  245. });
  246. return describe("for a small update that will not expire", function() {
  247. it("should insert the update into mongo", function() {
  248. return this.db.docHistory.save.calledWithMatch({
  249. pack: this.newUpdates,
  250. project_id: ObjectId(this.project_id),
  251. doc_id: ObjectId(this.doc_id),
  252. n: this.newUpdates.length,
  253. v: this.newUpdates[0].v,
  254. v_end: this.newUpdates[this.newUpdates.length-1].v
  255. }).should.equal(true);
  256. });
  257. it("should not set any expiry time", function() {
  258. return this.db.docHistory.save.neverCalledWithMatch(sinon.match.has("expiresAt")).should.equal(true);
  259. });
  260. return it("should call the callback", function() {
  261. return this.callback.called.should.equal(true);
  262. });
  263. });
  264. });
  265. return describe("when there is an old previous update in mongo", function() {
  266. beforeEach(function() {
  267. this.lastUpdate = {
  268. _id: "12345",
  269. pack: [
  270. { op: "op-1", meta: "meta-1", v: 1},
  271. { op: "op-2", meta: "meta-2", v: 2}
  272. ],
  273. n : 2,
  274. sz : 100,
  275. meta: {start_ts: Date.now() - (30 * 24 * 3600 * 1000)},
  276. expiresAt: new Date(Date.now() - (30 * 24 * 3600 * 1000))
  277. };
  278. return this.PackManager.flushCompressedUpdates(this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, true, this.callback);
  279. });
  280. return describe("for a small update that will expire", function() {
  281. it("should insert the update into mongo", function() {
  282. return this.db.docHistory.save.calledWithMatch({
  283. pack: this.newUpdates,
  284. project_id: ObjectId(this.project_id),
  285. doc_id: ObjectId(this.doc_id),
  286. n: this.newUpdates.length,
  287. v: this.newUpdates[0].v,
  288. v_end: this.newUpdates[this.newUpdates.length-1].v
  289. }).should.equal(true);
  290. });
  291. it("should set an expiry time in the future", function() {
  292. return this.db.docHistory.save.calledWithMatch({
  293. expiresAt: new Date(Date.now() + (7 * 24 * 3600 * 1000))
  294. }).should.equal(true);
  295. });
  296. return it("should call the callback", function() {
  297. return this.callback.called.should.equal(true);
  298. });
  299. });
  300. });
  301. });
  302. describe("getOpsByVersionRange", function() {});
  303. describe("loadPacksByVersionRange", function() {});
  304. describe("fetchPacksIfNeeded", function() {});
  305. describe("makeProjectIterator", function() {});
  306. describe("getPackById", function() {});
  307. describe("increaseTTL", function() {});
  308. describe("getIndex", function() {});
  309. describe("getPackFromIndex", function() {});
  310. // getLastPackFromIndex:
  311. // getIndexWithKeys
  312. // initialiseIndex
  313. // updateIndex
  314. // findCompletedPacks
  315. // findUnindexedPacks
  316. // insertPacksIntoIndexWithLock
  317. // _insertPacksIntoIndex
  318. // archivePack
  319. // checkArchivedPack
  320. // processOldPack
  321. // updateIndexIfNeeded
  322. // findUnarchivedPacks
  323. return describe("checkArchiveNotInProgress", function() {
  324. describe("when an archive is in progress", function() {
  325. beforeEach(function() {
  326. this.db.docHistoryIndex =
  327. {findOne: sinon.stub().callsArgWith(2, null, {inS3:false})};
  328. return this.PackManager.checkArchiveNotInProgress(this.project_id, this.doc_id, this.pack_id, this.callback);
  329. });
  330. it("should call the callback", function() {
  331. return this.callback.called.should.equal(true);
  332. });
  333. return it("should return an error", function() {
  334. return this.callback.calledWith(sinon.match.has('message')).should.equal(true);
  335. });
  336. });
  337. describe("when an archive is completed", function() {
  338. beforeEach(function() {
  339. this.db.docHistoryIndex =
  340. {findOne: sinon.stub().callsArgWith(2, null, {inS3:true})};
  341. return this.PackManager.checkArchiveNotInProgress(this.project_id, this.doc_id, this.pack_id, this.callback);
  342. });
  343. it("should call the callback", function() {
  344. return this.callback.called.should.equal(true);
  345. });
  346. return it("should return an error", function() {
  347. return this.callback.calledWith(sinon.match.has('message')).should.equal(true);
  348. });
  349. });
  350. return describe("when the archive has not started or completed", function() {
  351. beforeEach(function() {
  352. this.db.docHistoryIndex =
  353. {findOne: sinon.stub().callsArgWith(2, null, {})};
  354. return this.PackManager.checkArchiveNotInProgress(this.project_id, this.doc_id, this.pack_id, this.callback);
  355. });
  356. it("should call the callback with no error", function() {
  357. return this.callback.called.should.equal(true);
  358. });
  359. return it("should return with no error", function() {
  360. return (typeof this.callback.lastCall.args[0]).should.equal('undefined');
  361. });
  362. });
  363. });
  364. });
  365. // describe "setTTLOnArchivedPack", ->
  366. // beforeEach ->
  367. // @pack_id = "somepackid"
  368. // @onedayinms = 86400000
  369. // @db.docHistory =
  370. // findAndModify : sinon.stub().callsArgWith(1)
  371. // it "should set expires to 1 day", (done)->
  372. // #@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
  373. // @PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
  374. // args = @db.docHistory.findAndModify.args[0][0]
  375. // args.query._id.should.equal @pack_id
  376. // args.update['$set'].expiresAt.should.equal @onedayinms
  377. // done()
  378. // describe "_getOneDayInFutureWithRandomDelay", ->
  379. // beforeEach ->
  380. // @onedayinms = 86400000
  381. // @thirtyMins = 1000 * 60 * 30
  382. // it "should give 1 day + 30 mins random time", (done)->
  383. // loops = 10000
  384. // while --loops > 0
  385. // randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
  386. // randomDelay.should.be.above(0)
  387. // randomDelay.should.be.below(@thirtyMins + 1)
  388. // done()
  389. function __range__(left, right, inclusive) {
  390. const range = [];
  391. const ascending = left < right;
  392. const end = !inclusive ? right : ascending ? right + 1 : right - 1;
  393. for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
  394. range.push(i);
  395. }
  396. return range;
  397. }