SyncManagerTests.js 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265
  1. // @ts-check
  2. import sinon from 'sinon'
  3. import { expect } from 'chai'
  4. import mongodb from 'mongodb-legacy'
  5. import tk from 'timekeeper'
  6. import { Comment, Range } from 'overleaf-editor-core'
  7. import { strict as esmock } from 'esmock'
  8. const { ObjectId } = mongodb
  9. const MODULE_PATH = '../../../../app/js/SyncManager.js'
  10. const timestamp = new Date()
  11. function resyncProjectStructureUpdate(docs, files) {
  12. return {
  13. resyncProjectStructure: { docs, files },
  14. meta: {
  15. ts: timestamp,
  16. },
  17. }
  18. }
  19. function docContentSyncUpdate(
  20. doc,
  21. content,
  22. ranges = {},
  23. resolvedComments = []
  24. ) {
  25. return {
  26. path: doc.path,
  27. doc: doc.doc,
  28. resyncDocContent: {
  29. content,
  30. ranges,
  31. resolvedComments,
  32. },
  33. meta: {
  34. ts: timestamp,
  35. },
  36. }
  37. }
  38. function makeComment(commentId, pos, text) {
  39. return {
  40. id: commentId,
  41. op: { p: pos, c: text, t: commentId },
  42. meta: {
  43. ts: timestamp,
  44. },
  45. }
  46. }
  47. describe('SyncManager', function () {
  48. beforeEach(async function () {
  49. this.now = new Date()
  50. tk.freeze(this.now)
  51. this.projectId = new ObjectId().toString()
  52. this.historyId = 'mock-overleaf-id'
  53. this.syncState = { origin: { kind: 'history-resync' } }
  54. this.db = {
  55. projectHistorySyncState: {
  56. findOne: sinon.stub().resolves(this.syncState),
  57. updateOne: sinon.stub().resolves(),
  58. },
  59. }
  60. this.extendLock = sinon.stub().resolves()
  61. this.LockManager = {
  62. promises: {
  63. runWithLock: sinon.stub().callsFake(async (key, runner) => {
  64. await runner(this.extendLock)
  65. }),
  66. },
  67. }
  68. this.UpdateCompressor = {
  69. diffAsShareJsOps: sinon.stub().returns([]),
  70. }
  71. this.UpdateTranslator = {
  72. isTextUpdate: sinon.stub(),
  73. _convertPathname: sinon.stub(),
  74. }
  75. this.WebApiManager = {
  76. promises: {
  77. getHistoryId: sinon.stub(),
  78. requestResync: sinon.stub().resolves(),
  79. },
  80. }
  81. this.WebApiManager.promises.getHistoryId
  82. .withArgs(this.projectId)
  83. .resolves(this.historyId)
  84. this.ErrorRecorder = {
  85. promises: {
  86. record: sinon.stub().resolves(),
  87. recordSyncStart: sinon.stub().resolves(),
  88. },
  89. }
  90. this.RedisManager = {}
  91. this.SnapshotManager = {
  92. promises: {
  93. getLatestSnapshot: sinon.stub(),
  94. },
  95. }
  96. this.HistoryStoreManager = {
  97. getBlobStore: sinon.stub(),
  98. _getBlobHashFromString: sinon.stub().returns('random-hash'),
  99. }
  100. this.HashManager = {
  101. _getBlobHashFromString: sinon.stub(),
  102. }
  103. this.Metrics = { inc: sinon.stub() }
  104. this.Settings = {
  105. redis: {
  106. lock: {
  107. key_schema: {
  108. projectHistoryLock({ project_id: projectId }) {
  109. return `ProjectHistoryLock:${projectId}`
  110. },
  111. },
  112. },
  113. },
  114. }
  115. this.SyncManager = await esmock(MODULE_PATH, {
  116. '../../../../app/js/LockManager.js': this.LockManager,
  117. '../../../../app/js/UpdateCompressor.js': this.UpdateCompressor,
  118. '../../../../app/js/UpdateTranslator.js': this.UpdateTranslator,
  119. '../../../../app/js/mongodb.js': { ObjectId, db: this.db },
  120. '../../../../app/js/WebApiManager.js': this.WebApiManager,
  121. '../../../../app/js/ErrorRecorder.js': this.ErrorRecorder,
  122. '../../../../app/js/RedisManager.js': this.RedisManager,
  123. '../../../../app/js/SnapshotManager.js': this.SnapshotManager,
  124. '../../../../app/js/HistoryStoreManager.js': this.HistoryStoreManager,
  125. '../../../../app/js/HashManager.js': this.HashManager,
  126. '@overleaf/metrics': this.Metrics,
  127. '@overleaf/settings': this.Settings,
  128. })
  129. })
  130. afterEach(function () {
  131. tk.reset()
  132. })
  133. describe('startResync', function () {
  134. describe('if a sync is not in progress', function () {
  135. beforeEach(async function () {
  136. this.db.projectHistorySyncState.findOne.resolves({})
  137. await this.SyncManager.promises.startResync(this.projectId)
  138. })
  139. it('takes the project lock', function () {
  140. expect(this.LockManager.promises.runWithLock).to.have.been.calledWith(
  141. `ProjectHistoryLock:${this.projectId}`
  142. )
  143. })
  144. it('gets the sync state from mongo', function () {
  145. expect(this.db.projectHistorySyncState.findOne).to.have.been.calledWith(
  146. { project_id: new ObjectId(this.projectId) }
  147. )
  148. })
  149. it('requests a resync from web', function () {
  150. expect(
  151. this.WebApiManager.promises.requestResync
  152. ).to.have.been.calledWith(this.projectId)
  153. })
  154. it('sets the sync state in mongo and prevents it expiring', function () {
  155. expect(
  156. this.db.projectHistorySyncState.updateOne
  157. ).to.have.been.calledWith(
  158. {
  159. project_id: new ObjectId(this.projectId),
  160. },
  161. sinon.match({
  162. $set: {
  163. resyncProjectStructure: true,
  164. resyncDocContents: [],
  165. origin: { kind: 'history-resync' },
  166. },
  167. $currentDate: { lastUpdated: true },
  168. $inc: { resyncCount: 1 },
  169. $unset: { expiresAt: true },
  170. }),
  171. {
  172. upsert: true,
  173. }
  174. )
  175. })
  176. })
  177. describe('if project structure sync is in progress', function () {
  178. beforeEach(function () {
  179. const syncState = { resyncProjectStructure: true }
  180. this.db.projectHistorySyncState.findOne.resolves(syncState)
  181. })
  182. it('returns an error if already syncing', async function () {
  183. await expect(
  184. this.SyncManager.promises.startResync(this.projectId)
  185. ).to.be.rejectedWith('sync ongoing')
  186. })
  187. })
  188. describe('if doc content sync in is progress', function () {
  189. beforeEach(async function () {
  190. const syncState = { resyncDocContents: ['/foo.tex'] }
  191. this.db.projectHistorySyncState.findOne.resolves(syncState)
  192. })
  193. it('returns an error if already syncing', async function () {
  194. await expect(
  195. this.SyncManager.promises.startResync(this.projectId)
  196. ).to.be.rejectedWith('sync ongoing')
  197. })
  198. })
  199. })
  200. describe('setResyncState', function () {
  201. describe('when the sync is starting', function () {
  202. beforeEach(function () {
  203. this.syncState = {
  204. toRaw() {
  205. return {
  206. resyncProjectStructure: true,
  207. resyncDocContents: [],
  208. origin: { kind: 'history-resync' },
  209. }
  210. },
  211. isSyncOngoing: sinon.stub().returns(true),
  212. }
  213. })
  214. it('sets the sync state in mongo and prevents it expiring', async function () {
  215. // SyncState is a private class of SyncManager
  216. // we know the interface however:
  217. await this.SyncManager.promises.setResyncState(
  218. this.projectId,
  219. this.syncState
  220. )
  221. expect(
  222. this.db.projectHistorySyncState.updateOne
  223. ).to.have.been.calledWith(
  224. { project_id: new ObjectId(this.projectId) },
  225. sinon.match({
  226. $set: this.syncState.toRaw(),
  227. $currentDate: { lastUpdated: true },
  228. $inc: { resyncCount: 1 },
  229. $unset: { expiresAt: true },
  230. }),
  231. { upsert: true }
  232. )
  233. })
  234. })
  235. describe('when the sync is ending', function () {
  236. beforeEach(function () {
  237. this.syncState = {
  238. toRaw() {
  239. return {
  240. resyncProjectStructure: false,
  241. resyncDocContents: [],
  242. origin: { kind: 'history-resync' },
  243. }
  244. },
  245. isSyncOngoing: sinon.stub().returns(false),
  246. }
  247. })
  248. it('sets the sync state entry in mongo to expire', async function () {
  249. await this.SyncManager.promises.setResyncState(
  250. this.projectId,
  251. this.syncState
  252. )
  253. expect(
  254. this.db.projectHistorySyncState.updateOne
  255. ).to.have.been.calledWith(
  256. { project_id: new ObjectId(this.projectId) },
  257. sinon.match({
  258. $set: {
  259. resyncProjectStructure: false,
  260. resyncDocContents: [],
  261. origin: { kind: 'history-resync' },
  262. expiresAt: new Date(this.now.getTime() + 90 * 24 * 3600 * 1000),
  263. },
  264. $currentDate: { lastUpdated: true },
  265. }),
  266. { upsert: true }
  267. )
  268. })
  269. })
  270. describe('when the new sync state is null', function () {
  271. it('does not update the sync state in mongo', async function () {
  272. // SyncState is a private class of SyncManager
  273. // we know the interface however:
  274. await this.SyncManager.promises.setResyncState(this.projectId, null)
  275. expect(this.db.projectHistorySyncState.updateOne).to.not.have.been
  276. .called
  277. })
  278. })
  279. })
  280. describe('skipUpdatesDuringSync', function () {
  281. describe('if a sync is not in progress', function () {
  282. beforeEach(async function () {
  283. this.db.projectHistorySyncState.findOne.resolves({})
  284. this.updates = ['some', 'mock', 'updates']
  285. this.result = await this.SyncManager.promises.skipUpdatesDuringSync(
  286. this.projectId,
  287. this.updates
  288. )
  289. })
  290. it('returns all updates', function () {
  291. expect(this.result.updates).to.deep.equal(this.updates)
  292. })
  293. it('should not return any newSyncState', function () {
  294. expect(this.result.syncState).to.be.null
  295. })
  296. })
  297. describe('if a sync in is progress', function () {
  298. beforeEach(function () {
  299. this.renameUpdate = {
  300. pathname: 'old.tex',
  301. newPathname: 'new.tex',
  302. }
  303. this.projectStructureSyncUpdate = {
  304. resyncProjectStructure: {
  305. docs: [{ path: 'new.tex' }],
  306. files: [],
  307. },
  308. }
  309. this.textUpdate = {
  310. doc: new ObjectId(),
  311. op: [{ i: 'a', p: 4 }],
  312. meta: {
  313. pathname: 'new.tex',
  314. doc_length: 4,
  315. },
  316. }
  317. this.docContentSyncUpdate = {
  318. path: 'new.tex',
  319. resyncDocContent: {
  320. content: 'a',
  321. },
  322. }
  323. this.UpdateTranslator.isTextUpdate
  324. .withArgs(this.renameUpdate)
  325. .returns(false)
  326. this.UpdateTranslator.isTextUpdate
  327. .withArgs(this.projectStructureSyncUpdate)
  328. .returns(false)
  329. this.UpdateTranslator.isTextUpdate
  330. .withArgs(this.docContentSyncUpdate)
  331. .returns(false)
  332. this.UpdateTranslator.isTextUpdate
  333. .withArgs(this.textUpdate)
  334. .returns(true)
  335. const syncState = {
  336. resyncProjectStructure: true,
  337. resyncDocContents: [],
  338. origin: { kind: 'history-resync' },
  339. }
  340. this.db.projectHistorySyncState.findOne.resolves(syncState)
  341. })
  342. it('remove updates before a project structure sync update', async function () {
  343. const updates = [
  344. this.renameUpdate,
  345. this.textUpdate,
  346. this.projectStructureSyncUpdate,
  347. ]
  348. const { updates: filteredUpdates, syncState } =
  349. await this.SyncManager.promises.skipUpdatesDuringSync(
  350. this.projectId,
  351. updates
  352. )
  353. expect(filteredUpdates).to.deep.equal([this.projectStructureSyncUpdate])
  354. expect(syncState.toRaw()).to.deep.equal({
  355. resyncProjectStructure: false,
  356. resyncDocContents: ['new.tex'],
  357. origin: { kind: 'history-resync' },
  358. })
  359. })
  360. it('allow project structure updates after project structure sync update', async function () {
  361. const updates = [this.projectStructureSyncUpdate, this.renameUpdate]
  362. const { updates: filteredUpdates, syncState } =
  363. await this.SyncManager.promises.skipUpdatesDuringSync(
  364. this.projectId,
  365. updates
  366. )
  367. expect(filteredUpdates).to.deep.equal([
  368. this.projectStructureSyncUpdate,
  369. this.renameUpdate,
  370. ])
  371. expect(syncState.toRaw()).to.deep.equal({
  372. resyncProjectStructure: false,
  373. resyncDocContents: ['new.tex'],
  374. origin: { kind: 'history-resync' },
  375. })
  376. })
  377. it('remove text updates for a doc before doc sync update', async function () {
  378. const updates = [
  379. this.projectStructureSyncUpdate,
  380. this.textUpdate,
  381. this.docContentSyncUpdate,
  382. ]
  383. const { updates: filteredUpdates, syncState } =
  384. await this.SyncManager.promises.skipUpdatesDuringSync(
  385. this.projectId,
  386. updates
  387. )
  388. expect(filteredUpdates).to.deep.equal([
  389. this.projectStructureSyncUpdate,
  390. this.docContentSyncUpdate,
  391. ])
  392. expect(syncState.toRaw()).to.deep.equal({
  393. resyncProjectStructure: false,
  394. resyncDocContents: [],
  395. origin: { kind: 'history-resync' },
  396. })
  397. })
  398. it('allow text updates for a doc after doc sync update', async function () {
  399. const updates = [
  400. this.projectStructureSyncUpdate,
  401. this.docContentSyncUpdate,
  402. this.textUpdate,
  403. ]
  404. const { updates: filteredUpdates, syncState } =
  405. await this.SyncManager.promises.skipUpdatesDuringSync(
  406. this.projectId,
  407. updates
  408. )
  409. expect(filteredUpdates).to.deep.equal([
  410. this.projectStructureSyncUpdate,
  411. this.docContentSyncUpdate,
  412. this.textUpdate,
  413. ])
  414. expect(syncState.toRaw()).to.deep.equal({
  415. resyncProjectStructure: false,
  416. resyncDocContents: [],
  417. origin: { kind: 'history-resync' },
  418. })
  419. })
  420. })
  421. })
  422. describe('expandSyncUpdates', function () {
  423. beforeEach(function () {
  424. this.persistedDoc = {
  425. doc: 'doc-id',
  426. path: 'main.tex',
  427. }
  428. this.persistedDocContent = 'the quick brown fox jumps over the lazy fox'
  429. this.persistedFile = {
  430. file: 'file-id',
  431. path: '1.png',
  432. _hash: 'abcde',
  433. }
  434. this.loadedSnapshotDoc = {
  435. isEditable: sinon.stub().returns(true),
  436. getContent: sinon.stub().returns(this.persistedDocContent),
  437. getComments: sinon
  438. .stub()
  439. .returns({ toArray: sinon.stub().returns([]) }),
  440. getHash: sinon.stub().returns(null),
  441. }
  442. this.fileMap = {
  443. 'main.tex': {
  444. isEditable: sinon.stub().returns(true),
  445. getContent: sinon.stub().returns(null),
  446. getHash: sinon.stub().returns(null),
  447. load: sinon.stub().resolves(this.loadedSnapshotDoc),
  448. },
  449. '1.png': {
  450. isEditable: sinon.stub().returns(false),
  451. data: { hash: this.persistedFile._hash },
  452. },
  453. }
  454. this.UpdateTranslator._convertPathname
  455. .withArgs('main.tex')
  456. .returns('main.tex')
  457. this.UpdateTranslator._convertPathname
  458. .withArgs('/main.tex')
  459. .returns('main.tex')
  460. this.UpdateTranslator._convertPathname
  461. .withArgs('another.tex')
  462. .returns('another.tex')
  463. this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png')
  464. this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png')
  465. this.SnapshotManager.promises.getLatestSnapshot.resolves(this.fileMap)
  466. })
  467. it('returns updates if no sync updates are queued', async function () {
  468. const updates = ['some', 'mock', 'updates']
  469. const expandedUpdates = await this.SyncManager.promises.expandSyncUpdates(
  470. this.projectId,
  471. this.historyId,
  472. updates,
  473. this.extendLock
  474. )
  475. expect(expandedUpdates).to.equal(updates)
  476. expect(this.SnapshotManager.promises.getLatestSnapshot).to.not.have.been
  477. .called
  478. expect(this.extendLock).to.not.have.been.called
  479. })
  480. describe('expanding project structure sync updates', function () {
  481. it('queues nothing for expected docs and files', async function () {
  482. const updates = [
  483. resyncProjectStructureUpdate(
  484. [this.persistedDoc],
  485. [this.persistedFile]
  486. ),
  487. ]
  488. const expandedUpdates =
  489. await this.SyncManager.promises.expandSyncUpdates(
  490. this.projectId,
  491. this.historyId,
  492. updates,
  493. this.extendLock
  494. )
  495. expect(expandedUpdates).to.deep.equal([])
  496. expect(this.extendLock).to.have.been.called
  497. })
  498. it('queues file removes for unexpected files', async function () {
  499. const updates = [resyncProjectStructureUpdate([this.persistedDoc], [])]
  500. const expandedUpdates =
  501. await this.SyncManager.promises.expandSyncUpdates(
  502. this.projectId,
  503. this.historyId,
  504. updates,
  505. this.extendLock
  506. )
  507. expect(expandedUpdates).to.deep.equal([
  508. {
  509. pathname: this.persistedFile.path,
  510. new_pathname: '',
  511. meta: {
  512. resync: true,
  513. ts: timestamp,
  514. origin: { kind: 'history-resync' },
  515. },
  516. },
  517. ])
  518. expect(this.extendLock).to.have.been.called
  519. })
  520. it('queues doc removes for unexpected docs', async function () {
  521. const updates = [resyncProjectStructureUpdate([], [this.persistedFile])]
  522. const expandedUpdates =
  523. await this.SyncManager.promises.expandSyncUpdates(
  524. this.projectId,
  525. this.historyId,
  526. updates,
  527. this.extendLock
  528. )
  529. expect(expandedUpdates).to.deep.equal([
  530. {
  531. pathname: this.persistedDoc.path,
  532. new_pathname: '',
  533. meta: {
  534. resync: true,
  535. ts: timestamp,
  536. origin: { kind: 'history-resync' },
  537. },
  538. },
  539. ])
  540. expect(this.extendLock).to.have.been.called
  541. })
  542. it('queues file additions for missing files', async function () {
  543. const newFile = {
  544. path: '2.png',
  545. file: {},
  546. url: 'filestore/2.png',
  547. }
  548. const updates = [
  549. resyncProjectStructureUpdate(
  550. [this.persistedDoc],
  551. [this.persistedFile, newFile]
  552. ),
  553. ]
  554. const expandedUpdates =
  555. await this.SyncManager.promises.expandSyncUpdates(
  556. this.projectId,
  557. this.historyId,
  558. updates,
  559. this.extendLock
  560. )
  561. expect(expandedUpdates).to.deep.equal([
  562. {
  563. pathname: newFile.path,
  564. file: newFile.file,
  565. url: newFile.url,
  566. meta: {
  567. resync: true,
  568. ts: timestamp,
  569. origin: { kind: 'history-resync' },
  570. },
  571. },
  572. ])
  573. expect(this.extendLock).to.have.been.called
  574. })
  575. it('queues blank doc additions for missing docs', async function () {
  576. const newDoc = {
  577. path: 'another.tex',
  578. doc: new ObjectId().toString(),
  579. }
  580. const updates = [
  581. resyncProjectStructureUpdate(
  582. [this.persistedDoc, newDoc],
  583. [this.persistedFile]
  584. ),
  585. ]
  586. const expandedUpdates =
  587. await this.SyncManager.promises.expandSyncUpdates(
  588. this.projectId,
  589. this.historyId,
  590. updates,
  591. this.extendLock
  592. )
  593. expect(expandedUpdates).to.deep.equal([
  594. {
  595. pathname: newDoc.path,
  596. doc: newDoc.doc,
  597. docLines: '',
  598. meta: {
  599. resync: true,
  600. ts: timestamp,
  601. origin: { kind: 'history-resync' },
  602. },
  603. },
  604. ])
  605. expect(this.extendLock).to.have.been.called
  606. })
  607. it('removes and re-adds files if whether they are binary differs', async function () {
  608. const fileWichWasADoc = {
  609. path: this.persistedDoc.path,
  610. url: 'filestore/2.png',
  611. _hash: 'other-hash',
  612. }
  613. const updates = [
  614. resyncProjectStructureUpdate(
  615. [],
  616. [fileWichWasADoc, this.persistedFile]
  617. ),
  618. ]
  619. const expandedUpdates =
  620. await this.SyncManager.promises.expandSyncUpdates(
  621. this.projectId,
  622. this.historyId,
  623. updates,
  624. this.extendLock
  625. )
  626. expect(expandedUpdates).to.deep.equal([
  627. {
  628. pathname: fileWichWasADoc.path,
  629. new_pathname: '',
  630. meta: {
  631. resync: true,
  632. ts: timestamp,
  633. origin: { kind: 'history-resync' },
  634. },
  635. },
  636. {
  637. pathname: fileWichWasADoc.path,
  638. file: fileWichWasADoc.file,
  639. url: fileWichWasADoc.url,
  640. meta: {
  641. resync: true,
  642. ts: timestamp,
  643. origin: { kind: 'history-resync' },
  644. },
  645. },
  646. ])
  647. expect(this.extendLock).to.have.been.called
  648. })
  649. it("does not remove and re-add files if the expected file doesn't have a hash", async function () {
  650. const fileWichWasADoc = {
  651. path: this.persistedDoc.path,
  652. url: 'filestore/2.png',
  653. }
  654. const updates = [
  655. resyncProjectStructureUpdate(
  656. [],
  657. [fileWichWasADoc, this.persistedFile]
  658. ),
  659. ]
  660. const expandedUpdates =
  661. await this.SyncManager.promises.expandSyncUpdates(
  662. this.projectId,
  663. this.historyId,
  664. updates,
  665. this.extendLock
  666. )
  667. expect(expandedUpdates).to.deep.equal([])
  668. expect(this.extendLock).to.have.been.called
  669. })
  670. it('does not remove and re-add editable files if there is a binary file with same hash', async function () {
  671. const binaryFile = {
  672. file: Object().toString(),
  673. // The paths in the resyncProjectStructureUpdate must have a leading slash ('/')
  674. // The other unit tests in this file are incorrectly missing the leading slash.
  675. // The leading slash is present in web where the paths are created with
  676. // ProjectEntityHandler.getAllEntitiesFromProject in ProjectEntityUpdateHandler.resyncProjectHistory.
  677. path: '/' + this.persistedDoc.path,
  678. url: 'filestore/12345',
  679. _hash: 'abcdef',
  680. }
  681. this.fileMap['main.tex'].data = { hash: 'abcdef' }
  682. const updates = [
  683. resyncProjectStructureUpdate([], [binaryFile, this.persistedFile]),
  684. ]
  685. const expandedUpdates =
  686. await this.SyncManager.promises.expandSyncUpdates(
  687. this.projectId,
  688. this.historyId,
  689. updates,
  690. this.extendLock
  691. )
  692. expect(expandedUpdates).to.deep.equal([])
  693. expect(this.extendLock).to.have.been.called
  694. })
  695. it('removes and re-adds binary files if they do not have same hash', async function () {
  696. const persistedFileWithNewContent = {
  697. _hash: 'anotherhashvalue',
  698. hello: 'world',
  699. path: '1.png',
  700. url: 'filestore-new-url',
  701. }
  702. const updates = [
  703. resyncProjectStructureUpdate(
  704. [this.persistedDoc],
  705. [persistedFileWithNewContent]
  706. ),
  707. ]
  708. const expandedUpdates =
  709. await this.SyncManager.promises.expandSyncUpdates(
  710. this.projectId,
  711. this.historyId,
  712. updates,
  713. this.extendLock
  714. )
  715. expect(expandedUpdates).to.deep.equal([
  716. {
  717. pathname: persistedFileWithNewContent.path,
  718. new_pathname: '',
  719. meta: {
  720. resync: true,
  721. ts: timestamp,
  722. origin: { kind: 'history-resync' },
  723. },
  724. },
  725. {
  726. pathname: persistedFileWithNewContent.path,
  727. file: persistedFileWithNewContent.file,
  728. url: persistedFileWithNewContent.url,
  729. meta: {
  730. resync: true,
  731. ts: timestamp,
  732. origin: { kind: 'history-resync' },
  733. },
  734. },
  735. ])
  736. expect(this.extendLock).to.have.been.called
  737. })
  738. it('preserves other updates', async function () {
  739. const update = { mock: 'update' }
  740. const updates = [
  741. update,
  742. resyncProjectStructureUpdate(
  743. [this.persistedDoc],
  744. [this.persistedFile]
  745. ),
  746. ]
  747. const expandedUpdates =
  748. await this.SyncManager.promises.expandSyncUpdates(
  749. this.projectId,
  750. this.historyId,
  751. updates,
  752. this.extendLock
  753. )
  754. expect(expandedUpdates).to.deep.equal([update])
  755. expect(this.extendLock).to.have.been.called
  756. })
  757. })
  758. describe('expanding doc contents sync updates', function () {
  759. it('returns errors from diffAsShareJsOps', async function () {
  760. const diffError = new Error('test')
  761. this.UpdateCompressor.diffAsShareJsOps.throws(diffError)
  762. const updates = [
  763. resyncProjectStructureUpdate(
  764. [this.persistedDoc],
  765. [this.persistedFile]
  766. ),
  767. docContentSyncUpdate(this.persistedDoc, this.persistedDocContent),
  768. ]
  769. await expect(
  770. this.SyncManager.promises.expandSyncUpdates(
  771. this.projectId,
  772. this.historyId,
  773. updates,
  774. this.extendLock
  775. )
  776. ).to.be.rejectedWith(diffError)
  777. expect(this.extendLock).to.have.been.called
  778. })
  779. it('handles an update for a file that is missing from the snapshot', async function () {
  780. const updates = [docContentSyncUpdate('not-in-snapshot.txt', 'test')]
  781. await expect(
  782. this.SyncManager.promises.expandSyncUpdates(
  783. this.projectId,
  784. this.historyId,
  785. updates,
  786. this.extendLock
  787. )
  788. ).to.be.rejectedWith('unrecognised file: not in snapshot')
  789. })
  790. it('queues nothing for in docs whose contents is in sync', async function () {
  791. const updates = [
  792. resyncProjectStructureUpdate(
  793. [this.persistedDoc],
  794. [this.persistedFile]
  795. ),
  796. docContentSyncUpdate(this.persistedDoc, this.persistedDocContent),
  797. ]
  798. this.UpdateCompressor.diffAsShareJsOps.returns([])
  799. const expandedUpdates =
  800. await this.SyncManager.promises.expandSyncUpdates(
  801. this.projectId,
  802. this.historyId,
  803. updates,
  804. this.extendLock
  805. )
  806. expect(expandedUpdates).to.deep.equal([])
  807. expect(this.extendLock).to.have.been.called
  808. })
  809. it('queues text updates for docs whose contents is out of sync', async function () {
  810. const updates = [
  811. resyncProjectStructureUpdate(
  812. [this.persistedDoc],
  813. [this.persistedFile]
  814. ),
  815. docContentSyncUpdate(
  816. this.persistedDoc,
  817. 'the fox jumps over the lazy dog'
  818. ),
  819. ]
  820. this.UpdateCompressor.diffAsShareJsOps.returns([
  821. { d: 'quick brown ', p: 4 },
  822. ])
  823. const expandedUpdates =
  824. await this.SyncManager.promises.expandSyncUpdates(
  825. this.projectId,
  826. this.historyId,
  827. updates,
  828. this.extendLock
  829. )
  830. expect(expandedUpdates).to.deep.equal([
  831. {
  832. doc: this.persistedDoc.doc,
  833. op: [{ d: 'quick brown ', p: 4 }],
  834. meta: {
  835. pathname: this.persistedDoc.path,
  836. doc_length: this.persistedDocContent.length,
  837. resync: true,
  838. ts: timestamp,
  839. origin: { kind: 'history-resync' },
  840. },
  841. },
  842. ])
  843. expect(this.extendLock).to.have.been.called
  844. })
  845. it('queues text updates for docs created by project structure sync', async function () {
  846. this.UpdateCompressor.diffAsShareJsOps.returns([{ i: 'a', p: 0 }])
  847. const newDoc = {
  848. path: 'another.tex',
  849. doc: new ObjectId().toString(),
  850. }
  851. const newDocContent = 'a'
  852. const updates = [
  853. resyncProjectStructureUpdate(
  854. [this.persistedDoc, newDoc],
  855. [this.persistedFile]
  856. ),
  857. docContentSyncUpdate(newDoc, newDocContent),
  858. ]
  859. const expandedUpdates =
  860. await this.SyncManager.promises.expandSyncUpdates(
  861. this.projectId,
  862. this.historyId,
  863. updates,
  864. this.extendLock
  865. )
  866. expect(expandedUpdates).to.deep.equal([
  867. {
  868. pathname: newDoc.path,
  869. doc: newDoc.doc,
  870. docLines: '',
  871. meta: {
  872. resync: true,
  873. ts: timestamp,
  874. origin: { kind: 'history-resync' },
  875. },
  876. },
  877. {
  878. doc: newDoc.doc,
  879. op: [{ i: 'a', p: 0 }],
  880. meta: {
  881. pathname: newDoc.path,
  882. doc_length: 0,
  883. resync: true,
  884. ts: timestamp,
  885. origin: { kind: 'history-resync' },
  886. },
  887. },
  888. ])
  889. expect(this.extendLock).to.have.been.called
  890. })
  891. it('skips text updates for docs when hashes match', async function () {
  892. this.fileMap['main.tex'].getHash.returns('special-hash')
  893. this.HashManager._getBlobHashFromString.returns('special-hash')
  894. const updates = [
  895. resyncProjectStructureUpdate(
  896. [this.persistedDoc],
  897. [this.persistedFile]
  898. ),
  899. docContentSyncUpdate(this.persistedDoc, 'hello'),
  900. ]
  901. const expandedUpdates =
  902. await this.SyncManager.promises.expandSyncUpdates(
  903. this.projectId,
  904. this.historyId,
  905. updates,
  906. this.extendLock
  907. )
  908. expect(expandedUpdates).to.deep.equal([])
  909. expect(this.extendLock).to.have.been.called
  910. })
  911. it('computes text updates for docs when hashes differ', async function () {
  912. this.fileMap['main.tex'].getHash.returns('first-hash')
  913. this.HashManager._getBlobHashFromString.returns('second-hash')
  914. this.UpdateCompressor.diffAsShareJsOps.returns([
  915. { i: 'test diff', p: 0 },
  916. ])
  917. const updates = [
  918. resyncProjectStructureUpdate(
  919. [this.persistedDoc],
  920. [this.persistedFile]
  921. ),
  922. docContentSyncUpdate(this.persistedDoc, 'hello'),
  923. ]
  924. const expandedUpdates =
  925. await this.SyncManager.promises.expandSyncUpdates(
  926. this.projectId,
  927. this.historyId,
  928. updates,
  929. this.extendLock
  930. )
  931. expect(expandedUpdates).to.deep.equal([
  932. {
  933. doc: this.persistedDoc.doc,
  934. op: [{ i: 'test diff', p: 0 }],
  935. meta: {
  936. pathname: this.persistedDoc.path,
  937. doc_length: this.persistedDocContent.length,
  938. resync: true,
  939. ts: timestamp,
  940. origin: { kind: 'history-resync' },
  941. },
  942. },
  943. ])
  944. expect(this.extendLock).to.have.been.called
  945. })
  946. describe('for docs whose contents is out of sync', function () {
  947. beforeEach(async function () {
  948. const updates = [
  949. resyncProjectStructureUpdate(
  950. [this.persistedDoc],
  951. [this.persistedFile]
  952. ),
  953. docContentSyncUpdate(this.persistedDoc, 'a'),
  954. ]
  955. this.loadedSnapshotDoc.getContent.returns('stored content')
  956. this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
  957. this.expandedUpdates =
  958. await this.SyncManager.promises.expandSyncUpdates(
  959. this.projectId,
  960. this.historyId,
  961. updates,
  962. this.extendLock
  963. )
  964. })
  965. it('loads content from the history service when needed', function () {
  966. expect(this.expandedUpdates).to.deep.equal([
  967. {
  968. doc: this.persistedDoc.doc,
  969. op: [{ d: 'sdf', p: 1 }],
  970. meta: {
  971. pathname: this.persistedDoc.path,
  972. doc_length: 'stored content'.length,
  973. resync: true,
  974. ts: timestamp,
  975. origin: { kind: 'history-resync' },
  976. },
  977. },
  978. ])
  979. expect(this.extendLock).to.have.been.called
  980. })
  981. })
  982. })
  983. describe('syncing comments', function () {
  984. beforeEach(function () {
  985. this.loadedSnapshotDoc.getComments.returns({
  986. toArray: sinon
  987. .stub()
  988. .returns([
  989. new Comment('comment1', [new Range(4, 5)]),
  990. new Comment('comment2', [new Range(10, 5)], true),
  991. ]),
  992. })
  993. this.comments = [
  994. makeComment('comment1', 4, 'quick'),
  995. makeComment('comment2', 10, 'brown'),
  996. ]
  997. this.resolvedComments = ['comment2']
  998. })
  999. it('does nothing if comments have not changed', async function () {
  1000. const updates = [
  1001. docContentSyncUpdate(
  1002. this.persistedDoc,
  1003. this.persistedDocContent,
  1004. {
  1005. comments: this.comments,
  1006. },
  1007. this.resolvedComments
  1008. ),
  1009. ]
  1010. const expandedUpdates =
  1011. await this.SyncManager.promises.expandSyncUpdates(
  1012. this.projectId,
  1013. this.historyId,
  1014. updates,
  1015. this.extendLock
  1016. )
  1017. expect(expandedUpdates).to.deep.equal([])
  1018. })
  1019. it('adds missing comments', async function () {
  1020. this.comments.push(makeComment('comment3', 20, 'jumps'))
  1021. const updates = [
  1022. docContentSyncUpdate(
  1023. this.persistedDoc,
  1024. this.persistedDocContent,
  1025. {
  1026. comments: this.comments,
  1027. },
  1028. this.resolvedComments
  1029. ),
  1030. ]
  1031. const expandedUpdates =
  1032. await this.SyncManager.promises.expandSyncUpdates(
  1033. this.projectId,
  1034. this.historyId,
  1035. updates,
  1036. this.extendLock
  1037. )
  1038. expect(expandedUpdates).to.deep.equal([
  1039. {
  1040. doc: this.persistedDoc.doc,
  1041. op: [
  1042. {
  1043. c: 'jumps',
  1044. p: 20,
  1045. t: 'comment3',
  1046. },
  1047. ],
  1048. meta: {
  1049. origin: {
  1050. kind: 'history-resync',
  1051. },
  1052. pathname: this.persistedDoc.path,
  1053. resync: true,
  1054. ts: timestamp,
  1055. doc_length: this.persistedDocContent.length,
  1056. },
  1057. },
  1058. ])
  1059. })
  1060. it('deletes extra comments', async function () {
  1061. this.comments.splice(0, 1)
  1062. const updates = [
  1063. docContentSyncUpdate(
  1064. this.persistedDoc,
  1065. this.persistedDocContent,
  1066. {
  1067. comments: this.comments,
  1068. },
  1069. this.resolvedComments
  1070. ),
  1071. ]
  1072. const expandedUpdates =
  1073. await this.SyncManager.promises.expandSyncUpdates(
  1074. this.projectId,
  1075. this.historyId,
  1076. updates,
  1077. this.extendLock
  1078. )
  1079. expect(expandedUpdates).to.deep.equal([
  1080. {
  1081. pathname: this.persistedDoc.path,
  1082. deleteComment: 'comment1',
  1083. meta: {
  1084. origin: {
  1085. kind: 'history-resync',
  1086. },
  1087. resync: true,
  1088. ts: timestamp,
  1089. },
  1090. },
  1091. ])
  1092. })
  1093. it('updates comments when ranges differ', async function () {
  1094. this.comments[1] = makeComment('comment2', 16, 'fox')
  1095. const updates = [
  1096. docContentSyncUpdate(
  1097. this.persistedDoc,
  1098. this.persistedDocContent,
  1099. {
  1100. comments: this.comments,
  1101. },
  1102. this.resolvedComments
  1103. ),
  1104. ]
  1105. const expandedUpdates =
  1106. await this.SyncManager.promises.expandSyncUpdates(
  1107. this.projectId,
  1108. this.historyId,
  1109. updates,
  1110. this.extendLock
  1111. )
  1112. expect(expandedUpdates).to.deep.equal([
  1113. {
  1114. doc: 'doc-id',
  1115. op: [
  1116. {
  1117. c: 'fox',
  1118. p: 16,
  1119. t: 'comment2',
  1120. },
  1121. ],
  1122. meta: {
  1123. origin: {
  1124. kind: 'history-resync',
  1125. },
  1126. resync: true,
  1127. ts: timestamp,
  1128. pathname: this.persistedDoc.path,
  1129. doc_length: this.persistedDocContent.length,
  1130. },
  1131. },
  1132. ])
  1133. })
  1134. it('sets the resolved state when it differs', async function () {
  1135. this.resolvedComments = ['comment1']
  1136. const updates = [
  1137. docContentSyncUpdate(
  1138. this.persistedDoc,
  1139. this.persistedDocContent,
  1140. {
  1141. comments: this.comments,
  1142. },
  1143. this.resolvedComments
  1144. ),
  1145. ]
  1146. const expandedUpdates =
  1147. await this.SyncManager.promises.expandSyncUpdates(
  1148. this.projectId,
  1149. this.historyId,
  1150. updates,
  1151. this.extendLock
  1152. )
  1153. expect(expandedUpdates).to.have.deep.members([
  1154. {
  1155. pathname: this.persistedDoc.path,
  1156. commentId: 'comment1',
  1157. resolved: true,
  1158. },
  1159. {
  1160. pathname: this.persistedDoc.path,
  1161. commentId: 'comment2',
  1162. resolved: false,
  1163. },
  1164. ])
  1165. })
  1166. })
  1167. })
  1168. })