SendingUpdatesTests.js 56 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117
  1. import { expect } from 'chai'
  2. import Settings from '@overleaf/settings'
  3. import assert from 'assert'
  4. import async from 'async'
  5. import crypto from 'crypto'
  6. import mongodb from 'mongodb-legacy'
  7. import nock from 'nock'
  8. import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
  9. import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
  10. const { ObjectId } = mongodb
  11. const MockHistoryStore = () => nock('http://127.0.0.1:3100')
  12. const MockFileStore = () => nock('http://127.0.0.1:3009')
  13. const MockWeb = () => nock('http://127.0.0.1:3000')
  14. // Some helper methods to make the tests more compact
  15. function slTextUpdate(historyId, doc, userId, v, ts, op) {
  16. return {
  17. projectHistoryId: historyId,
  18. doc: doc.id,
  19. op,
  20. v,
  21. meta: {
  22. user_id: userId,
  23. ts: ts.getTime(),
  24. pathname: doc.pathname,
  25. doc_length: doc.length,
  26. },
  27. }
  28. }
  29. function slAddDocUpdate(historyId, doc, userId, ts, docLines, ranges = {}) {
  30. return {
  31. projectHistoryId: historyId,
  32. pathname: doc.pathname,
  33. ranges,
  34. docLines,
  35. doc: doc.id,
  36. meta: { user_id: userId, ts: ts.getTime() },
  37. }
  38. }
  39. function slAddDocUpdateWithVersion(
  40. historyId,
  41. doc,
  42. userId,
  43. ts,
  44. docLines,
  45. projectVersion,
  46. ranges = {}
  47. ) {
  48. const result = slAddDocUpdate(historyId, doc, userId, ts, docLines, ranges)
  49. result.version = projectVersion
  50. return result
  51. }
  52. function slAddFileUpdate(historyId, file, userId, ts, projectId) {
  53. return {
  54. projectHistoryId: historyId,
  55. pathname: file.pathname,
  56. url: `http://127.0.0.1:3009/project/${projectId}/file/${file.id}`,
  57. file: file.id,
  58. ranges: undefined,
  59. meta: { user_id: userId, ts: ts.getTime() },
  60. }
  61. }
  62. function slRenameUpdate(historyId, doc, userId, ts, pathname, newPathname) {
  63. return {
  64. projectHistoryId: historyId,
  65. pathname,
  66. new_pathname: newPathname,
  67. doc: doc.id,
  68. meta: { user_id: userId, ts: ts.getTime() },
  69. }
  70. }
  71. function olUpdate(doc, userId, ts, operations, v) {
  72. return {
  73. v2Authors: [userId],
  74. timestamp: ts.toJSON(),
  75. authors: [],
  76. operations,
  77. v2DocVersions: {
  78. [doc.id]: {
  79. pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
  80. v: v || 1,
  81. },
  82. },
  83. }
  84. }
  85. function olTextOperation(doc, textOperation) {
  86. return {
  87. pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
  88. textOperation,
  89. }
  90. }
  91. function olAddCommentOperation(doc, commentId, pos, length) {
  92. return {
  93. pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
  94. commentId,
  95. ranges: [{ pos, length }],
  96. }
  97. }
  98. function olTextUpdate(doc, userId, ts, textOperation, v) {
  99. return olUpdate(doc, userId, ts, [olTextOperation(doc, textOperation)], v)
  100. }
  101. function olTextUpdates(doc, userId, ts, textOperations, v) {
  102. return olUpdate(
  103. doc,
  104. userId,
  105. ts,
  106. textOperations.map(textOperation => olTextOperation(doc, textOperation)),
  107. v
  108. )
  109. }
  110. function olRenameUpdate(doc, userId, ts, pathname, newPathname) {
  111. return {
  112. v2Authors: [userId],
  113. timestamp: ts.toJSON(),
  114. authors: [],
  115. operations: [
  116. {
  117. pathname,
  118. newPathname,
  119. },
  120. ],
  121. }
  122. }
  123. function olAddDocUpdate(doc, userId, ts, fileHash, rangesHash = undefined) {
  124. const update = {
  125. v2Authors: [userId],
  126. timestamp: ts.toJSON(),
  127. authors: [],
  128. operations: [
  129. {
  130. pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
  131. file: {
  132. hash: fileHash,
  133. },
  134. },
  135. ],
  136. }
  137. if (rangesHash) {
  138. update.operations[0].file.rangesHash = rangesHash
  139. }
  140. return update
  141. }
  142. function olAddDocUpdateWithVersion(
  143. doc,
  144. userId,
  145. ts,
  146. fileHash,
  147. version,
  148. rangesHash = undefined
  149. ) {
  150. const result = olAddDocUpdate(doc, userId, ts, fileHash, rangesHash)
  151. result.projectVersion = version
  152. return result
  153. }
  154. function olAddFileUpdate(file, userId, ts, fileHash) {
  155. return {
  156. v2Authors: [userId],
  157. timestamp: ts.toJSON(),
  158. authors: [],
  159. operations: [
  160. {
  161. pathname: file.pathname.replace(/^\//, ''), // Strip leading /
  162. file: {
  163. hash: fileHash,
  164. },
  165. },
  166. ],
  167. }
  168. }
  169. describe('Sending Updates', function () {
  170. const historyId = new ObjectId().toString()
  171. beforeEach(function (done) {
  172. this.timestamp = new Date()
  173. ProjectHistoryApp.ensureRunning(error => {
  174. if (error) {
  175. return done(error)
  176. }
  177. this.userId = new ObjectId().toString()
  178. this.projectId = new ObjectId().toString()
  179. this.docId = new ObjectId().toString()
  180. this.doc = {
  181. id: this.docId,
  182. pathname: '/main.tex',
  183. length: 5,
  184. }
  185. MockHistoryStore().post('/api/projects').reply(200, {
  186. projectId: historyId,
  187. })
  188. MockWeb()
  189. .get(`/project/${this.projectId}/details`)
  190. .reply(200, {
  191. name: 'Test Project',
  192. overleaf: {
  193. history: {
  194. id: historyId,
  195. },
  196. },
  197. })
  198. ProjectHistoryClient.initializeProject(historyId, done)
  199. })
  200. })
  201. afterEach(function () {
  202. nock.cleanAll()
  203. })
  204. describe('basic update types', function () {
  205. beforeEach(function () {
  206. MockHistoryStore()
  207. .get(`/api/projects/${historyId}/latest/history`)
  208. .reply(200, {
  209. chunk: {
  210. startVersion: 0,
  211. history: {
  212. snapshot: {},
  213. changes: [],
  214. },
  215. },
  216. })
  217. })
  218. it('should send add doc updates to the history store', function (done) {
  219. const fileHash = '0a207c060e61f3b88eaee0a8cd0696f46fb155eb'
  220. const createBlob = MockHistoryStore()
  221. .put(`/api/projects/${historyId}/blobs/${fileHash}`, 'a\nb')
  222. .reply(201)
  223. const addFile = MockHistoryStore()
  224. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  225. expect(body).to.deep.equal([
  226. olAddDocUpdate(this.doc, this.userId, this.timestamp, fileHash),
  227. ])
  228. return true
  229. })
  230. .query({ end_version: 0 })
  231. .reply(204)
  232. async.series(
  233. [
  234. cb => {
  235. ProjectHistoryClient.pushRawUpdate(
  236. this.projectId,
  237. slAddDocUpdate(
  238. historyId,
  239. this.doc,
  240. this.userId,
  241. this.timestamp,
  242. 'a\nb'
  243. ),
  244. cb
  245. )
  246. },
  247. cb => {
  248. ProjectHistoryClient.flushProject(this.projectId, cb)
  249. },
  250. ],
  251. error => {
  252. if (error) {
  253. return done(error)
  254. }
  255. assert(
  256. createBlob.isDone(),
  257. '/api/projects/:historyId/blobs/:hash should have been called'
  258. )
  259. assert(
  260. addFile.isDone(),
  261. `/api/projects/${historyId}/changes should have been called`
  262. )
  263. done()
  264. }
  265. )
  266. })
  267. it('should send ranges to the history store', function (done) {
  268. const fileHash = '49e886093b3eacbc12b99a1eb5aeaa44a6b9d90e'
  269. const rangesHash = 'fa9a429ff518bc9e5b2507a96ff0646b566eca65'
  270. const historyRanges = {
  271. trackedChanges: [
  272. {
  273. range: { pos: 4, length: 3 },
  274. tracking: {
  275. type: 'delete',
  276. userId: 'user-id-1',
  277. ts: '2024-01-01T00:00:00.000Z',
  278. },
  279. },
  280. ],
  281. comments: [
  282. {
  283. ranges: [{ pos: 0, length: 3 }],
  284. id: 'comment-id-1',
  285. },
  286. ],
  287. }
  288. // We need to set up the ranges mock first, as we will call it last..
  289. const createRangesBlob = MockHistoryStore()
  290. .put(`/api/projects/${historyId}/blobs/${rangesHash}`, historyRanges)
  291. .reply(201)
  292. const createBlob = MockHistoryStore()
  293. .put(`/api/projects/${historyId}/blobs/${fileHash}`, 'foo barbaz')
  294. .reply(201)
  295. const addFile = MockHistoryStore()
  296. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  297. expect(body).to.deep.equal([
  298. olAddDocUpdate(
  299. this.doc,
  300. this.userId,
  301. this.timestamp,
  302. fileHash,
  303. rangesHash
  304. ),
  305. ])
  306. return true
  307. })
  308. .query({ end_version: 0 })
  309. .reply(204)
  310. async.series(
  311. [
  312. cb => {
  313. ProjectHistoryClient.pushRawUpdate(
  314. this.projectId,
  315. slAddDocUpdate(
  316. historyId,
  317. this.doc,
  318. this.userId,
  319. this.timestamp,
  320. 'foo barbaz',
  321. {
  322. changes: [
  323. {
  324. op: { p: 4, d: 'bar' },
  325. metadata: {
  326. ts: 1704067200000,
  327. user_id: 'user-id-1',
  328. },
  329. },
  330. ],
  331. comments: [
  332. {
  333. op: {
  334. p: 0,
  335. c: 'foo',
  336. t: 'comment-id-1',
  337. },
  338. metadata: { resolved: false },
  339. },
  340. ],
  341. }
  342. ),
  343. cb
  344. )
  345. },
  346. cb => {
  347. ProjectHistoryClient.flushProject(this.projectId, cb)
  348. },
  349. ],
  350. error => {
  351. if (error) {
  352. return done(error)
  353. }
  354. assert(
  355. createBlob.isDone(),
  356. '/api/projects/:historyId/blobs/:hash should have been called to create content blob'
  357. )
  358. assert(
  359. createRangesBlob.isDone(),
  360. '/api/projects/:historyId/blobs/:hash should have been called to create ranges blob'
  361. )
  362. assert(
  363. addFile.isDone(),
  364. `/api/projects/${historyId}/changes should have been called`
  365. )
  366. done()
  367. }
  368. )
  369. })
  370. it('should strip non-BMP characters in add doc updates before sending to the history store', function (done) {
  371. const fileHash = '11509fe05a41f9cdc51ea081342b5a4fc7c8d0fc'
  372. const createBlob = MockHistoryStore()
  373. .put(
  374. `/api/projects/${historyId}/blobs/${fileHash}`,
  375. 'a\nb\uFFFD\uFFFDc'
  376. )
  377. .reply(201)
  378. const addFile = MockHistoryStore()
  379. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  380. expect(body).to.deep.equal([
  381. olAddDocUpdate(this.doc, this.userId, this.timestamp, fileHash),
  382. ])
  383. return true
  384. })
  385. .query({ end_version: 0 })
  386. .reply(204)
  387. async.series(
  388. [
  389. cb => {
  390. ProjectHistoryClient.pushRawUpdate(
  391. this.projectId,
  392. slAddDocUpdate(
  393. historyId,
  394. this.doc,
  395. this.userId,
  396. this.timestamp,
  397. 'a\nb\uD800\uDC00c'
  398. ),
  399. cb
  400. )
  401. },
  402. cb => {
  403. ProjectHistoryClient.flushProject(this.projectId, cb)
  404. },
  405. ],
  406. error => {
  407. if (error) {
  408. return done(error)
  409. }
  410. assert(
  411. createBlob.isDone(),
  412. '/api/projects/:historyId/blobs/:hash should have been called'
  413. )
  414. assert(
  415. addFile.isDone(),
  416. `/api/projects/${historyId}/changes should have been called`
  417. )
  418. done()
  419. }
  420. )
  421. })
  422. it('should send text updates to the history store', function (done) {
  423. const createChange = MockHistoryStore()
  424. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  425. expect(body).to.deep.equal([
  426. olTextUpdate(this.doc, this.userId, this.timestamp, [3, '\nc', 2]),
  427. ])
  428. return true
  429. })
  430. .query({ end_version: 0 })
  431. .reply(204)
  432. async.series(
  433. [
  434. cb => {
  435. ProjectHistoryClient.pushRawUpdate(
  436. this.projectId,
  437. slTextUpdate(
  438. historyId,
  439. this.doc,
  440. this.userId,
  441. 1,
  442. this.timestamp,
  443. [{ p: 3, i: '\nc' }]
  444. ),
  445. cb
  446. )
  447. },
  448. cb => {
  449. ProjectHistoryClient.flushProject(this.projectId, cb)
  450. },
  451. ],
  452. error => {
  453. if (error) {
  454. return done(error)
  455. }
  456. assert(
  457. createChange.isDone(),
  458. `/api/projects/${historyId}/changes should have been called`
  459. )
  460. done()
  461. }
  462. )
  463. })
  464. it('should send renames to the history store', function (done) {
  465. const createChange = MockHistoryStore()
  466. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  467. expect(body).to.deep.equal([
  468. olRenameUpdate(
  469. this.doc,
  470. this.userId,
  471. this.timestamp,
  472. 'main.tex',
  473. 'main2.tex'
  474. ),
  475. ])
  476. return true
  477. })
  478. .query({ end_version: 0 })
  479. .reply(204)
  480. async.series(
  481. [
  482. cb => {
  483. ProjectHistoryClient.pushRawUpdate(
  484. this.projectId,
  485. slRenameUpdate(
  486. historyId,
  487. this.doc,
  488. this.userId,
  489. this.timestamp,
  490. '/main.tex',
  491. '/main2.tex'
  492. ),
  493. cb
  494. )
  495. },
  496. cb => {
  497. ProjectHistoryClient.flushProject(this.projectId, cb)
  498. },
  499. ],
  500. error => {
  501. if (error) {
  502. return done(error)
  503. }
  504. assert(
  505. createChange.isDone(),
  506. `/api/projects/${historyId}/changes should have been called`
  507. )
  508. done()
  509. }
  510. )
  511. })
  512. it('should send add file updates to the history store', function (done) {
  513. const file = {
  514. id: new ObjectId().toString(),
  515. pathname: '/test.png',
  516. contents: Buffer.from([1, 2, 3]),
  517. hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
  518. }
  519. const fileStoreRequest = MockFileStore()
  520. .get(`/project/${this.projectId}/file/${file.id}`)
  521. .reply(200, file.contents)
  522. const createBlob = MockHistoryStore()
  523. .put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
  524. .reply(201)
  525. const addFile = MockHistoryStore()
  526. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  527. expect(body).to.deep.equal([
  528. olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
  529. ])
  530. return true
  531. })
  532. .query({ end_version: 0 })
  533. .reply(204)
  534. async.series(
  535. [
  536. cb => {
  537. ProjectHistoryClient.pushRawUpdate(
  538. this.projectId,
  539. slAddFileUpdate(
  540. historyId,
  541. file,
  542. this.userId,
  543. this.timestamp,
  544. this.projectId
  545. ),
  546. cb
  547. )
  548. },
  549. cb => {
  550. ProjectHistoryClient.flushProject(this.projectId, cb)
  551. },
  552. ],
  553. error => {
  554. if (error) {
  555. return done(error)
  556. }
  557. assert(
  558. fileStoreRequest.isDone(),
  559. `/project/${this.projectId}/file/${file.id} should have been called`
  560. )
  561. assert(
  562. createBlob.isDone(),
  563. `/api/projects/${historyId}/latest/files should have been called`
  564. )
  565. assert(
  566. addFile.isDone(),
  567. `/api/projects/${historyId}/latest/files should have been called`
  568. )
  569. done()
  570. }
  571. )
  572. })
  573. it('should send a stub to the history store when the file is large', function (done) {
  574. const fileContents = Buffer.alloc(Settings.maxFileSizeInBytes + 1, 'X')
  575. const fileSize = Buffer.byteLength(fileContents)
  576. const fileHash = crypto
  577. .createHash('sha1')
  578. .update('blob ' + fileSize + '\x00')
  579. .update(fileContents, 'utf8')
  580. .digest('hex')
  581. const file = {
  582. id: new ObjectId().toString(),
  583. pathname: '/large.png',
  584. contents: fileContents,
  585. hash: fileHash,
  586. }
  587. const stubContents = [
  588. 'FileTooLargeError v1',
  589. 'File too large to be stored in history service',
  590. `id project-${this.projectId}-file-${file.id}`,
  591. `size ${fileSize} bytes`,
  592. `hash ${fileHash}`,
  593. '\0', // null byte to make this a binary file
  594. ].join('\n')
  595. const stubHash = crypto
  596. .createHash('sha1')
  597. .update('blob ' + Buffer.byteLength(stubContents) + '\x00')
  598. .update(stubContents, 'utf8')
  599. .digest('hex')
  600. const stub = {
  601. id: file.id,
  602. pathname: file.pathname,
  603. contents: stubContents,
  604. hash: stubHash,
  605. }
  606. const fileStoreRequest = MockFileStore()
  607. .get(`/project/${this.projectId}/file/${file.id}`)
  608. .reply(200, file.contents)
  609. const createBlob = MockHistoryStore()
  610. .put(`/api/projects/${historyId}/blobs/${stub.hash}`, stub.contents)
  611. .reply(201)
  612. const addFile = MockHistoryStore()
  613. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  614. expect(body).to.deep.equal([
  615. olAddFileUpdate(stub, this.userId, this.timestamp, stub.hash),
  616. ])
  617. return true
  618. })
  619. .query({ end_version: 0 })
  620. .reply(204)
  621. async.series(
  622. [
  623. cb => {
  624. ProjectHistoryClient.pushRawUpdate(
  625. this.projectId,
  626. slAddFileUpdate(
  627. historyId,
  628. file,
  629. this.userId,
  630. this.timestamp,
  631. this.projectId
  632. ),
  633. cb
  634. )
  635. },
  636. cb => {
  637. ProjectHistoryClient.flushProject(this.projectId, cb)
  638. },
  639. ],
  640. error => {
  641. if (error) {
  642. return done(error)
  643. }
  644. assert(
  645. addFile.isDone(),
  646. `/api/projects/${historyId}/latest/files should have been called`
  647. )
  648. assert(
  649. createBlob.isDone(),
  650. `/api/projects/${historyId}/latest/files should have been called`
  651. )
  652. assert(
  653. fileStoreRequest.isDone(),
  654. `/project/${this.projectId}/file/${file.id} should have been called`
  655. )
  656. done()
  657. }
  658. )
  659. })
  660. it('should handle comment ops', function (done) {
  661. const createChange = MockHistoryStore()
  662. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  663. expect(body).to.deep.equal([
  664. olUpdate(this.doc, this.userId, this.timestamp, [
  665. olTextOperation(this.doc, [3, '\nc', 2]),
  666. olAddCommentOperation(this.doc, 'comment-id-1', 3, 2),
  667. ]),
  668. olUpdate(
  669. this.doc,
  670. this.userId,
  671. this.timestamp,
  672. [olAddCommentOperation(this.doc, 'comment-id-2', 2, 1)],
  673. 2
  674. ),
  675. ])
  676. return true
  677. })
  678. .query({ end_version: 0 })
  679. .reply(204)
  680. async.series(
  681. [
  682. cb => {
  683. ProjectHistoryClient.pushRawUpdate(
  684. this.projectId,
  685. slTextUpdate(
  686. historyId,
  687. this.doc,
  688. this.userId,
  689. 1,
  690. this.timestamp,
  691. [
  692. { p: 3, i: '\nc' },
  693. { p: 3, c: '\nc', t: 'comment-id-1' },
  694. ]
  695. ),
  696. cb
  697. )
  698. },
  699. cb => {
  700. ProjectHistoryClient.pushRawUpdate(
  701. this.projectId,
  702. slTextUpdate(
  703. historyId,
  704. this.doc,
  705. this.userId,
  706. 2,
  707. this.timestamp,
  708. [{ p: 2, c: 'b', t: 'comment-id-2' }]
  709. ),
  710. cb
  711. )
  712. },
  713. cb => {
  714. ProjectHistoryClient.flushProject(this.projectId, cb)
  715. },
  716. ],
  717. error => {
  718. if (error) {
  719. return done(error)
  720. }
  721. assert(
  722. createChange.isDone(),
  723. `/api/projects/${historyId}/changes should have been called`
  724. )
  725. done()
  726. }
  727. )
  728. })
  729. it('should be able to process lots of updates in batches', function (done) {
  730. const BATCH_SIZE = 500
  731. const createFirstChangeBatch = MockHistoryStore()
  732. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  733. expect(body).to.deep.equal([
  734. olTextUpdate(
  735. this.doc,
  736. this.userId,
  737. this.timestamp,
  738. ['a'.repeat(BATCH_SIZE), 6],
  739. BATCH_SIZE - 1
  740. ),
  741. ])
  742. return true
  743. })
  744. .query({ end_version: 0 })
  745. .reply(204)
  746. const createSecondChangeBatch = MockHistoryStore()
  747. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  748. expect(body).to.deep.equal([
  749. olTextUpdate(
  750. this.doc,
  751. this.userId,
  752. this.timestamp,
  753. ['a'.repeat(50), BATCH_SIZE + 6],
  754. BATCH_SIZE - 1 + 50
  755. ),
  756. ])
  757. return true
  758. })
  759. .query({ end_version: 500 })
  760. .reply(204)
  761. // these need mocking again for the second batch
  762. MockHistoryStore()
  763. .get(`/api/projects/${historyId}/latest/history`)
  764. .reply(200, {
  765. chunk: {
  766. startVersion: BATCH_SIZE,
  767. history: {
  768. snapshot: {},
  769. changes: [],
  770. },
  771. },
  772. })
  773. MockWeb()
  774. .get(`/project/${this.projectId}/details`)
  775. .reply(200, {
  776. name: 'Test Project',
  777. overleaf: {
  778. history: {
  779. id: historyId,
  780. },
  781. },
  782. })
  783. const pushChange = (n, cb) => {
  784. this.doc.length += 1
  785. ProjectHistoryClient.pushRawUpdate(
  786. this.projectId,
  787. slTextUpdate(historyId, this.doc, this.userId, n, this.timestamp, [
  788. { p: 0, i: 'a' },
  789. ]),
  790. cb
  791. )
  792. }
  793. async.series(
  794. [
  795. cb => {
  796. async.times(BATCH_SIZE + 50, pushChange, cb)
  797. },
  798. cb => {
  799. ProjectHistoryClient.flushProject(this.projectId, cb)
  800. },
  801. ],
  802. error => {
  803. if (error) {
  804. return done(error)
  805. }
  806. assert(
  807. createFirstChangeBatch.isDone(),
  808. `/api/projects/${historyId}/changes should have been called for the first batch`
  809. )
  810. assert(
  811. createSecondChangeBatch.isDone(),
  812. `/api/projects/${historyId}/changes should have been called for the second batch`
  813. )
  814. done()
  815. }
  816. )
  817. })
  818. })
  819. describe('compressing updates', function () {
  820. beforeEach(function () {
  821. MockHistoryStore()
  822. .get(`/api/projects/${historyId}/latest/history`)
  823. .reply(200, {
  824. chunk: {
  825. startVersion: 0,
  826. history: {
  827. snapshot: {},
  828. changes: [],
  829. },
  830. },
  831. })
  832. })
  833. it('should concat adjacent text updates', function (done) {
  834. const createChange = MockHistoryStore()
  835. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  836. expect(body).to.deep.equal([
  837. olTextUpdate(
  838. this.doc,
  839. this.userId,
  840. this.timestamp,
  841. [3, 'foobaz', 2],
  842. 2
  843. ),
  844. ])
  845. return true
  846. })
  847. .query({ end_version: 0 })
  848. .reply(204)
  849. async.series(
  850. [
  851. cb => {
  852. ProjectHistoryClient.pushRawUpdate(
  853. this.projectId,
  854. slTextUpdate(
  855. historyId,
  856. this.doc,
  857. this.userId,
  858. 1,
  859. this.timestamp,
  860. [
  861. { p: 3, i: 'foobar' },
  862. { p: 6, d: 'bar' },
  863. ]
  864. ),
  865. cb
  866. )
  867. this.doc.length += 3
  868. },
  869. cb => {
  870. ProjectHistoryClient.pushRawUpdate(
  871. this.projectId,
  872. slTextUpdate(
  873. historyId,
  874. this.doc,
  875. this.userId,
  876. 2,
  877. this.timestamp,
  878. [{ p: 6, i: 'baz' }]
  879. ),
  880. cb
  881. )
  882. },
  883. cb => {
  884. ProjectHistoryClient.flushProject(this.projectId, cb)
  885. },
  886. ],
  887. error => {
  888. if (error) {
  889. return done(error)
  890. }
  891. assert(
  892. createChange.isDone(),
  893. `/api/projects/${historyId}/changes should have been called`
  894. )
  895. done()
  896. }
  897. )
  898. })
  899. it('should take the timestamp of the first update', function (done) {
  900. const timestamp1 = new Date(this.timestamp)
  901. const timestamp2 = new Date(this.timestamp.getTime() + 10000)
  902. const createChange = MockHistoryStore()
  903. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  904. expect(body).to.deep.equal([
  905. olTextUpdate(
  906. this.doc,
  907. this.userId,
  908. timestamp1,
  909. [3, 'foobaz', 2],
  910. 2
  911. ),
  912. ])
  913. return true
  914. })
  915. .query({ end_version: 0 })
  916. .reply(204)
  917. async.series(
  918. [
  919. cb => {
  920. ProjectHistoryClient.pushRawUpdate(
  921. this.projectId,
  922. slTextUpdate(historyId, this.doc, this.userId, 1, timestamp1, [
  923. { p: 3, i: 'foo' },
  924. ]),
  925. cb
  926. )
  927. this.doc.length += 3
  928. },
  929. cb => {
  930. ProjectHistoryClient.pushRawUpdate(
  931. this.projectId,
  932. slTextUpdate(historyId, this.doc, this.userId, 2, timestamp2, [
  933. { p: 6, i: 'baz' },
  934. ]),
  935. cb
  936. )
  937. },
  938. cb => {
  939. ProjectHistoryClient.flushProject(this.projectId, cb)
  940. },
  941. ],
  942. error => {
  943. if (error) {
  944. return done(error)
  945. }
  946. assert(
  947. createChange.isDone(),
  948. `/api/projects/${historyId}/changes should have been called`
  949. )
  950. done()
  951. }
  952. )
  953. })
  954. it('should not concat updates more than 60 seconds apart', function (done) {
  955. const timestamp1 = new Date(this.timestamp)
  956. const timestamp2 = new Date(this.timestamp.getTime() + 120000)
  957. const createChange = MockHistoryStore()
  958. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  959. expect(body).to.deep.equal([
  960. olTextUpdate(this.doc, this.userId, timestamp1, [3, 'foo', 2], 1),
  961. olTextUpdate(this.doc, this.userId, timestamp2, [6, 'baz', 2], 2),
  962. ])
  963. return true
  964. })
  965. .query({ end_version: 0 })
  966. .reply(204)
  967. async.series(
  968. [
  969. cb => {
  970. ProjectHistoryClient.pushRawUpdate(
  971. this.projectId,
  972. slTextUpdate(historyId, this.doc, this.userId, 1, timestamp1, [
  973. { p: 3, i: 'foo' },
  974. ]),
  975. cb
  976. )
  977. this.doc.length += 3
  978. },
  979. cb => {
  980. ProjectHistoryClient.pushRawUpdate(
  981. this.projectId,
  982. slTextUpdate(historyId, this.doc, this.userId, 2, timestamp2, [
  983. { p: 6, i: 'baz' },
  984. ]),
  985. cb
  986. )
  987. },
  988. cb => {
  989. ProjectHistoryClient.flushProject(this.projectId, cb)
  990. },
  991. ],
  992. error => {
  993. if (error) {
  994. return done(error)
  995. }
  996. assert(
  997. createChange.isDone(),
  998. `/api/projects/${historyId}/changes should have been called`
  999. )
  1000. done()
  1001. }
  1002. )
  1003. })
  1004. it('should not concat updates with different user_ids', function (done) {
  1005. const userId1 = new ObjectId().toString()
  1006. const userId2 = new ObjectId().toString()
  1007. const createChange = MockHistoryStore()
  1008. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1009. expect(body).to.deep.equal([
  1010. olTextUpdate(this.doc, userId1, this.timestamp, [3, 'foo', 2], 1),
  1011. olTextUpdate(this.doc, userId2, this.timestamp, [6, 'baz', 2], 2),
  1012. ])
  1013. return true
  1014. })
  1015. .query({ end_version: 0 })
  1016. .reply(204)
  1017. async.series(
  1018. [
  1019. cb => {
  1020. ProjectHistoryClient.pushRawUpdate(
  1021. this.projectId,
  1022. slTextUpdate(historyId, this.doc, userId1, 1, this.timestamp, [
  1023. { p: 3, i: 'foo' },
  1024. ]),
  1025. cb
  1026. )
  1027. this.doc.length += 3
  1028. },
  1029. cb => {
  1030. ProjectHistoryClient.pushRawUpdate(
  1031. this.projectId,
  1032. slTextUpdate(historyId, this.doc, userId2, 2, this.timestamp, [
  1033. { p: 6, i: 'baz' },
  1034. ]),
  1035. cb
  1036. )
  1037. },
  1038. cb => {
  1039. ProjectHistoryClient.flushProject(this.projectId, cb)
  1040. },
  1041. ],
  1042. error => {
  1043. if (error) {
  1044. return done(error)
  1045. }
  1046. assert(
  1047. createChange.isDone(),
  1048. `/api/projects/${historyId}/changes should have been called`
  1049. )
  1050. done()
  1051. }
  1052. )
  1053. })
  1054. it('should not concat updates with different docs', function (done) {
  1055. const doc1 = {
  1056. id: new ObjectId().toString(),
  1057. pathname: '/doc1.tex',
  1058. length: 10,
  1059. }
  1060. const doc2 = {
  1061. id: new ObjectId().toString(),
  1062. pathname: '/doc2.tex',
  1063. length: 10,
  1064. }
  1065. const createChange = MockHistoryStore()
  1066. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1067. expect(body).to.deep.equal([
  1068. olTextUpdate(doc1, this.userId, this.timestamp, [3, 'foo', 7], 1),
  1069. olTextUpdate(doc2, this.userId, this.timestamp, [6, 'baz', 4], 2),
  1070. ])
  1071. return true
  1072. })
  1073. .query({ end_version: 0 })
  1074. .reply(204)
  1075. async.series(
  1076. [
  1077. cb => {
  1078. ProjectHistoryClient.pushRawUpdate(
  1079. this.projectId,
  1080. slTextUpdate(historyId, doc1, this.userId, 1, this.timestamp, [
  1081. { p: 3, i: 'foo' },
  1082. ]),
  1083. cb
  1084. )
  1085. },
  1086. cb => {
  1087. ProjectHistoryClient.pushRawUpdate(
  1088. this.projectId,
  1089. slTextUpdate(historyId, doc2, this.userId, 2, this.timestamp, [
  1090. { p: 6, i: 'baz' },
  1091. ]),
  1092. cb
  1093. )
  1094. },
  1095. cb => {
  1096. ProjectHistoryClient.flushProject(this.projectId, cb)
  1097. },
  1098. ],
  1099. error => {
  1100. if (error) {
  1101. return done(error)
  1102. }
  1103. assert(
  1104. createChange.isDone(),
  1105. `/api/projects/${historyId}/changes should have been called`
  1106. )
  1107. done()
  1108. }
  1109. )
  1110. })
  1111. it('should not send updates without any ops', function (done) {
  1112. const createChange = MockHistoryStore()
  1113. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1114. expect(body).to.deep.equal([])
  1115. return true
  1116. })
  1117. .query({ end_version: 0 })
  1118. .reply(204)
  1119. async.series(
  1120. [
  1121. cb => {
  1122. // These blank ops can get sent by doc-updater on setDocs from Dropbox that don't change anything
  1123. ProjectHistoryClient.pushRawUpdate(
  1124. this.projectId,
  1125. slTextUpdate(
  1126. historyId,
  1127. this.doc,
  1128. this.userId,
  1129. 1,
  1130. this.timestamp,
  1131. []
  1132. ),
  1133. cb
  1134. )
  1135. },
  1136. cb => {
  1137. ProjectHistoryClient.flushProject(this.projectId, cb)
  1138. },
  1139. ],
  1140. error => {
  1141. if (error) {
  1142. return done(error)
  1143. }
  1144. assert(
  1145. !createChange.isDone(),
  1146. `/api/projects/${historyId}/changes should not have been called`
  1147. )
  1148. done()
  1149. }
  1150. )
  1151. })
  1152. it('should not send ops that compress to nothing', function (done) {
  1153. const createChange = MockHistoryStore()
  1154. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1155. expect(body).to.deep.equal([])
  1156. return true
  1157. })
  1158. .query({ end_version: 0 })
  1159. .reply(204)
  1160. async.series(
  1161. [
  1162. cb => {
  1163. ProjectHistoryClient.pushRawUpdate(
  1164. this.projectId,
  1165. slTextUpdate(
  1166. historyId,
  1167. this.doc,
  1168. this.userId,
  1169. 1,
  1170. this.timestamp,
  1171. [{ i: 'foo', p: 3 }]
  1172. ),
  1173. cb
  1174. )
  1175. this.doc.length += 3
  1176. },
  1177. cb => {
  1178. ProjectHistoryClient.pushRawUpdate(
  1179. this.projectId,
  1180. slTextUpdate(
  1181. historyId,
  1182. this.doc,
  1183. this.userId,
  1184. 2,
  1185. this.timestamp,
  1186. [{ d: 'foo', p: 3 }]
  1187. ),
  1188. cb
  1189. )
  1190. },
  1191. cb => {
  1192. ProjectHistoryClient.flushProject(this.projectId, cb)
  1193. },
  1194. ],
  1195. error => {
  1196. if (error) {
  1197. return done(error)
  1198. }
  1199. assert(
  1200. !createChange.isDone(),
  1201. `/api/projects/${historyId}/changes should not have been called`
  1202. )
  1203. done()
  1204. }
  1205. )
  1206. })
  1207. it('should not send ops from a diff that are blank', function (done) {
  1208. this.doc.length = 300
  1209. // Test case taken from a real life document where it was generating blank insert and
  1210. // delete ops from a diff, and the blank delete was erroring on the OL history from
  1211. // a text operation like [42, 0, 512], where the 0 was invalid.
  1212. const createChange = MockHistoryStore()
  1213. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1214. expect(body).to.deep.equal([
  1215. olTextUpdates(this.doc, this.userId, this.timestamp, [
  1216. [
  1217. 87,
  1218. -1,
  1219. 67,
  1220. '|l|ll|}\n\\hline',
  1221. -4,
  1222. 30,
  1223. ' \\hline',
  1224. 87,
  1225. ' \\\\ \\hline',
  1226. 24,
  1227. ],
  1228. ]),
  1229. ])
  1230. return true
  1231. })
  1232. .query({ end_version: 0 })
  1233. .reply(204)
  1234. async.series(
  1235. [
  1236. cb => {
  1237. ProjectHistoryClient.pushRawUpdate(
  1238. this.projectId,
  1239. slTextUpdate(
  1240. historyId,
  1241. this.doc,
  1242. this.userId,
  1243. 1,
  1244. this.timestamp,
  1245. [
  1246. {
  1247. p: 73,
  1248. d: '\\begin{table}[h]\n\\centering\n\\caption{My caption}\n\\label{my-label}\n\\begin{tabular}{lll}\n & A & B \\\\\nLiter t up & 2 & 1 \\\\\nLiter Whiskey & 1 & 2 \\\\\nPris pr. liter & 200 & 250\n\\end{tabular}\n\\end{table}',
  1249. },
  1250. {
  1251. p: 73,
  1252. i: '\\begin{table}[]\n\\centering\n\\caption{My caption}\n\\label{my-label}\n\\begin{tabular}{|l|ll|}\n\\hline\n & A & B \\\\ \\hline\nLiter t up & 2 & 1 \\\\\nLiter Whiskey & 1 & 2 \\\\\nPris pr. liter & 200 & 250 \\\\ \\hline\n\\end{tabular}\n\\end{table}',
  1253. },
  1254. ]
  1255. ),
  1256. cb
  1257. )
  1258. },
  1259. cb => {
  1260. ProjectHistoryClient.flushProject(this.projectId, cb)
  1261. },
  1262. ],
  1263. error => {
  1264. if (error) {
  1265. return done(error)
  1266. }
  1267. assert(
  1268. createChange.isDone(),
  1269. `/api/projects/${historyId}/changes should have been called`
  1270. )
  1271. done()
  1272. }
  1273. )
  1274. })
  1275. it('should not concat text updates across project structure ops', function (done) {
  1276. const newDoc = {
  1277. id: new ObjectId().toString(),
  1278. pathname: '/main.tex',
  1279. hash: '0a207c060e61f3b88eaee0a8cd0696f46fb155eb',
  1280. docLines: 'a\nb',
  1281. }
  1282. MockHistoryStore()
  1283. .put(`/api/projects/${historyId}/blobs/${newDoc.hash}`)
  1284. .reply(201)
  1285. const createChange = MockHistoryStore()
  1286. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1287. expect(body).to.deep.equal([
  1288. olTextUpdate(
  1289. this.doc,
  1290. this.userId,
  1291. this.timestamp,
  1292. [3, 'foo', 2],
  1293. 1
  1294. ),
  1295. olAddDocUpdate(newDoc, this.userId, this.timestamp, newDoc.hash),
  1296. olTextUpdate(
  1297. this.doc,
  1298. this.userId,
  1299. this.timestamp,
  1300. [6, 'baz', 2],
  1301. 2
  1302. ),
  1303. ])
  1304. return true
  1305. })
  1306. .query({ end_version: 0 })
  1307. .reply(204)
  1308. async.series(
  1309. [
  1310. cb => {
  1311. ProjectHistoryClient.pushRawUpdate(
  1312. this.projectId,
  1313. slTextUpdate(
  1314. historyId,
  1315. this.doc,
  1316. this.userId,
  1317. 1,
  1318. this.timestamp,
  1319. [
  1320. { p: 3, i: 'foobar' },
  1321. { p: 6, d: 'bar' },
  1322. ]
  1323. ),
  1324. cb
  1325. )
  1326. this.doc.length += 3
  1327. },
  1328. cb => {
  1329. ProjectHistoryClient.pushRawUpdate(
  1330. this.projectId,
  1331. slAddDocUpdate(
  1332. historyId,
  1333. newDoc,
  1334. this.userId,
  1335. this.timestamp,
  1336. newDoc.docLines
  1337. ),
  1338. cb
  1339. )
  1340. },
  1341. cb => {
  1342. ProjectHistoryClient.pushRawUpdate(
  1343. this.projectId,
  1344. slTextUpdate(
  1345. historyId,
  1346. this.doc,
  1347. this.userId,
  1348. 2,
  1349. this.timestamp,
  1350. [{ p: 6, i: 'baz' }]
  1351. ),
  1352. cb
  1353. )
  1354. },
  1355. cb => {
  1356. ProjectHistoryClient.flushProject(this.projectId, cb)
  1357. },
  1358. ],
  1359. error => {
  1360. if (error) {
  1361. return done(error)
  1362. }
  1363. assert(
  1364. createChange.isDone(),
  1365. `/api/projects/${historyId}/changes should have been called`
  1366. )
  1367. done()
  1368. }
  1369. )
  1370. })
  1371. it('should track the doc length when splitting ops', function (done) {
  1372. this.doc.length = 10
  1373. const createChange = MockHistoryStore()
  1374. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1375. expect(body).to.deep.equal([
  1376. olTextUpdate(this.doc, this.userId, this.timestamp, [3, -3, 4], 1),
  1377. olTextUpdate(
  1378. this.doc,
  1379. this.userId,
  1380. this.timestamp,
  1381. [3, 'barbaz', 4],
  1382. 2
  1383. ), // This has a base length of 10
  1384. ])
  1385. return true
  1386. })
  1387. .query({ end_version: 0 })
  1388. .reply(204)
  1389. async.series(
  1390. [
  1391. cb => {
  1392. ProjectHistoryClient.pushRawUpdate(
  1393. this.projectId,
  1394. slTextUpdate(
  1395. historyId,
  1396. this.doc,
  1397. this.userId,
  1398. 1,
  1399. this.timestamp,
  1400. [
  1401. { p: 3, d: 'foo' },
  1402. { p: 3, i: 'bar' }, // Make sure the length of the op generated from this is 7, not 10
  1403. ]
  1404. ),
  1405. cb
  1406. )
  1407. },
  1408. cb => {
  1409. ProjectHistoryClient.pushRawUpdate(
  1410. this.projectId,
  1411. slTextUpdate(
  1412. historyId,
  1413. this.doc,
  1414. this.userId,
  1415. 2,
  1416. this.timestamp,
  1417. [{ p: 6, i: 'baz' }]
  1418. ),
  1419. cb
  1420. )
  1421. },
  1422. cb => {
  1423. ProjectHistoryClient.flushProject(this.projectId, cb)
  1424. },
  1425. ],
  1426. error => {
  1427. if (error) {
  1428. return done(error)
  1429. }
  1430. assert(
  1431. createChange.isDone(),
  1432. `/api/projects/${historyId}/changes should have been called`
  1433. )
  1434. done()
  1435. }
  1436. )
  1437. })
  1438. })
  1439. describe('with bad pathnames', function () {
  1440. beforeEach(function () {
  1441. MockHistoryStore()
  1442. .get(`/api/projects/${historyId}/latest/history`)
  1443. .reply(200, {
  1444. chunk: {
  1445. startVersion: 0,
  1446. history: {
  1447. snapshot: {},
  1448. changes: [],
  1449. },
  1450. },
  1451. })
  1452. })
  1453. it('should replace \\ with _ and workaround * in pathnames', function (done) {
  1454. const doc = {
  1455. id: this.doc.id,
  1456. pathname: '\\main.tex',
  1457. hash: 'b07b6b7a27667965f733943737124395c7577bea',
  1458. docLines: 'aaabbbccc',
  1459. length: 9,
  1460. }
  1461. MockHistoryStore()
  1462. .put(`/api/projects/${historyId}/blobs/${doc.hash}`)
  1463. .reply(201)
  1464. const createChange = MockHistoryStore()
  1465. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1466. expect(body).to.deep.equal([
  1467. olAddDocUpdate(
  1468. { id: doc.id, pathname: '_main.tex' },
  1469. this.userId,
  1470. this.timestamp,
  1471. doc.hash
  1472. ),
  1473. olRenameUpdate(
  1474. { id: doc.id, pathname: '_main.tex' },
  1475. this.userId,
  1476. this.timestamp,
  1477. '_main.tex',
  1478. '_main2.tex'
  1479. ),
  1480. olTextUpdate(
  1481. { id: doc.id, pathname: '_main2.tex' },
  1482. this.userId,
  1483. this.timestamp,
  1484. [3, 'foo', 6],
  1485. 2
  1486. ),
  1487. olRenameUpdate(
  1488. { id: doc.id, pathname: '_main2.tex' },
  1489. this.userId,
  1490. this.timestamp,
  1491. '_main2.tex',
  1492. '_main__ASTERISK__.tex'
  1493. ),
  1494. ])
  1495. return true
  1496. })
  1497. .query({ end_version: 0 })
  1498. .reply(204)
  1499. async.series(
  1500. [
  1501. cb => {
  1502. ProjectHistoryClient.pushRawUpdate(
  1503. this.projectId,
  1504. slAddDocUpdate(
  1505. historyId,
  1506. doc,
  1507. this.userId,
  1508. this.timestamp,
  1509. doc.docLines
  1510. ),
  1511. cb
  1512. )
  1513. },
  1514. cb => {
  1515. ProjectHistoryClient.pushRawUpdate(
  1516. this.projectId,
  1517. slRenameUpdate(
  1518. historyId,
  1519. doc,
  1520. this.userId,
  1521. this.timestamp,
  1522. '/\\main.tex',
  1523. '/\\main2.tex'
  1524. ),
  1525. cb
  1526. )
  1527. doc.pathname = '\\main2.tex'
  1528. },
  1529. cb => {
  1530. ProjectHistoryClient.pushRawUpdate(
  1531. this.projectId,
  1532. slTextUpdate(historyId, doc, this.userId, 2, this.timestamp, [
  1533. { p: 3, i: 'foo' },
  1534. ]),
  1535. cb
  1536. )
  1537. },
  1538. cb => {
  1539. ProjectHistoryClient.pushRawUpdate(
  1540. this.projectId,
  1541. slRenameUpdate(
  1542. historyId,
  1543. doc,
  1544. this.userId,
  1545. this.timestamp,
  1546. '/\\main2.tex',
  1547. '/\\main*.tex'
  1548. ),
  1549. cb
  1550. )
  1551. doc.pathname = '\\main*.tex'
  1552. },
  1553. cb => {
  1554. ProjectHistoryClient.flushProject(this.projectId, cb)
  1555. },
  1556. ],
  1557. error => {
  1558. if (error) {
  1559. return done(error)
  1560. }
  1561. assert(
  1562. createChange.isDone(),
  1563. `/api/projects/${historyId}/changes should have been called`
  1564. )
  1565. done()
  1566. }
  1567. )
  1568. })
  1569. it('should workaround pathnames beginning with spaces', function (done) {
  1570. const doc = {
  1571. id: this.doc.id,
  1572. pathname: 'main.tex',
  1573. hash: 'b07b6b7a27667965f733943737124395c7577bea',
  1574. docLines: 'aaabbbccc',
  1575. length: 9,
  1576. }
  1577. MockHistoryStore()
  1578. .put(`/api/projects/${historyId}/blobs/${doc.hash}`)
  1579. .reply(201)
  1580. const createChange = MockHistoryStore()
  1581. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1582. expect(body).to.deep.equal([
  1583. olAddDocUpdate(
  1584. { id: doc.id, pathname: 'main.tex' },
  1585. this.userId,
  1586. this.timestamp,
  1587. doc.hash
  1588. ),
  1589. olRenameUpdate(
  1590. { id: doc.id },
  1591. this.userId,
  1592. this.timestamp,
  1593. 'main.tex',
  1594. 'foo/__SPACE__main.tex'
  1595. ),
  1596. ])
  1597. return true
  1598. })
  1599. .query({ end_version: 0 })
  1600. .reply(204)
  1601. async.series(
  1602. [
  1603. cb => {
  1604. ProjectHistoryClient.pushRawUpdate(
  1605. this.projectId,
  1606. slAddDocUpdate(
  1607. historyId,
  1608. doc,
  1609. this.userId,
  1610. this.timestamp,
  1611. doc.docLines
  1612. ),
  1613. cb
  1614. )
  1615. },
  1616. cb => {
  1617. ProjectHistoryClient.pushRawUpdate(
  1618. this.projectId,
  1619. slRenameUpdate(
  1620. historyId,
  1621. doc,
  1622. this.userId,
  1623. this.timestamp,
  1624. '/main.tex',
  1625. '/foo/ main.tex'
  1626. ),
  1627. cb
  1628. )
  1629. doc.pathname = '/foo/ main.tex'
  1630. },
  1631. cb => {
  1632. ProjectHistoryClient.flushProject(this.projectId, cb)
  1633. },
  1634. ],
  1635. error => {
  1636. if (error) {
  1637. return done(error)
  1638. }
  1639. assert(
  1640. createChange.isDone(),
  1641. `/api/projects/${historyId}/changes should have been called`
  1642. )
  1643. done()
  1644. }
  1645. )
  1646. })
  1647. })
  1648. describe('with bad response from filestore', function () {
  1649. beforeEach(function () {
  1650. MockHistoryStore()
  1651. .get(`/api/projects/${historyId}/latest/history`)
  1652. .reply(200, {
  1653. chunk: {
  1654. startVersion: 0,
  1655. history: {
  1656. snapshot: {},
  1657. changes: [],
  1658. },
  1659. },
  1660. })
  1661. })
  1662. it('should return a 500 if the filestore returns a 500', function (done) {
  1663. const file = {
  1664. id: new ObjectId().toString(),
  1665. pathname: '/test.png',
  1666. contents: Buffer.from([1, 2, 3]),
  1667. hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
  1668. }
  1669. const fileStoreRequest = MockFileStore()
  1670. .get(`/project/${this.projectId}/file/${file.id}`)
  1671. .reply(500)
  1672. const createBlob = MockHistoryStore()
  1673. .put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
  1674. .reply(201)
  1675. const addFile = MockHistoryStore()
  1676. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1677. expect(body).to.deep.equal([
  1678. olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
  1679. ])
  1680. return true
  1681. })
  1682. .query({ end_version: 0 })
  1683. .reply(204)
  1684. async.series(
  1685. [
  1686. cb => {
  1687. ProjectHistoryClient.pushRawUpdate(
  1688. this.projectId,
  1689. slAddFileUpdate(
  1690. historyId,
  1691. file,
  1692. this.userId,
  1693. this.timestamp,
  1694. this.projectId
  1695. ),
  1696. cb
  1697. )
  1698. },
  1699. cb => {
  1700. ProjectHistoryClient.flushProject(
  1701. this.projectId,
  1702. { allowErrors: true },
  1703. (error, res) => {
  1704. if (error) {
  1705. return cb(error)
  1706. }
  1707. expect(res.statusCode).to.equal(500)
  1708. cb()
  1709. }
  1710. )
  1711. },
  1712. ],
  1713. error => {
  1714. if (error) {
  1715. return done(error)
  1716. }
  1717. assert(
  1718. fileStoreRequest.isDone(),
  1719. `/project/${this.projectId}/file/${file.id} should have been called`
  1720. )
  1721. assert(
  1722. !createBlob.isDone(),
  1723. `/api/projects/${historyId}/latest/files should not have been called`
  1724. )
  1725. assert(
  1726. !addFile.isDone(),
  1727. `/api/projects/${historyId}/latest/files should not have been called`
  1728. )
  1729. done()
  1730. }
  1731. )
  1732. })
  1733. it('should return a 500 if the filestore request errors', function (done) {
  1734. const file = {
  1735. id: new ObjectId().toString(),
  1736. pathname: '/test.png',
  1737. contents: Buffer.from([1, 2, 3]),
  1738. hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
  1739. }
  1740. const fileStoreRequest = MockFileStore()
  1741. .get(`/project/${this.projectId}/file/${file.id}`)
  1742. .replyWithError('oh no!')
  1743. const createBlob = MockHistoryStore()
  1744. .put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
  1745. .reply(201)
  1746. const addFile = MockHistoryStore()
  1747. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1748. expect(body).to.deep.equal([
  1749. olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
  1750. ])
  1751. return true
  1752. })
  1753. .query({ end_version: 0 })
  1754. .reply(204)
  1755. async.series(
  1756. [
  1757. cb => {
  1758. ProjectHistoryClient.pushRawUpdate(
  1759. this.projectId,
  1760. slAddFileUpdate(
  1761. historyId,
  1762. file,
  1763. this.userId,
  1764. this.timestamp,
  1765. this.projectId
  1766. ),
  1767. cb
  1768. )
  1769. },
  1770. cb => {
  1771. ProjectHistoryClient.flushProject(
  1772. this.projectId,
  1773. { allowErrors: true },
  1774. (error, res) => {
  1775. if (error) {
  1776. return cb(error)
  1777. }
  1778. expect(res.statusCode).to.equal(500)
  1779. cb()
  1780. }
  1781. )
  1782. },
  1783. ],
  1784. error => {
  1785. if (error) {
  1786. return done(error)
  1787. }
  1788. assert(
  1789. fileStoreRequest.isDone(),
  1790. `/project/${this.projectId}/file/${file.id} should have been called`
  1791. )
  1792. assert(
  1793. !createBlob.isDone(),
  1794. `/api/projects/${historyId}/latest/files should not have been called`
  1795. )
  1796. assert(
  1797. !addFile.isDone(),
  1798. `/api/projects/${historyId}/latest/files should not have been called`
  1799. )
  1800. done()
  1801. }
  1802. )
  1803. })
  1804. })
  1805. describe('with an existing projectVersion field', function () {
  1806. beforeEach(function () {
  1807. MockHistoryStore()
  1808. .get(`/api/projects/${historyId}/latest/history`)
  1809. .reply(200, {
  1810. chunk: {
  1811. startVersion: 0,
  1812. history: {
  1813. snapshot: { projectVersion: '100.0' },
  1814. changes: [],
  1815. },
  1816. },
  1817. })
  1818. })
  1819. it('should discard project structure updates which have already been applied', function (done) {
  1820. const newDoc = []
  1821. for (let i = 0; i <= 2; i++) {
  1822. newDoc[i] = {
  1823. id: new ObjectId().toString(),
  1824. pathname: `/main${i}.tex`,
  1825. hash: '0a207c060e61f3b88eaee0a8cd0696f46fb155eb',
  1826. docLines: 'a\nb',
  1827. }
  1828. }
  1829. MockHistoryStore()
  1830. .put(`/api/projects/${historyId}/blobs/${newDoc[0].hash}`)
  1831. .times(3)
  1832. .reply(201)
  1833. const createChange = MockHistoryStore()
  1834. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1835. expect(body).to.deep.equal([
  1836. olAddDocUpdateWithVersion(
  1837. newDoc[1],
  1838. this.userId,
  1839. this.timestamp,
  1840. newDoc[1].hash,
  1841. '101.0'
  1842. ),
  1843. olAddDocUpdateWithVersion(
  1844. newDoc[2],
  1845. this.userId,
  1846. this.timestamp,
  1847. newDoc[2].hash,
  1848. '102.0'
  1849. ),
  1850. ])
  1851. return true
  1852. })
  1853. .query({ end_version: 0 })
  1854. .reply(204)
  1855. async.series(
  1856. [
  1857. cb => {
  1858. ProjectHistoryClient.pushRawUpdate(
  1859. this.projectId,
  1860. slAddDocUpdateWithVersion(
  1861. historyId,
  1862. newDoc[0],
  1863. this.userId,
  1864. this.timestamp,
  1865. newDoc[0].docLines,
  1866. '100.0'
  1867. ),
  1868. cb
  1869. )
  1870. },
  1871. cb => {
  1872. ProjectHistoryClient.pushRawUpdate(
  1873. this.projectId,
  1874. slAddDocUpdateWithVersion(
  1875. historyId,
  1876. newDoc[1],
  1877. this.userId,
  1878. this.timestamp,
  1879. newDoc[1].docLines,
  1880. '101.0'
  1881. ),
  1882. cb
  1883. )
  1884. },
  1885. cb => {
  1886. ProjectHistoryClient.pushRawUpdate(
  1887. this.projectId,
  1888. slAddDocUpdateWithVersion(
  1889. historyId,
  1890. newDoc[2],
  1891. this.userId,
  1892. this.timestamp,
  1893. newDoc[2].docLines,
  1894. '102.0'
  1895. ),
  1896. cb
  1897. )
  1898. },
  1899. cb => {
  1900. ProjectHistoryClient.flushProject(this.projectId, cb)
  1901. },
  1902. ],
  1903. error => {
  1904. if (error) {
  1905. return done(error)
  1906. }
  1907. assert(
  1908. createChange.isDone(),
  1909. `/api/projects/${historyId}/changes should have been called`
  1910. )
  1911. done()
  1912. }
  1913. )
  1914. })
  1915. })
  1916. describe('with an existing docVersions field', function () {
  1917. beforeEach(function () {
  1918. MockHistoryStore()
  1919. .get(`/api/projects/${historyId}/latest/history`)
  1920. .reply(200, {
  1921. chunk: {
  1922. startVersion: 0,
  1923. history: {
  1924. snapshot: { v2DocVersions: { [this.doc.id]: { v: 100 } } }, // version 100 below already applied
  1925. changes: [],
  1926. },
  1927. },
  1928. })
  1929. })
  1930. it('should discard doc updates which have already been applied', function (done) {
  1931. const createChange = MockHistoryStore()
  1932. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  1933. expect(body).to.deep.equal([
  1934. olTextUpdate(
  1935. this.doc,
  1936. this.userId,
  1937. this.timestamp,
  1938. [6, 'baz', 2],
  1939. 101
  1940. ),
  1941. ])
  1942. return true
  1943. })
  1944. .query({ end_version: 0 })
  1945. .reply(204)
  1946. async.series(
  1947. [
  1948. cb => {
  1949. ProjectHistoryClient.pushRawUpdate(
  1950. this.projectId,
  1951. slTextUpdate(
  1952. historyId,
  1953. this.doc,
  1954. this.userId,
  1955. 100,
  1956. this.timestamp,
  1957. [
  1958. { p: 3, i: 'foobar' }, // these ops should be skipped
  1959. { p: 6, d: 'bar' },
  1960. ]
  1961. ),
  1962. cb
  1963. )
  1964. this.doc.length += 3
  1965. },
  1966. cb => {
  1967. ProjectHistoryClient.pushRawUpdate(
  1968. this.projectId,
  1969. slTextUpdate(
  1970. historyId,
  1971. this.doc,
  1972. this.userId,
  1973. 101,
  1974. this.timestamp,
  1975. [
  1976. { p: 6, i: 'baz' }, // this op should be applied
  1977. ]
  1978. ),
  1979. cb
  1980. )
  1981. },
  1982. cb => {
  1983. ProjectHistoryClient.flushProject(this.projectId, cb)
  1984. },
  1985. ],
  1986. error => {
  1987. if (error) {
  1988. return done(error)
  1989. }
  1990. assert(
  1991. createChange.isDone(),
  1992. `/api/projects/${historyId}/changes should have been called`
  1993. )
  1994. done()
  1995. }
  1996. )
  1997. })
  1998. })
  1999. })