SyncTests.js 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625
  1. import async from 'async'
  2. import nock from 'nock'
  3. import { expect } from 'chai'
  4. import request from 'request'
  5. import assert from 'assert'
  6. import mongodb from 'mongodb-legacy'
  7. import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
  8. import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
  9. const { ObjectId } = mongodb
  10. const EMPTY_FILE_HASH = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
  11. const MockHistoryStore = () => nock('http://127.0.0.1:3100')
  12. const MockFileStore = () => nock('http://127.0.0.1:3009')
  13. const MockWeb = () => nock('http://127.0.0.1:3000')
  14. describe('Syncing with web and doc-updater', function () {
  15. const historyId = new ObjectId().toString()
  16. beforeEach(function (done) {
  17. this.timestamp = new Date()
  18. ProjectHistoryApp.ensureRunning(error => {
  19. if (error) {
  20. throw error
  21. }
  22. this.project_id = new ObjectId().toString()
  23. this.doc_id = new ObjectId().toString()
  24. this.file_id = new ObjectId().toString()
  25. MockHistoryStore().post('/api/projects').reply(200, {
  26. projectId: historyId,
  27. })
  28. MockWeb()
  29. .get(`/project/${this.project_id}/details`)
  30. .reply(200, {
  31. name: 'Test Project',
  32. overleaf: {
  33. history: {
  34. id: historyId,
  35. },
  36. },
  37. })
  38. MockHistoryStore()
  39. .get(`/api/projects/${historyId}/latest/history`)
  40. .reply(200, {
  41. chunk: {
  42. startVersion: 0,
  43. history: {
  44. changes: [],
  45. },
  46. },
  47. })
  48. ProjectHistoryClient.initializeProject(historyId, done)
  49. })
  50. })
  51. afterEach(function () {
  52. nock.cleanAll()
  53. })
  54. describe('resyncing project history', function () {
  55. describe('without project-history enabled', function () {
  56. beforeEach(function () {
  57. MockWeb().post(`/project/${this.project_id}/history/resync`).reply(404)
  58. })
  59. it('404s if project-history is not enabled', function (done) {
  60. request.post(
  61. {
  62. url: `http://127.0.0.1:3054/project/${this.project_id}/resync`,
  63. },
  64. (error, res, body) => {
  65. if (error) {
  66. return done(error)
  67. }
  68. expect(res.statusCode).to.equal(404)
  69. done()
  70. }
  71. )
  72. })
  73. })
  74. describe('with project-history enabled', function () {
  75. beforeEach(function () {
  76. MockWeb().post(`/project/${this.project_id}/history/resync`).reply(204)
  77. })
  78. describe('when a doc is missing', function () {
  79. it('should send add doc updates to the history store', function (done) {
  80. MockHistoryStore()
  81. .get(`/api/projects/${historyId}/latest/history`)
  82. .reply(200, {
  83. chunk: {
  84. history: {
  85. snapshot: {
  86. files: {
  87. persistedDoc: { hash: EMPTY_FILE_HASH, stringLength: 0 },
  88. },
  89. },
  90. changes: [],
  91. },
  92. startVersion: 0,
  93. },
  94. })
  95. MockHistoryStore()
  96. .get(`/api/projects/${historyId}/blobs/${EMPTY_FILE_HASH}`)
  97. .reply(200, '')
  98. const createBlob = MockHistoryStore()
  99. .put(`/api/projects/${historyId}/blobs/${EMPTY_FILE_HASH}`, '')
  100. .reply(201)
  101. const addFile = MockHistoryStore()
  102. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  103. expect(body).to.deep.equal([
  104. {
  105. v2Authors: [],
  106. authors: [],
  107. timestamp: this.timestamp.toJSON(),
  108. operations: [
  109. {
  110. pathname: 'main.tex',
  111. file: {
  112. hash: EMPTY_FILE_HASH,
  113. },
  114. },
  115. ],
  116. origin: { kind: 'test-origin' },
  117. },
  118. ])
  119. return true
  120. })
  121. .query({ end_version: 0 })
  122. .reply(204)
  123. async.series(
  124. [
  125. cb => {
  126. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  127. },
  128. cb => {
  129. const update = {
  130. projectHistoryId: historyId,
  131. resyncProjectStructure: {
  132. docs: [
  133. { path: '/main.tex', doc: this.doc_id },
  134. { path: '/persistedDoc', doc: 'other-doc-id' },
  135. ],
  136. files: [],
  137. },
  138. meta: {
  139. ts: this.timestamp,
  140. },
  141. }
  142. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  143. },
  144. cb => {
  145. ProjectHistoryClient.flushProject(this.project_id, cb)
  146. },
  147. ],
  148. error => {
  149. if (error) {
  150. return done(error)
  151. }
  152. assert(
  153. createBlob.isDone(),
  154. '/api/projects/:historyId/blobs/:hash should have been called'
  155. )
  156. assert(
  157. addFile.isDone(),
  158. `/api/projects/${historyId}/changes should have been called`
  159. )
  160. done()
  161. }
  162. )
  163. })
  164. })
  165. describe('when a file is missing', function () {
  166. it('should send add file updates to the history store', function (done) {
  167. MockHistoryStore()
  168. .get(`/api/projects/${historyId}/latest/history`)
  169. .reply(200, {
  170. chunk: {
  171. history: {
  172. snapshot: {
  173. files: {
  174. persistedFile: { hash: EMPTY_FILE_HASH, byteLength: 0 },
  175. },
  176. },
  177. changes: [],
  178. },
  179. startVersion: 0,
  180. },
  181. })
  182. const fileContents = Buffer.from([1, 2, 3])
  183. const fileHash = 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74'
  184. MockFileStore()
  185. .get(`/project/${this.project_id}/file/${this.file_id}`)
  186. .reply(200, fileContents)
  187. const createBlob = MockHistoryStore()
  188. .put(`/api/projects/${historyId}/blobs/${fileHash}`, fileContents)
  189. .reply(201)
  190. const addFile = MockHistoryStore()
  191. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  192. expect(body).to.deep.equal([
  193. {
  194. v2Authors: [],
  195. authors: [],
  196. timestamp: this.timestamp.toJSON(),
  197. operations: [
  198. {
  199. pathname: 'test.png',
  200. file: {
  201. hash: fileHash,
  202. },
  203. },
  204. ],
  205. origin: { kind: 'test-origin' },
  206. },
  207. ])
  208. return true
  209. })
  210. .query({ end_version: 0 })
  211. .reply(204)
  212. async.series(
  213. [
  214. cb => {
  215. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  216. },
  217. cb => {
  218. const update = {
  219. projectHistoryId: historyId,
  220. resyncProjectStructure: {
  221. docs: [],
  222. files: [
  223. {
  224. file: this.file_id,
  225. path: '/test.png',
  226. url: `http://127.0.0.1:3009/project/${this.project_id}/file/${this.file_id}`,
  227. },
  228. { path: '/persistedFile' },
  229. ],
  230. },
  231. meta: {
  232. ts: this.timestamp,
  233. },
  234. }
  235. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  236. },
  237. cb => {
  238. ProjectHistoryClient.flushProject(this.project_id, cb)
  239. },
  240. ],
  241. error => {
  242. if (error) {
  243. throw error
  244. }
  245. assert(
  246. createBlob.isDone(),
  247. '/api/projects/:historyId/blobs/:hash should have been called'
  248. )
  249. assert(
  250. addFile.isDone(),
  251. `/api/projects/${historyId}/changes should have been called`
  252. )
  253. done()
  254. }
  255. )
  256. })
  257. })
  258. describe("when a file exists which shouldn't", function () {
  259. it('should send remove file updates to the history store', function (done) {
  260. MockHistoryStore()
  261. .get(`/api/projects/${historyId}/latest/history`)
  262. .reply(200, {
  263. chunk: {
  264. history: {
  265. snapshot: {
  266. files: {
  267. docToKeep: { hash: EMPTY_FILE_HASH, stringLength: 0 },
  268. docToDelete: { hash: EMPTY_FILE_HASH, stringLength: 0 },
  269. },
  270. },
  271. changes: [],
  272. },
  273. startVersion: 0,
  274. },
  275. })
  276. MockHistoryStore()
  277. .get(`/api/projects/${historyId}/blobs/${EMPTY_FILE_HASH}`)
  278. .reply(200, '')
  279. .get(`/api/projects/${historyId}/blobs/${EMPTY_FILE_HASH}`)
  280. .reply(200, '') // blob is requested once for each file
  281. const deleteFile = MockHistoryStore()
  282. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  283. expect(body).to.deep.equal([
  284. {
  285. v2Authors: [],
  286. authors: [],
  287. timestamp: this.timestamp.toJSON(),
  288. operations: [
  289. {
  290. pathname: 'docToDelete',
  291. newPathname: '',
  292. },
  293. ],
  294. origin: { kind: 'test-origin' },
  295. },
  296. ])
  297. return true
  298. })
  299. .query({ end_version: 0 })
  300. .reply(204)
  301. async.series(
  302. [
  303. cb => {
  304. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  305. },
  306. cb => {
  307. const update = {
  308. projectHistoryId: historyId,
  309. resyncProjectStructure: {
  310. docs: [{ path: 'docToKeep' }],
  311. files: [],
  312. },
  313. meta: {
  314. ts: this.timestamp,
  315. },
  316. }
  317. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  318. },
  319. cb => {
  320. ProjectHistoryClient.flushProject(this.project_id, cb)
  321. },
  322. ],
  323. error => {
  324. if (error) {
  325. throw error
  326. }
  327. assert(
  328. deleteFile.isDone(),
  329. `/api/projects/${historyId}/changes should have been called`
  330. )
  331. done()
  332. }
  333. )
  334. })
  335. })
  336. describe("when a doc's contents is not up to date", function () {
  337. beforeEach(function () {
  338. MockHistoryStore()
  339. .get(`/api/projects/${historyId}/latest/history`)
  340. .reply(200, {
  341. chunk: {
  342. history: {
  343. snapshot: {
  344. files: {
  345. 'main.tex': {
  346. hash: '0a207c060e61f3b88eaee0a8cd0696f46fb155eb',
  347. stringLength: 3,
  348. },
  349. },
  350. },
  351. changes: [],
  352. },
  353. startVersion: 0,
  354. },
  355. })
  356. MockHistoryStore()
  357. .get(
  358. `/api/projects/${historyId}/blobs/0a207c060e61f3b88eaee0a8cd0696f46fb155eb`
  359. )
  360. .reply(200, 'a\nb')
  361. })
  362. it('should send test updates to the history store', function (done) {
  363. const addFile = MockHistoryStore()
  364. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  365. expect(body).to.deep.equal([
  366. {
  367. v2Authors: [],
  368. authors: [],
  369. timestamp: this.timestamp.toJSON(),
  370. operations: [
  371. {
  372. pathname: 'main.tex',
  373. textOperation: [3, '\nc'],
  374. },
  375. ],
  376. origin: { kind: 'test-origin' },
  377. },
  378. ])
  379. return true
  380. })
  381. .query({ end_version: 0 })
  382. .reply(204)
  383. async.series(
  384. [
  385. cb => {
  386. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  387. },
  388. cb => {
  389. const update = {
  390. projectHistoryId: historyId,
  391. resyncProjectStructure: {
  392. docs: [{ path: '/main.tex' }],
  393. files: [],
  394. },
  395. meta: {
  396. ts: this.timestamp,
  397. },
  398. }
  399. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  400. },
  401. cb => {
  402. const update = {
  403. path: '/main.tex',
  404. projectHistoryId: historyId,
  405. resyncDocContent: {
  406. content: 'a\nb\nc',
  407. },
  408. doc: this.doc_id,
  409. meta: {
  410. ts: this.timestamp,
  411. },
  412. }
  413. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  414. },
  415. cb => {
  416. ProjectHistoryClient.flushProject(this.project_id, cb)
  417. },
  418. ],
  419. error => {
  420. if (error) {
  421. throw error
  422. }
  423. assert(
  424. addFile.isDone(),
  425. `/api/projects/${historyId}/changes should have been called`
  426. )
  427. done()
  428. }
  429. )
  430. })
  431. it('should strip non-BMP characters in updates before sending to the history store', function (done) {
  432. const addFile = MockHistoryStore()
  433. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  434. expect(body).to.deep.equal([
  435. {
  436. v2Authors: [],
  437. authors: [],
  438. timestamp: this.timestamp.toJSON(),
  439. operations: [
  440. {
  441. pathname: 'main.tex',
  442. textOperation: [3, '\n\uFFFD\uFFFDc'],
  443. },
  444. ],
  445. origin: { kind: 'test-origin' },
  446. },
  447. ])
  448. return true
  449. })
  450. .query({ end_version: 0 })
  451. .reply(204)
  452. async.series(
  453. [
  454. cb => {
  455. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  456. },
  457. cb => {
  458. const update = {
  459. projectHistoryId: historyId,
  460. resyncProjectStructure: {
  461. docs: [{ path: '/main.tex' }],
  462. files: [],
  463. },
  464. meta: {
  465. ts: this.timestamp,
  466. },
  467. }
  468. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  469. },
  470. cb => {
  471. const update = {
  472. path: '/main.tex',
  473. projectHistoryId: historyId,
  474. resyncDocContent: {
  475. content: 'a\nb\n\uD800\uDC00c',
  476. },
  477. doc: this.doc_id,
  478. meta: {
  479. ts: this.timestamp,
  480. },
  481. }
  482. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  483. },
  484. cb => {
  485. ProjectHistoryClient.flushProject(this.project_id, cb)
  486. },
  487. ],
  488. error => {
  489. if (error) {
  490. throw error
  491. }
  492. assert(
  493. addFile.isDone(),
  494. `/api/projects/${historyId}/changes should have been called`
  495. )
  496. done()
  497. }
  498. )
  499. })
  500. it('should fix comments in the history store', function (done) {
  501. const commentId = 'comment-id'
  502. const addComment = MockHistoryStore()
  503. .post(`/api/projects/${historyId}/legacy_changes`, body => {
  504. expect(body).to.deep.equal([
  505. {
  506. v2Authors: [],
  507. authors: [],
  508. timestamp: this.timestamp.toJSON(),
  509. operations: [
  510. {
  511. pathname: 'main.tex',
  512. commentId,
  513. ranges: [{ pos: 1, length: 10 }],
  514. },
  515. ],
  516. origin: { kind: 'test-origin' },
  517. },
  518. ])
  519. return true
  520. })
  521. .query({ end_version: 0 })
  522. .reply(204)
  523. async.series(
  524. [
  525. cb => {
  526. ProjectHistoryClient.resyncHistory(this.project_id, cb)
  527. },
  528. cb => {
  529. const update = {
  530. projectHistoryId: historyId,
  531. resyncProjectStructure: {
  532. docs: [{ path: '/main.tex' }],
  533. files: [],
  534. },
  535. meta: {
  536. ts: this.timestamp,
  537. },
  538. }
  539. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  540. },
  541. cb => {
  542. const update = {
  543. path: '/main.tex',
  544. projectHistoryId: historyId,
  545. resyncDocContent: {
  546. content: 'a\nb',
  547. ranges: {
  548. comments: [
  549. {
  550. id: commentId,
  551. op: {
  552. c: 'a',
  553. p: 0,
  554. hpos: 1,
  555. hlen: 10,
  556. t: commentId,
  557. },
  558. meta: {
  559. user_id: 'user-id',
  560. ts: this.timestamp,
  561. },
  562. },
  563. ],
  564. },
  565. },
  566. doc: this.doc_id,
  567. meta: {
  568. ts: this.timestamp,
  569. },
  570. }
  571. ProjectHistoryClient.pushRawUpdate(this.project_id, update, cb)
  572. },
  573. cb => {
  574. ProjectHistoryClient.flushProject(this.project_id, cb)
  575. },
  576. ],
  577. error => {
  578. if (error) {
  579. return done(error)
  580. }
  581. assert(
  582. addComment.isDone(),
  583. `/api/projects/${historyId}/changes should have been called`
  584. )
  585. done()
  586. }
  587. )
  588. })
  589. })
  590. })
  591. })
  592. })