Merge pull request #20277 from overleaf/jpa-ro-mirror-chunk

[misc] use Chunk for read-only mirror in frontend

GitOrigin-RevId: b97e520c7a63015179792542310d18f346752e92

libraries/overleaf-editor-core/lib/chunk.js

@@ -6,10 +6,11 @@ const OError = require('@overleaf/o-error')
 const History = require('./history')
 
 /**
- * @import { BlobStore } from "./types"
+ * @import { BlobStore, RawChunk } from "./types"
  * @import Change from "./change"
  * @import Snapshot from "./snapshot"
  */
+
 class ConflictingEndVersion extends OError {
   constructor(clientEndVersion, latestEndVersion) {
     const message =
@@ -84,6 +85,10 @@ class Chunk {
     this.startVersion = startVersion
   }
 
+  /**
+   * @param {RawChunk} raw
+   * @return {Chunk}
+   */
   static fromRaw(raw) {
     return new Chunk(History.fromRaw(raw.history), raw.startVersion)
   }

libraries/overleaf-editor-core/lib/types.ts

@@ -76,6 +76,16 @@ export type RawSnapshot = {
   v2DocVersions?: RawV2DocVersions | null
 }
 
+export type RawHistory = {
+  snapshot: RawSnapshot
+  changes: RawChange[]
+}
+
+export type RawChunk = {
+  history: RawHistory
+  startVersion: number
+}
+
 export type RawFileMap = Record<string, RawFile>
 
 export type RawFile = { metadata?: Object } & RawFileData

services/project-history/app/js/HttpController.js

@@ -234,6 +234,22 @@ export function getFileMetadataSnapshot(req, res, next) {
   )
 }
 
+export function getMostRecentChunk(req, res, next) {
+  const { project_id: projectId } = req.params
+  WebApiManager.getHistoryId(projectId, (error, historyId) => {
+    if (error) return next(OError.tag(error))
+
+    HistoryStoreManager.getMostRecentChunk(
+      projectId,
+      historyId,
+      (err, data) => {
+        if (err) return next(OError.tag(err))
+        res.json(data)
+      }
+    )
+  })
+}
+
 export function getLatestSnapshot(req, res, next) {
   const { project_id: projectId } = req.params
   WebApiManager.getHistoryId(projectId, (error, historyId) => {
@@ -241,10 +257,11 @@ export function getLatestSnapshot(req, res, next) {
     SnapshotManager.getLatestSnapshot(
       projectId,
       historyId,
-      (error, { snapshot, version }) => {
+      (error, details) => {
         if (error != null) {
           return next(error)
         }
+        const { snapshot, version } = details
         res.json({ snapshot: snapshot.toRaw(), version })
       }
     )
@@ -270,6 +287,29 @@ export function getChangesSince(req, res, next) {
   })
 }
 
+export function getChangesInChunkSince(req, res, next) {
+  const { project_id: projectId } = req.params
+  const { since } = req.query
+  WebApiManager.getHistoryId(projectId, (error, historyId) => {
+    if (error) return next(OError.tag(error))
+    SnapshotManager.getChangesInChunkSince(
+      projectId,
+      historyId,
+      since,
+      (error, details) => {
+        if (error != null) {
+          return next(error)
+        }
+        const { latestStartVersion, changes } = details
+        res.json({
+          latestStartVersion,
+          changes: changes.map(c => c.toRaw()),
+        })
+      }
+    )
+  })
+}
+
 export function getProjectSnapshot(req, res, next) {
   const { project_id: projectId, version } = req.params
   SnapshotManager.getProjectSnapshot(

services/project-history/app/js/Router.js

@@ -22,6 +22,10 @@ export function initialize(app) {
   app.delete('/project/:project_id', HttpController.deleteProject)
 
   app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot)
+  app.get(
+    '/project/:project_id/latest/history',
+    HttpController.getMostRecentChunk
+  )
 
   app.get(
     '/project/:project_id/diff',
@@ -61,12 +65,22 @@ export function initialize(app) {
     '/project/:project_id/changes',
     validate({
       query: {
-        since: Joi.number().integer(),
+        since: Joi.number().integer().min(0),
       },
     }),
     HttpController.getChangesSince
   )
 
+  app.get(
+    '/project/:project_id/changes-in-chunk',
+    validate({
+      query: {
+        since: Joi.number().integer().min(0),
+      },
+    }),
+    HttpController.getChangesInChunkSince
+  )
+
   app.get('/project/:project_id/version', HttpController.latestVersion)
 
   app.post(

services/project-history/app/js/SnapshotManager.js

@@ -354,6 +354,39 @@ async function getChangesSince(projectId, historyId, sinceVersion) {
   return allChanges
 }
 
+async function getChangesInChunkSince(projectId, historyId, sinceVersion) {
+  const latestChunk = Core.Chunk.fromRaw(
+    (
+      await HistoryStoreManager.promises.getMostRecentChunk(
+        projectId,
+        historyId
+      )
+    ).chunk
+  )
+  if (sinceVersion > latestChunk.getEndVersion()) {
+    throw new Errors.BadRequestError(
+      'requested version past the end of the history'
+    )
+  }
+  const latestStartVersion = latestChunk.getStartVersion()
+  let chunk = latestChunk
+  if (sinceVersion < latestStartVersion) {
+    chunk = Core.Chunk.fromRaw(
+      (
+        await HistoryStoreManager.promises.getChunkAtVersion(
+          projectId,
+          historyId,
+          sinceVersion
+        )
+      ).chunk
+    )
+  }
+  const changes = chunk
+    .getChanges()
+    .slice(sinceVersion - chunk.getStartVersion())
+  return { latestStartVersion, changes }
+}
+
 async function _loadFilesLimit(snapshot, kind, blobStore) {
   await snapshot.fileMap.mapAsync(async file => {
     // only load changed files or files with tracked changes, others can be
@@ -369,6 +402,7 @@ async function _loadFilesLimit(snapshot, kind, blobStore) {
 // EXPORTS
 
 const getChangesSinceCb = callbackify(getChangesSince)
+const getChangesInChunkSinceCb = callbackify(getChangesInChunkSince)
 const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream)
 const getProjectSnapshotCb = callbackify(getProjectSnapshot)
 const getLatestSnapshotCb = callbackify(getLatestSnapshot)
@@ -379,6 +413,7 @@ const getPathsAtVersionCb = callbackify(getPathsAtVersion)
 
 export {
   getChangesSinceCb as getChangesSince,
+  getChangesInChunkSinceCb as getChangesInChunkSince,
   getFileSnapshotStreamCb as getFileSnapshotStream,
   getProjectSnapshotCb as getProjectSnapshot,
   getFileMetadataSnapshotCb as getFileMetadataSnapshot,
@@ -390,6 +425,7 @@ export {
 
 export const promises = {
   getChangesSince,
+  getChangesInChunkSince,
   getFileSnapshotStream,
   getProjectSnapshot,
   getLatestSnapshot,

services/project-history/test/acceptance/js/GetChangesInChunkSince.js

@@ -0,0 +1,158 @@
+import { expect } from 'chai'
+import mongodb from 'mongodb-legacy'
+import nock from 'nock'
+import Core from 'overleaf-editor-core'
+import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
+import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
+import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' }
+import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' }
+import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' }
+const { ObjectId } = mongodb
+
+const MockHistoryStore = () => nock('http://127.0.0.1:3100')
+const MockWeb = () => nock('http://127.0.0.1:3000')
+
+const fixture = path => new URL(`../fixtures/${path}`, import.meta.url)
+
+describe('GetChangesInChunkSince', function () {
+  let projectId, historyId
+  beforeEach(function (done) {
+    projectId = new ObjectId().toString()
+    historyId = new ObjectId().toString()
+    ProjectHistoryApp.ensureRunning(error => {
+      if (error) throw error
+
+      MockHistoryStore().post('/api/projects').reply(200, {
+        projectId: historyId,
+      })
+
+      ProjectHistoryClient.initializeProject(historyId, (error, olProject) => {
+        if (error) throw error
+        MockWeb()
+          .get(`/project/${projectId}/details`)
+          .reply(200, {
+            name: 'Test Project',
+            overleaf: { history: { id: olProject.id } },
+          })
+
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/latest/history`)
+          .replyWithFile(200, fixture('chunks/7-8.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/7/history`)
+          .replyWithFile(200, fixture('chunks/7-8.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/6/history`)
+          .replyWithFile(200, fixture('chunks/7-8.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/5/history`)
+          .replyWithFile(200, fixture('chunks/4-6.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/4/history`)
+          .replyWithFile(200, fixture('chunks/4-6.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/3/history`)
+          .replyWithFile(200, fixture('chunks/4-6.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/2/history`)
+          .replyWithFile(200, fixture('chunks/0-3.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/1/history`)
+          .replyWithFile(200, fixture('chunks/0-3.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/0/history`)
+          .replyWithFile(200, fixture('chunks/0-3.json'))
+
+        done()
+      })
+    })
+  })
+
+  afterEach(function () {
+    nock.cleanAll()
+  })
+
+  function expectChangesSince(version, n, changes, done) {
+    ProjectHistoryClient.getChangesInChunkSince(
+      projectId,
+      version,
+      {},
+      (error, got) => {
+        if (error) throw error
+        expect(got.latestStartVersion).to.equal(6)
+        expect(got.changes).to.have.length(n)
+        expect(got.changes.map(c => Core.Change.fromRaw(c))).to.deep.equal(
+          changes.map(c => Core.Change.fromRaw(c))
+        )
+        done()
+      }
+    )
+  }
+
+  const cases = {
+    8: {
+      name: 'when up-to-date, return zero changes',
+      n: 0,
+      changes: [],
+    },
+    7: {
+      name: 'when one version behind, return one change',
+      n: 1,
+      changes: latestChunk.chunk.history.changes.slice(1),
+    },
+    6: {
+      name: 'when at current chunk boundary, return latest chunk in full',
+      n: 2,
+      changes: latestChunk.chunk.history.changes,
+    },
+    5: {
+      name: 'when one version behind last chunk, return one change',
+      n: 1,
+      changes: previousChunk.chunk.history.changes.slice(2),
+    },
+    4: {
+      name: 'when in last chunk, return two changes',
+      n: 2,
+      changes: previousChunk.chunk.history.changes.slice(1),
+    },
+    3: {
+      name: 'when at previous chunk boundary, return just the previous chunk',
+      n: 3,
+      changes: previousChunk.chunk.history.changes,
+    },
+    2: {
+      name: 'when at end of first chunk, return one change',
+      n: 1,
+      changes: firstChunk.chunk.history.changes.slice(2),
+    },
+    1: {
+      name: 'when in first chunk, return two changes',
+      n: 2,
+      changes: firstChunk.chunk.history.changes.slice(1),
+    },
+    0: {
+      name: 'when from zero, return just the first chunk',
+      n: 3,
+      changes: firstChunk.chunk.history.changes,
+    },
+  }
+
+  for (const [since, { name, n, changes }] of Object.entries(cases)) {
+    it(name, function (done) {
+      expectChangesSince(since, n, changes, done)
+    })
+  }
+
+  it('should return an error when past the end version', function (done) {
+    ProjectHistoryClient.getChangesInChunkSince(
+      projectId,
+      9,
+      { allowErrors: true },
+      (error, _body, statusCode) => {
+        if (error) throw error
+        expect(statusCode).to.equal(400)
+        done()
+      }
+    )
+  })
+})

services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js

@@ -127,6 +127,25 @@ export function getChangesSince(projectId, since, options, callback) {
   )
 }
 
+export function getChangesInChunkSince(projectId, since, options, callback) {
+  request.get(
+    {
+      url: `http://127.0.0.1:3054/project/${projectId}/changes-in-chunk`,
+      qs: {
+        since,
+      },
+      json: true,
+    },
+    (error, res, body) => {
+      if (error) return callback(error)
+      if (!options.allowErrors) {
+        expect(res.statusCode).to.equal(200)
+      }
+      callback(null, body, res.statusCode)
+    }
+  )
+}
+
 export function getLatestSnapshot(projectId, callback) {
   request.get(
     {