Merge pull request #19842 from overleaf/jpa-ro-mirror-on-client

[misc] add readonly mirror of full project content on the client

GitOrigin-RevId: 940bd93bfd587f83ca383d10fc44579b38fc3e88

libraries/overleaf-editor-core/lib/types.ts

@@ -51,6 +51,25 @@ export type StringFileRawData = {
   trackedChanges?: TrackedChangeRawData[]
 }
 
+export type RawOrigin = {
+  kind: string
+}
+
+export type RawChange = {
+  operations: RawOperation[]
+  timestamp: string
+  authors?: (number | null)[]
+  v2Authors: string[]
+  origin: RawOrigin
+  projectVersion: string
+  v2DocVersions: RawV2DocVersions
+}
+
+export type RawOperation =
+  | RawEditFileOperation
+  // TODO(das7pad): add types for all the other operations
+  | object
+
 export type RawSnapshot = {
   files: RawFileMap
   projectVersion?: string

package-lock.json

@@ -22682,6 +22682,15 @@
         "node >=0.6.0"
       ]
     },
+    "node_modules/fake-indexeddb": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.0.0.tgz",
+      "integrity": "sha512-YEboHE5VfopUclOck7LncgIqskAqnv4q0EWbYCaxKKjAvO93c+TJIaBuGy8CBFdbg9nKdpN3AuPRwVBJ4k7NrQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/fast-copy": {
       "version": "2.1.7",
       "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-2.1.7.tgz",
@@ -44790,6 +44799,7 @@
         "eslint-plugin-react": "^7.32.2",
         "eslint-plugin-react-hooks": "^4.6.0",
         "events": "^3.3.0",
+        "fake-indexeddb": "^6.0.0",
         "fetch-mock": "^9.10.2",
         "formik": "^2.2.9",
         "glob": "^7.1.6",
@@ -53365,6 +53375,7 @@
         "express-bearer-token": "^2.4.0",
         "express-http-proxy": "^1.6.0",
         "express-session": "^1.17.1",
+        "fake-indexeddb": "^6.0.0",
         "fetch-mock": "^9.10.2",
         "formik": "^2.2.9",
         "fs-extra": "^4.0.2",
@@ -65322,6 +65333,12 @@
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
       "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
     },
+    "fake-indexeddb": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.0.0.tgz",
+      "integrity": "sha512-YEboHE5VfopUclOck7LncgIqskAqnv4q0EWbYCaxKKjAvO93c+TJIaBuGy8CBFdbg9nKdpN3AuPRwVBJ4k7NrQ==",
+      "dev": true
+    },
     "fast-copy": {
       "version": "2.1.7",
       "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-2.1.7.tgz",

services/project-history/app/js/HttpController.js

@@ -17,6 +17,8 @@ import * as RetryManager from './RetryManager.js'
 import * as FlushManager from './FlushManager.js'
 import { pipeline } from 'stream'
 
+const ONE_DAY_IN_SECONDS = 24 * 60 * 60
+
 export function getProjectBlob(req, res, next) {
   const projectId = req.params.project_id
   const blobHash = req.params.hash
@@ -27,6 +29,7 @@ export function getProjectBlob(req, res, next) {
       if (err != null) {
         return next(OError.tag(err))
       }
+      res.setHeader('Cache-Control', `private, max-age=${ONE_DAY_IN_SECONDS}`)
       pipeline(stream, res, err => {
         if (err) next(err)
         // res.end() is already called via 'end' event by pipeline.
@@ -216,6 +219,42 @@ export function getRangesSnapshot(req, res, next) {
   )
 }
 
+export function getLatestSnapshot(req, res, next) {
+  const { project_id: projectId } = req.params
+  WebApiManager.getHistoryId(projectId, (error, historyId) => {
+    if (error) return next(OError.tag(error))
+    SnapshotManager.getLatestSnapshot(
+      projectId,
+      historyId,
+      (error, { snapshot, version }) => {
+        if (error != null) {
+          return next(error)
+        }
+        res.json({ snapshot: snapshot.toRaw(), version })
+      }
+    )
+  })
+}
+
+export function getChangesSince(req, res, next) {
+  const { project_id: projectId } = req.params
+  const { since } = req.query
+  WebApiManager.getHistoryId(projectId, (error, historyId) => {
+    if (error) return next(OError.tag(error))
+    SnapshotManager.getChangesSince(
+      projectId,
+      historyId,
+      since,
+      (error, changes) => {
+        if (error != null) {
+          return next(error)
+        }
+        res.json(changes.map(c => c.toRaw()))
+      }
+    )
+  })
+}
+
 export function getProjectSnapshot(req, res, next) {
   const { project_id: projectId, version } = req.params
   SnapshotManager.getProjectSnapshot(

services/project-history/app/js/Router.js

@@ -21,6 +21,8 @@ export function initialize(app) {
 
   app.delete('/project/:project_id', HttpController.deleteProject)
 
+  app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot)
+
   app.get(
     '/project/:project_id/diff',
     validate({
@@ -55,6 +57,16 @@ export function initialize(app) {
     HttpController.getUpdates
   )
 
+  app.get(
+    '/project/:project_id/changes',
+    validate({
+      query: {
+        since: Joi.number().integer(),
+      },
+    }),
+    HttpController.getChangesSince
+  )
+
   app.get('/project/:project_id/version', HttpController.latestVersion)
 
   app.post(

services/project-history/app/js/SnapshotManager.js

@@ -263,6 +263,15 @@ async function _getSnapshotAtVersion(projectId, version) {
   return snapshot
 }
 
+async function getLatestSnapshotFiles(projectId, historyId) {
+  const { snapshot } = await getLatestSnapshot(projectId, historyId)
+  const snapshotFiles = await snapshot.loadFiles(
+    'lazy',
+    HistoryStoreManager.getBlobStore(historyId)
+  )
+  return snapshotFiles
+}
+
 async function getLatestSnapshot(projectId, historyId) {
   const data = await HistoryStoreManager.promises.getMostRecentChunk(
     projectId,
@@ -277,11 +286,48 @@ async function getLatestSnapshot(projectId, historyId) {
   const snapshot = chunk.getSnapshot()
   const changes = chunk.getChanges()
   snapshot.applyAll(changes)
-  const snapshotFiles = await snapshot.loadFiles(
-    'lazy',
-    HistoryStoreManager.getBlobStore(historyId)
-  )
-  return snapshotFiles
+  return {
+    snapshot,
+    version: chunk.getEndVersion(),
+  }
+}
+
+async function getChangesSince(projectId, historyId, sinceVersion) {
+  const allChanges = []
+  let nextVersion
+  while (true) {
+    let data
+    if (nextVersion) {
+      data = await HistoryStoreManager.promises.getChunkAtVersion(
+        projectId,
+        historyId,
+        nextVersion
+      )
+    } else {
+      data = await HistoryStoreManager.promises.getMostRecentChunk(
+        projectId,
+        historyId
+      )
+    }
+    if (data == null || data.chunk == null) {
+      throw new OError('undefined chunk')
+    }
+    const chunk = Core.Chunk.fromRaw(data.chunk)
+    if (sinceVersion > chunk.getEndVersion()) {
+      throw new OError('requested version past the end')
+    }
+    const changes = chunk.getChanges()
+    if (chunk.getStartVersion() > sinceVersion) {
+      allChanges.unshift(...changes)
+      nextVersion = chunk.getStartVersion()
+    } else {
+      allChanges.unshift(
+        ...changes.slice(sinceVersion - chunk.getStartVersion())
+      )
+      break
+    }
+  }
+  return allChanges
 }
 
 async function _loadFilesLimit(snapshot, kind, blobStore) {
@@ -298,24 +344,30 @@ async function _loadFilesLimit(snapshot, kind, blobStore) {
 
 // EXPORTS
 
+const getChangesSinceCb = callbackify(getChangesSince)
 const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream)
 const getProjectSnapshotCb = callbackify(getProjectSnapshot)
 const getLatestSnapshotCb = callbackify(getLatestSnapshot)
+const getLatestSnapshotFilesCb = callbackify(getLatestSnapshotFiles)
 const getRangesSnapshotCb = callbackify(getRangesSnapshot)
 const getPathsAtVersionCb = callbackify(getPathsAtVersion)
 
 export {
+  getChangesSinceCb as getChangesSince,
   getFileSnapshotStreamCb as getFileSnapshotStream,
   getProjectSnapshotCb as getProjectSnapshot,
   getLatestSnapshotCb as getLatestSnapshot,
+  getLatestSnapshotFilesCb as getLatestSnapshotFiles,
   getRangesSnapshotCb as getRangesSnapshot,
   getPathsAtVersionCb as getPathsAtVersion,
 }
 
 export const promises = {
+  getChangesSince,
   getFileSnapshotStream,
   getProjectSnapshot,
   getLatestSnapshot,
+  getLatestSnapshotFiles,
   getRangesSnapshot,
   getPathsAtVersion,
 }

services/project-history/app/js/SyncManager.js

@@ -203,7 +203,7 @@ async function expandSyncUpdates(
   const syncState = await _getResyncState(projectId)
 
   // compute the current snapshot from the most recent chunk
-  const snapshotFiles = await SnapshotManager.promises.getLatestSnapshot(
+  const snapshotFiles = await SnapshotManager.promises.getLatestSnapshotFiles(
     projectId,
     projectHistoryId
   )

services/project-history/app/js/types.ts

@@ -1,5 +1,5 @@
 import { HistoryRanges } from '../../../document-updater/app/js/types'
-import { LinkedFileData } from 'overleaf-editor-core/lib/types'
+import { LinkedFileData, RawOrigin } from 'overleaf-editor-core/lib/types'
 
 export type Update =
   | TextUpdate
@@ -161,10 +161,6 @@ export type UpdateWithBlob<T extends Update = Update> = {
     : never
 }
 
-export type RawOrigin = {
-  kind: string
-}
-
 export type TrackingProps = {
   type: 'insert' | 'delete'
   userId: string

services/project-history/test/acceptance/js/GetChangesSince.js

@@ -0,0 +1,122 @@
+import { expect } from 'chai'
+import mongodb from 'mongodb-legacy'
+import nock from 'nock'
+import Core from 'overleaf-editor-core'
+import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
+import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
+import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' }
+import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' }
+import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' }
+const { ObjectId } = mongodb
+
+const MockHistoryStore = () => nock('http://127.0.0.1:3100')
+const MockWeb = () => nock('http://127.0.0.1:3000')
+
+const fixture = path => new URL(`../fixtures/${path}`, import.meta.url)
+
+describe('GetChangesSince', function () {
+  let projectId, historyId
+  beforeEach(function (done) {
+    projectId = new ObjectId().toString()
+    historyId = new ObjectId().toString()
+    ProjectHistoryApp.ensureRunning(error => {
+      if (error) throw error
+
+      MockHistoryStore().post('/api/projects').reply(200, {
+        projectId: historyId,
+      })
+
+      ProjectHistoryClient.initializeProject(historyId, (error, olProject) => {
+        if (error) throw error
+        MockWeb()
+          .get(`/project/${projectId}/details`)
+          .reply(200, {
+            name: 'Test Project',
+            overleaf: { history: { id: olProject.id } },
+          })
+
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/latest/history`)
+          .replyWithFile(200, fixture('chunks/7-8.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/6/history`)
+          .replyWithFile(200, fixture('chunks/4-6.json'))
+        MockHistoryStore()
+          .get(`/api/projects/${historyId}/versions/3/history`)
+          .replyWithFile(200, fixture('chunks/0-3.json'))
+
+        done()
+      })
+    })
+  })
+
+  afterEach(function () {
+    nock.cleanAll()
+  })
+
+  function expectChangesSince(version, changes, done) {
+    ProjectHistoryClient.getChangesSince(
+      projectId,
+      version,
+      {},
+      (error, got) => {
+        if (error) throw error
+        expect(got.map(c => Core.Change.fromRaw(c))).to.deep.equal(
+          changes.map(c => Core.Change.fromRaw(c))
+        )
+        done()
+      }
+    )
+  }
+
+  it('should return zero changes since the latest version', function (done) {
+    expectChangesSince(8, [], done)
+  })
+
+  it('should return one change when behind one version', function (done) {
+    expectChangesSince(7, [latestChunk.chunk.history.changes[1]], done)
+  })
+
+  it('should return changes when at the chunk boundary', function (done) {
+    expect(latestChunk.chunk.startVersion).to.equal(6)
+    expectChangesSince(6, latestChunk.chunk.history.changes, done)
+  })
+
+  it('should return changes spanning multiple chunks', function (done) {
+    expectChangesSince(
+      1,
+      [
+        ...firstChunk.chunk.history.changes.slice(1),
+        ...previousChunk.chunk.history.changes,
+        ...latestChunk.chunk.history.changes,
+      ],
+      done
+    )
+  })
+
+  it('should return all changes when going back to the beginning', function (done) {
+    expectChangesSince(
+      0,
+      [
+        ...firstChunk.chunk.history.changes,
+        ...previousChunk.chunk.history.changes,
+        ...latestChunk.chunk.history.changes,
+      ],
+      done
+    )
+  })
+
+  it('should return an error when past the end version', function (done) {
+    ProjectHistoryClient.getChangesSince(
+      projectId,
+      9,
+      { allowErrors: true },
+      (error, body, statusCode) => {
+        if (error) throw error
+        expect(statusCode).to.equal(500)
+        expect(body).to.deep.equal({ message: 'an internal error occurred' })
+        done()
+      }
+    )
+  })
+})

services/project-history/test/acceptance/js/LatestSnapshotTests.js

@@ -0,0 +1,78 @@
+import { expect } from 'chai'
+import mongodb from 'mongodb-legacy'
+import nock from 'nock'
+import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
+import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
+const { ObjectId } = mongodb
+
+const MockHistoryStore = () => nock('http://127.0.0.1:3100')
+const MockWeb = () => nock('http://127.0.0.1:3000')
+
+const fixture = path => new URL(`../fixtures/${path}`, import.meta.url)
+
+describe('LatestSnapshot', function () {
+  beforeEach(function (done) {
+    ProjectHistoryApp.ensureRunning(error => {
+      if (error) {
+        throw error
+      }
+
+      this.historyId = new ObjectId().toString()
+      MockHistoryStore().post('/api/projects').reply(200, {
+        projectId: this.historyId,
+      })
+
+      ProjectHistoryClient.initializeProject(
+        this.historyId,
+        (error, v1Project) => {
+          if (error) {
+            throw error
+          }
+          this.projectId = new ObjectId().toString()
+          MockWeb()
+            .get(`/project/${this.projectId}/details`)
+            .reply(200, {
+              name: 'Test Project',
+              overleaf: { history: { id: v1Project.id } },
+            })
+          done()
+        }
+      )
+    })
+  })
+
+  afterEach(function () {
+    nock.cleanAll()
+  })
+
+  it('should return the snapshot with applied changes, metadata and without full content', function (done) {
+    MockHistoryStore()
+      .get(`/api/projects/${this.historyId}/latest/history`)
+      .replyWithFile(200, fixture('chunks/0-3.json'))
+
+    ProjectHistoryClient.getLatestSnapshot(this.projectId, (error, body) => {
+      if (error) {
+        throw error
+      }
+      expect(body).to.deep.equal({
+        snapshot: {
+          files: {
+            'main.tex': {
+              hash: 'f28571f561d198b87c24cc6a98b78e87b665e22d',
+              stringLength: 20649,
+              operations: [{ textOperation: [1912, 'Hello world', 18726] }],
+              metadata: { main: true },
+            },
+            'foo.tex': {
+              hash: '4f785a4c192155b240e3042b3a7388b47603f423',
+              stringLength: 41,
+              operations: [{ textOperation: [26, '\n\nFour five six'] }],
+            },
+          },
+        },
+        version: 3,
+      })
+      done()
+    })
+  })
+})

services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js

@@ -108,6 +108,41 @@ export function getFileTreeDiff(projectId, from, to, callback) {
   )
 }
 
+export function getChangesSince(projectId, since, options, callback) {
+  request.get(
+    {
+      url: `http://127.0.0.1:3054/project/${projectId}/changes`,
+      qs: {
+        since,
+      },
+      json: true,
+    },
+    (error, res, body) => {
+      if (error) return callback(error)
+      if (!options.allowErrors) {
+        expect(res.statusCode).to.equal(200)
+      }
+      callback(null, body, res.statusCode)
+    }
+  )
+}
+
+export function getLatestSnapshot(projectId, callback) {
+  request.get(
+    {
+      url: `http://127.0.0.1:3054/project/${projectId}/snapshot`,
+      json: true,
+    },
+    (error, res, body) => {
+      if (error) {
+        return callback(error)
+      }
+      expect(res.statusCode).to.equal(200)
+      callback(null, body)
+    }
+  )
+}
+
 export function getSnapshot(projectId, pathname, version, options, callback) {
   if (typeof options === 'function') {
     callback = options

services/project-history/test/unit/js/HttpController/HttpControllerTests.js

@@ -84,6 +84,7 @@ describe('HttpController', function () {
       json: sinon.stub(),
       send: sinon.stub(),
       sendStatus: sinon.stub(),
+      setHeader: sinon.stub(),
     }
   })
 
@@ -105,6 +106,13 @@ describe('HttpController', function () {
         .should.equal(true)
       this.pipeline.should.have.been.calledWith(this.stream, this.res)
     })
+
+    it('should set caching header', function () {
+      this.res.setHeader.should.have.been.calledWith(
+        'Cache-Control',
+        'private, max-age=86400'
+      )
+    })
   })
 
   describe('initializeProject', function () {

services/project-history/test/unit/js/SnapshotManager/SnapshotManagerTests.js

@@ -479,7 +479,7 @@ Four five six\
     })
   })
 
-  describe('getLatestSnapshot', function () {
+  describe('getLatestSnapshotFiles', function () {
     describe('for a project', function () {
       beforeEach(async function () {
         this.HistoryStoreManager.promises.getMostRecentChunk.resolves({
@@ -543,7 +543,7 @@ Four five six\
           )),
           getObject: sinon.stub().rejects(),
         })
-        this.data = await this.SnapshotManager.promises.getLatestSnapshot(
+        this.data = await this.SnapshotManager.promises.getLatestSnapshotFiles(
           this.projectId,
           this.historyId
         )
@@ -571,7 +571,7 @@ Four five six\
       beforeEach(async function () {
         this.HistoryStoreManager.promises.getMostRecentChunk.resolves(null)
         expect(
-          this.SnapshotManager.promises.getLatestSnapshot(
+          this.SnapshotManager.promises.getLatestSnapshotFiles(
             this.projectId,
             this.historyId
           )

services/project-history/test/unit/js/SyncManager/SyncManagerTests.js

@@ -114,7 +114,7 @@ describe('SyncManager', function () {
 
     this.SnapshotManager = {
       promises: {
-        getLatestSnapshot: sinon.stub(),
+        getLatestSnapshotFiles: sinon.stub(),
       },
     }
 
@@ -517,7 +517,9 @@ describe('SyncManager', function () {
         .returns('another.tex')
       this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png')
       this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png')
-      this.SnapshotManager.promises.getLatestSnapshot.resolves(this.fileMap)
+      this.SnapshotManager.promises.getLatestSnapshotFiles.resolves(
+        this.fileMap
+      )
     })
 
     it('returns updates if no sync updates are queued', async function () {
@@ -530,8 +532,8 @@ describe('SyncManager', function () {
       )
 
       expect(expandedUpdates).to.equal(updates)
-      expect(this.SnapshotManager.promises.getLatestSnapshot).to.not.have.been
-        .called
+      expect(this.SnapshotManager.promises.getLatestSnapshotFiles).to.not.have
+        .been.called
       expect(this.extendLock).to.not.have.been.called
     })
 

services/web/app/src/Features/Errors/Errors.js

@@ -5,6 +5,9 @@ const settings = require('@overleaf/settings')
 // backward-compatible (can be instantiated with string as argument instead
 // of object)
 class BackwardCompatibleError extends OError {
+  /**
+   * @param {string | { message: string, info?: Object }} messageOrOptions
+   */
   constructor(messageOrOptions) {
     if (typeof messageOrOptions === 'string') {
       super(messageOrOptions)

services/web/app/src/Features/Project/ProjectController.js

@@ -337,6 +337,7 @@ const _ProjectController = {
       'revert-file',
       'revert-project',
       'review-panel-redesign',
+      !anonymous && 'ro-mirror-on-client',
       'track-pdf-download',
       !anonymous && 'writefull-oauth-promotion',
       'ieee-stylesheet',

services/web/app/src/Features/Project/ProjectEditorHandler.js

@@ -127,6 +127,7 @@ module.exports = ProjectEditorHandler = {
       name: file.name,
       linkedFileData: file.linkedFileData,
       created: file.created,
+      hash: file.hash,
     }
   },
 

services/web/app/src/Features/SplitTests/SplitTestMiddleware.js

@@ -1,5 +1,7 @@
 const SplitTestHandler = require('./SplitTestHandler')
 const logger = require('@overleaf/logger')
+const { expressify } = require('@overleaf/promise-utils')
+const Errors = require('../Errors/Errors')
 
 function loadAssignmentsInLocals(splitTestNames) {
   return async function (req, res, next) {
@@ -17,6 +19,31 @@ function loadAssignmentsInLocals(splitTestNames) {
   }
 }
 
+function ensureSplitTestEnabledForUser(
+  splitTestName,
+  enabledVariant = 'enabled'
+) {
+  return expressify(async function (req, res, next) {
+    const { variant } = await SplitTestHandler.promises.getAssignment(
+      req,
+      res,
+      splitTestName
+    )
+    if (variant !== enabledVariant) {
+      throw new Errors.ForbiddenError({
+        message: 'missing split test access',
+        info: {
+          splitTestName,
+          variant,
+          enabledVariant,
+        },
+      })
+    }
+    next()
+  })
+}
+
 module.exports = {
   loadAssignmentsInLocals,
+  ensureSplitTestEnabledForUser,
 }

services/web/app/src/infrastructure/RateLimiter.js

@@ -11,6 +11,8 @@ const rclient = RedisWrapper.client('ratelimiter')
  * Wrapper over the RateLimiterRedis class
  */
 class RateLimiter {
+  #opts
+
   /**
    * Create a rate limiter.
    *
@@ -31,6 +33,7 @@ class RateLimiter {
    */
   constructor(name, opts = {}) {
     this.name = name
+    this.#opts = Object.assign({}, opts)
     this._rateLimiter = new RateLimiterFlexible.RateLimiterRedis({
       ...opts,
       keyPrefix: `rate-limit:${name}`,
@@ -46,6 +49,11 @@ class RateLimiter {
     }
   }
 
+  // Readonly access to the options, useful for aligning rate-limits.
+  getOptions() {
+    return Object.assign({}, this.#opts)
+  }
+
   async consume(key, points = 1, options = { method: 'unknown' }) {
     if (Settings.disableRateLimits) {
       // Return a fake result in case it's used somewhere

services/web/app/src/infrastructure/Server.js

@@ -296,6 +296,13 @@ webRouter.use(function addNoCacheHeader(req, res, next) {
     // don't set no-cache headers on a project file, as it's immutable and can be cached (privately)
     return next()
   }
+  const isProjectBlob = /^\/project\/[a-f0-9]{24}\/blob\/[a-f0-9]{40}$/.test(
+    req.path
+  )
+  if (isProjectBlob) {
+    // don't set no-cache headers on a project blobs, as they are immutable and can be cached (privately)
+    return next()
+  }
 
   const isWikiContent = /^\/learn(-scripts)?(\/|$)/i.test(req.path)
   if (isWikiContent) {

services/web/app/src/router.js

@@ -125,6 +125,20 @@ const rateLimiters = {
     points: 30,
     duration: 60 * 60,
   }),
+  flushHistory: new RateLimiter('flush-project-history', {
+    // Allow flushing once every 30s-1s (allow for network jitter).
+    points: 1,
+    duration: 30 - 1,
+  }),
+  getProjectBlob: new RateLimiter('get-project-blob', {
+    // Download project in full once per hour
+    points: Settings.maxEntitiesPerProject,
+    duration: 60 * 60,
+  }),
+  getHistorySnapshot: new RateLimiter(
+    'get-history-snapshot',
+    openProjectRateLimiter.getOptions()
+  ),
   endorseEmail: new RateLimiter('endorse-email', {
     points: 30,
     duration: 60,

services/web/config/settings.defaults.js

@@ -901,6 +901,8 @@ module.exports = {
     managedGroupEnrollmentInvite: [],
     ssoCertificateInfo: [],
     v1ImportDataScreen: [],
+    snapshotUtils: [],
+    offlineModeToolbarButtons: [],
   },
 
   moduleImportSequence: [

services/web/frontend/extracted-translations.json

@@ -476,6 +476,7 @@
   "from_provider": "",
   "from_url": "",
   "full_doc_history": "",
+  "full_project_search": "",
   "full_width": "",
   "generate_token": "",
   "generic_if_problem_continues_contact_us": "",
@@ -844,6 +845,7 @@
   "my_library": "",
   "n_items": "",
   "n_items_plural": "",
+  "n_matches": "",
   "n_more_updates_above": "",
   "n_more_updates_above_plural": "",
   "n_more_updates_below": "",
@@ -1100,6 +1102,7 @@
   "refresh_page_after_starting_free_trial": "",
   "refreshing": "",
   "regards": "",
+  "regular_expression": "",
   "reject": "",
   "reject_all": "",
   "reject_change": "",
@@ -1183,6 +1186,7 @@
   "saving": "",
   "saving_notification_with_seconds": "",
   "search": "",
+  "search_all_project_files": "",
   "search_bib_files": "",
   "search_by_citekey_author_year_title": "",
   "search_command_find": "",
@@ -1202,6 +1206,7 @@
   "search_replace_all": "",
   "search_replace_with": "",
   "search_search_for": "",
+  "search_terms": "",
   "search_whole_word": "",
   "search_within_selection": "",
   "searched_path_for_lines_containing": "",

services/web/frontend/js/features/editor-navigation-toolbar/components/toolbar-header.jsx

@@ -15,10 +15,20 @@ import ShareProjectButton from './share-project-button'
 import importOverleafModules from '../../../../macros/import-overleaf-module.macro'
 import BackToEditorButton from './back-to-editor-button'
 import getMeta from '@/utils/meta'
+import { isSplitTestEnabled } from '@/utils/splitTestUtils'
 
 const [publishModalModules] = importOverleafModules('publishModal')
 const PublishButton = publishModalModules?.import.default
 
+const offlineModeToolbarButtons = importOverleafModules(
+  'offlineModeToolbarButtons'
+)
+// double opt-in
+const enableROMirrorOnClient =
+  isSplitTestEnabled('ro-mirror-on-client') &&
+  new URLSearchParams(window.location.search).get('ro-mirror-on-client') ===
+    'enabled'
+
 const ToolbarHeader = React.memo(function ToolbarHeader({
   cobranding,
   onShowLeftMenuClick,
@@ -55,6 +65,12 @@ const ToolbarHeader = React.memo(function ToolbarHeader({
           <CobrandingLogo {...cobranding} />
         )}
         <BackToProjectsButton />
+        {enableROMirrorOnClient &&
+          offlineModeToolbarButtons.map(
+            ({ path, import: { default: OfflineModeToolbarButton } }) => {
+              return <OfflineModeToolbarButton key={path} />
+            }
+          )}
       </div>
       {getMeta('ol-showUpgradePrompt') && <UpgradePrompt />}
       <ProjectNameEditableLabel

services/web/frontend/js/features/file-view/components/file-view-header.tsx

@@ -12,6 +12,7 @@ import { LinkedFileIcon } from './file-view-icons'
 import { BinaryFile, hasProvider, LinkedFile } from '../types/binary-file'
 import FileViewRefreshButton from './file-view-refresh-button'
 import FileViewRefreshError from './file-view-refresh-error'
+import { useSnapshotContext } from '@/features/ide-react/context/snapshot-context'
 
 const tprFileViewInfo = importOverleafModules('tprFileViewInfo') as {
   import: { TPRFileViewInfo: ElementType }
@@ -49,6 +50,7 @@ type FileViewHeaderProps = {
 export default function FileViewHeader({ file }: FileViewHeaderProps) {
   const { _id: projectId } = useProjectContext()
   const { permissionsLevel } = useEditorContext()
+  const { fileTreeFromHistory } = useSnapshotContext()
   const { t } = useTranslation()
 
   const [refreshError, setRefreshError] = useState<Nullable<string>>(null)
@@ -88,8 +90,12 @@ export default function FileViewHeader({ file }: FileViewHeaderProps) {
       )}
       &nbsp;
       <a
-        download
-        href={`/project/${projectId}/file/${file.id}`}
+        download={file.name}
+        href={
+          fileTreeFromHistory
+            ? `/project/${projectId}/blob/${file.hash}`
+            : `/project/${projectId}/file/${file.id}`
+        }
         className="btn btn-secondary-info btn-secondary"
       >
         <Icon type="download" fw />

services/web/frontend/js/features/file-view/components/file-view-image.tsx

@@ -1,24 +1,29 @@
 import { useProjectContext } from '../../../shared/context/project-context'
+import { useSnapshotContext } from '@/features/ide-react/context/snapshot-context'
+import { BinaryFile } from '@/features/file-view/types/binary-file'
 
 export default function FileViewImage({
-  fileName,
-  fileId,
+  file,
   onLoad,
   onError,
 }: {
-  fileName: string
-  fileId: string
+  file: BinaryFile
   onLoad: () => void
   onError: () => void
 }) {
   const { _id: projectId } = useProjectContext()
+  const { fileTreeFromHistory } = useSnapshotContext()
 
   return (
     <img
-      src={`/project/${projectId}/file/${fileId}`}
+      src={
+        fileTreeFromHistory
+          ? `/project/${projectId}/blob/${file.hash}`
+          : `/project/${projectId}/file/${file.id}`
+      }
       onLoad={onLoad}
       onError={onError}
-      alt={fileName}
+      alt={file.name}
     />
   )
 }

services/web/frontend/js/features/file-view/components/file-view-text.tsx

@@ -2,6 +2,8 @@ import { useState, useEffect } from 'react'
 import { useProjectContext } from '../../../shared/context/project-context'
 import { debugConsole } from '@/utils/debugging'
 import useAbortController from '../../../shared/hooks/use-abort-controller'
+import { BinaryFile } from '@/features/file-view/types/binary-file'
+import { useSnapshotContext } from '@/features/ide-react/context/snapshot-context'
 
 const MAX_FILE_SIZE = 2 * 1024 * 1024
 
@@ -10,11 +12,12 @@ export default function FileViewText({
   onLoad,
   onError,
 }: {
-  file: { id: string }
+  file: BinaryFile
   onLoad: () => void
   onError: () => void
 }) {
   const { _id: projectId } = useProjectContext()
+  const { fileTreeFromHistory } = useSnapshotContext()
 
   const [textPreview, setTextPreview] = useState('')
   const [shouldShowDots, setShouldShowDots] = useState(false)
@@ -27,7 +30,9 @@ export default function FileViewText({
     if (inFlight) {
       return
     }
-    let path = `/project/${projectId}/file/${file.id}`
+    let path = fileTreeFromHistory
+      ? `/project/${projectId}/blob/${file.hash}`
+      : `/project/${projectId}/file/${file.id}`
     const fetchContentLengthTimeout = setTimeout(
       () => fetchContentLengthController.abort(),
       10000
@@ -77,8 +82,10 @@ export default function FileViewText({
         clearTimeout(fetchDataTimeout)
       })
   }, [
+    fileTreeFromHistory,
     projectId,
     file.id,
+    file.hash,
     onError,
     onLoad,
     inFlight,

services/web/frontend/js/features/file-view/components/file-view.jsx

@@ -44,12 +44,7 @@ export default function FileView({ file }) {
     <>
       <FileViewHeader file={file} />
       {isImageFile && (
-        <FileViewImage
-          fileName={file.name}
-          fileId={file.id}
-          onLoad={handleLoad}
-          onError={handleError}
-        />
+        <FileViewImage file={file} onLoad={handleLoad} onError={handleError} />
       )}
       {isEditableTextFile && (
         <FileViewText file={file} onLoad={handleLoad} onError={handleError} />
@@ -91,5 +86,6 @@ FileView.propTypes = {
   file: PropTypes.shape({
     id: PropTypes.string,
     name: PropTypes.string,
+    hash: PropTypes.string,
   }).isRequired,
 }

services/web/frontend/js/features/file-view/types/binary-file.ts

@@ -26,6 +26,7 @@ export type BinaryFile<T extends keyof LinkedFileData = keyof LinkedFileData> =
     type: string
     selected: boolean
     linkedFileData?: LinkedFileData[T]
+    hash: string
   }
 
 export type LinkedFile<T extends keyof LinkedFileData> = Required<BinaryFile<T>>

services/web/frontend/js/features/full-project-on-client/import-overleaf-editor-core-for-type-check.ts

@@ -1 +0,0 @@
-import 'overleaf-editor-core'

services/web/frontend/js/features/ide-react/context/react-context-root.tsx

@@ -22,6 +22,7 @@ import { UserSettingsProvider } from '@/shared/context/user-settings-context'
 import { PermissionsProvider } from '@/features/ide-react/context/permissions-context'
 import { FileTreeOpenProvider } from '@/features/ide-react/context/file-tree-open-context'
 import { OutlineProvider } from '@/features/ide-react/context/outline-context'
+import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context'
 
 export const ReactContextRoot: FC = ({ children }) => {
   return (
@@ -32,39 +33,41 @@ export const ReactContextRoot: FC = ({ children }) => {
             <UserProvider>
               <UserSettingsProvider>
                 <ProjectProvider>
-                  <FileTreeDataProvider>
-                    <FileTreePathProvider>
-                      <ReferencesProvider>
-                        <DetachProvider>
-                          <EditorProvider>
-                            <PermissionsProvider>
-                              <ProjectSettingsProvider>
-                                <LayoutProvider>
-                                  <EditorManagerProvider>
-                                    <LocalCompileProvider>
-                                      <DetachCompileProvider>
-                                        <ChatProvider>
-                                          <FileTreeOpenProvider>
-                                            <OnlineUsersProvider>
-                                              <MetadataProvider>
-                                                <OutlineProvider>
-                                                  {children}
-                                                </OutlineProvider>
-                                              </MetadataProvider>
-                                            </OnlineUsersProvider>
-                                          </FileTreeOpenProvider>
-                                        </ChatProvider>
-                                      </DetachCompileProvider>
-                                    </LocalCompileProvider>
-                                  </EditorManagerProvider>
-                                </LayoutProvider>
-                              </ProjectSettingsProvider>
-                            </PermissionsProvider>
-                          </EditorProvider>
-                        </DetachProvider>
-                      </ReferencesProvider>
-                    </FileTreePathProvider>
-                  </FileTreeDataProvider>
+                  <SnapshotProvider>
+                    <FileTreeDataProvider>
+                      <FileTreePathProvider>
+                        <ReferencesProvider>
+                          <DetachProvider>
+                            <EditorProvider>
+                              <PermissionsProvider>
+                                <ProjectSettingsProvider>
+                                  <LayoutProvider>
+                                    <EditorManagerProvider>
+                                      <LocalCompileProvider>
+                                        <DetachCompileProvider>
+                                          <ChatProvider>
+                                            <FileTreeOpenProvider>
+                                              <OnlineUsersProvider>
+                                                <MetadataProvider>
+                                                  <OutlineProvider>
+                                                    {children}
+                                                  </OutlineProvider>
+                                                </MetadataProvider>
+                                              </OnlineUsersProvider>
+                                            </FileTreeOpenProvider>
+                                          </ChatProvider>
+                                        </DetachCompileProvider>
+                                      </LocalCompileProvider>
+                                    </EditorManagerProvider>
+                                  </LayoutProvider>
+                                </ProjectSettingsProvider>
+                              </PermissionsProvider>
+                            </EditorProvider>
+                          </DetachProvider>
+                        </ReferencesProvider>
+                      </FileTreePathProvider>
+                    </FileTreeDataProvider>
+                  </SnapshotProvider>
                 </ProjectProvider>
               </UserSettingsProvider>
             </UserProvider>

services/web/frontend/js/features/ide-react/context/snapshot-context.tsx

@@ -0,0 +1,127 @@
+import {
+  createContext,
+  FC,
+  useContext,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react'
+import { Snapshot } from 'overleaf-editor-core'
+import { useProjectContext } from '@/shared/context/project-context'
+import { debugConsole } from '@/utils/debugging'
+import importOverleafModules from '../../../../macros/import-overleaf-module.macro'
+import { Folder } from '../../../../../types/folder'
+
+export const StubSnapshotUtils = {
+  SnapshotUpdater: class SnapshotUpdater {
+    // eslint-disable-next-line no-useless-constructor
+    constructor(readonly projectId: string) {}
+    refresh(): Promise<{ snapshot: Snapshot; snapshotVersion: number }> {
+      throw new Error('not implemented')
+    }
+
+    abort(): void {
+      throw new Error('not implemented')
+    }
+  },
+  buildFileTree(snapshot: Snapshot): Folder {
+    throw new Error('not implemented')
+  },
+  createFolder(_id: string, name: string): Folder {
+    throw new Error('not implemented')
+  },
+}
+
+const { SnapshotUpdater } =
+  (importOverleafModules('snapshotUtils')[0]
+    ?.import as typeof StubSnapshotUtils) || StubSnapshotUtils
+
+export type SnapshotLoadingState = '' | 'loading' | 'error'
+
+export const SnapshotContext = createContext<
+  | {
+      snapshotVersion: number
+      snapshot?: Snapshot
+      snapshotLoadingState: SnapshotLoadingState
+
+      fileTreeFromHistory: boolean
+      setFileTreeFromHistory: (v: boolean) => void
+    }
+  | undefined
+>(undefined)
+
+export const SnapshotProvider: FC = ({ children }) => {
+  const { _id: projectId } = useProjectContext()
+  const [snapshotLoadingState, setSnapshotLoadingState] =
+    useState<SnapshotLoadingState>('')
+  const [snapshotUpdater] = useState(() => new SnapshotUpdater(projectId))
+  const [snapshot, setSnapshot] = useState<Snapshot>()
+  const [snapshotVersion, setSnapshotVersion] = useState(-1)
+  const [fileTreeFromHistory, setFileTreeFromHistory] = useState(false)
+
+  useEffect(() => {
+    if (!fileTreeFromHistory) return
+
+    let stop = false
+    let handle: number
+    const refresh = () => {
+      setSnapshotLoadingState('loading')
+      snapshotUpdater
+        .refresh()
+        .then(({ snapshot, snapshotVersion }) => {
+          setSnapshot(snapshot)
+          setSnapshotVersion(snapshotVersion)
+          setSnapshotLoadingState('')
+        })
+        .catch(err => {
+          debugConsole.error(err)
+          setSnapshotLoadingState('error')
+        })
+        .finally(() => {
+          if (stop) return
+          // use a chain of timeouts to avoid concurrent updates
+          handle = window.setTimeout(refresh, 30_000)
+        })
+    }
+
+    refresh()
+    return () => {
+      stop = true
+      snapshotUpdater.abort()
+      clearInterval(handle)
+    }
+  }, [projectId, fileTreeFromHistory, snapshotUpdater])
+
+  const value = useMemo(
+    () => ({
+      snapshot,
+      snapshotVersion,
+      snapshotLoadingState,
+      fileTreeFromHistory,
+      setFileTreeFromHistory,
+    }),
+    [
+      snapshot,
+      snapshotVersion,
+      snapshotLoadingState,
+      fileTreeFromHistory,
+      setFileTreeFromHistory,
+    ]
+  )
+
+  return (
+    <SnapshotContext.Provider value={value}>
+      {children}
+    </SnapshotContext.Provider>
+  )
+}
+
+export function useSnapshotContext() {
+  const context = useContext(SnapshotContext)
+  if (!context) {
+    throw new Error(
+      'useSnapshotContext is only available within SnapshotProvider'
+    )
+  }
+  return context
+}

services/web/frontend/js/features/ide-react/util/file-view.ts

@@ -10,6 +10,7 @@ export function convertFileRefToBinaryFile(fileRef: FileRef): BinaryFile {
     selected: true,
     linkedFileData: fileRef.linkedFileData,
     created: fileRef.created ? new Date(fileRef.created) : new Date(),
+    hash: fileRef.hash,
   }
 }
 

services/web/frontend/js/infrastructure/fetch-json.ts

@@ -244,3 +244,10 @@ export function getUserFacingMessage(error: Error | null) {
 
   return error.message
 }
+
+export function isRateLimited(error?: Error | FetchError | any) {
+  if (error && error instanceof FetchError) {
+    return error.response?.status === 429
+  }
+  return false
+}

services/web/frontend/js/shared/context/file-tree-data-context.tsx

@@ -6,6 +6,7 @@ import {
   useMemo,
   useState,
   FC,
+  useEffect,
 } from 'react'
 import useScopeValue from '../hooks/use-scope-value'
 import {
@@ -22,6 +23,14 @@ import { Folder } from '../../../../types/folder'
 import { Project } from '../../../../types/project'
 import { MainDocument } from '../../../../types/project-settings'
 import { FindResult } from '@/features/file-tree/util/path'
+import {
+  StubSnapshotUtils,
+  useSnapshotContext,
+} from '@/features/ide-react/context/snapshot-context'
+import importOverleafModules from '../../../macros/import-overleaf-module.macro'
+const { buildFileTree, createFolder } =
+  (importOverleafModules('snapshotUtils')[0]
+    ?.import as typeof StubSnapshotUtils) || StubSnapshotUtils
 
 const FileTreeDataContext = createContext<
   | {
@@ -170,8 +179,29 @@ export const FileTreeDataProvider: FC = ({ children }) => {
   const [project] = useScopeValue<Project>('project')
   const [openDocId] = useScopeValue('editor.open_doc_id')
   const [, setOpenDocName] = useScopeValueSetterOnly('editor.open_doc_name')
+  const { fileTreeFromHistory, snapshot, snapshotVersion } =
+    useSnapshotContext()
+
+  const [rootFolder, setRootFolder] = useState(project?.rootFolder)
+
+  useEffect(() => {
+    if (fileTreeFromHistory) return
+    setRootFolder(project?.rootFolder)
+  }, [project, fileTreeFromHistory])
+
+  useEffect(() => {
+    if (!fileTreeFromHistory) return
+    if (!rootFolder || rootFolder?.[0]?._id) {
+      // Init or replace mongo rootFolder with stub while we load the snapshot.
+      // In the future, project:joined should only fire once the snapshot is ready.
+      setRootFolder([createFolder('', '')])
+    }
+  }, [fileTreeFromHistory, rootFolder])
 
-  const { rootFolder } = project || {}
+  useEffect(() => {
+    if (!fileTreeFromHistory || !snapshot) return
+    setRootFolder([buildFileTree(snapshot)])
+  }, [fileTreeFromHistory, snapshot, snapshotVersion])
 
   const [{ fileTreeData, fileCount }, dispatch] = useReducer(
     fileTreeMutableReducer,

services/web/frontend/stories/decorators/scope.tsx

@@ -42,6 +42,7 @@ import { EditorManagerProvider } from '@/features/ide-react/context/editor-manag
 import { FileTreeOpenProvider } from '@/features/ide-react/context/file-tree-open-context'
 import { MetadataProvider } from '@/features/ide-react/context/metadata-context'
 import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context'
+import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context'
 
 const scopeWatchers: [string, (value: any) => void][] = []
 
@@ -73,7 +74,7 @@ const initialize = () => {
           { _id: 'test-file-id', name: 'testfile.tex' },
           { _id: 'test-bib-file-id', name: 'testsources.bib' },
         ],
-        fileRefs: [{ _id: 'test-image-id', name: 'frog.jpg' }],
+        fileRefs: [{ _id: 'test-image-id', name: 'frog.jpg', hash: '42' }],
         folders: [],
       },
     ],
@@ -179,6 +180,7 @@ export const ScopeDecorator = (
     DetachProvider,
     EditorProvider,
     EditorManagerProvider,
+    SnapshotProvider,
     FileTreeDataProvider,
     FileTreeOpenProvider,
     FileTreePathProvider,
@@ -207,39 +209,41 @@ export const ScopeDecorator = (
             <Providers.UserProvider>
               <Providers.UserSettingsProvider>
                 <Providers.ProjectProvider>
-                  <Providers.FileTreeDataProvider>
-                    <Providers.FileTreePathProvider>
-                      <Providers.ReferencesProvider>
-                        <Providers.DetachProvider>
-                          <Providers.EditorProvider>
-                            <Providers.PermissionsProvider>
-                              <Providers.ProjectSettingsProvider>
-                                <Providers.LayoutProvider>
-                                  <Providers.EditorManagerProvider>
-                                    <Providers.LocalCompileProvider>
-                                      <Providers.DetachCompileProvider>
-                                        <Providers.ChatProvider>
-                                          <Providers.FileTreeOpenProvider>
-                                            <Providers.OnlineUsersProvider>
-                                              <Providers.MetadataProvider>
-                                                <Providers.OutlineProvider>
-                                                  <Story />
-                                                </Providers.OutlineProvider>
-                                              </Providers.MetadataProvider>
-                                            </Providers.OnlineUsersProvider>
-                                          </Providers.FileTreeOpenProvider>
-                                        </Providers.ChatProvider>
-                                      </Providers.DetachCompileProvider>
-                                    </Providers.LocalCompileProvider>
-                                  </Providers.EditorManagerProvider>
-                                </Providers.LayoutProvider>
-                              </Providers.ProjectSettingsProvider>
-                            </Providers.PermissionsProvider>
-                          </Providers.EditorProvider>
-                        </Providers.DetachProvider>
-                      </Providers.ReferencesProvider>
-                    </Providers.FileTreePathProvider>
-                  </Providers.FileTreeDataProvider>
+                  <Providers.SnapshotProvider>
+                    <Providers.FileTreeDataProvider>
+                      <Providers.FileTreePathProvider>
+                        <Providers.ReferencesProvider>
+                          <Providers.DetachProvider>
+                            <Providers.EditorProvider>
+                              <Providers.PermissionsProvider>
+                                <Providers.ProjectSettingsProvider>
+                                  <Providers.LayoutProvider>
+                                    <Providers.EditorManagerProvider>
+                                      <Providers.LocalCompileProvider>
+                                        <Providers.DetachCompileProvider>
+                                          <Providers.ChatProvider>
+                                            <Providers.FileTreeOpenProvider>
+                                              <Providers.OnlineUsersProvider>
+                                                <Providers.MetadataProvider>
+                                                  <Providers.OutlineProvider>
+                                                    <Story />
+                                                  </Providers.OutlineProvider>
+                                                </Providers.MetadataProvider>
+                                              </Providers.OnlineUsersProvider>
+                                            </Providers.FileTreeOpenProvider>
+                                          </Providers.ChatProvider>
+                                        </Providers.DetachCompileProvider>
+                                      </Providers.LocalCompileProvider>
+                                    </Providers.EditorManagerProvider>
+                                  </Providers.LayoutProvider>
+                                </Providers.ProjectSettingsProvider>
+                              </Providers.PermissionsProvider>
+                            </Providers.EditorProvider>
+                          </Providers.DetachProvider>
+                        </Providers.ReferencesProvider>
+                      </Providers.FileTreePathProvider>
+                    </Providers.FileTreeDataProvider>
+                  </Providers.SnapshotProvider>
                 </Providers.ProjectProvider>
               </Providers.UserSettingsProvider>
             </Providers.UserProvider>

services/web/locales/en.json

@@ -711,6 +711,7 @@
   "full_doc_history": "Full document history",
   "full_doc_history_info_v2": "You can see all the edits in your project and who made every change. Add labels to quickly access specific versions.",
   "full_document_history": "Full document <0>history</0>",
+  "full_project_search": "Full Project Search",
   "full_width": "Full width",
   "gallery": "Gallery",
   "gallery_find_more": "Find More __itemPlural__",
@@ -1234,6 +1235,7 @@
   "my_library": "My Library",
   "n_items": "__count__ item",
   "n_items_plural": "__count__ items",
+  "n_matches": "__n__ matches",
   "n_more_updates_above": "__count__ more update above",
   "n_more_updates_above_plural": "__count__ more updates above",
   "n_more_updates_below": "__count__ more update below",
@@ -1612,6 +1614,7 @@
   "registered": "Registered",
   "registering": "Registering",
   "registration_error": "Registration error",
+  "regular_expression": "Regular Expression",
   "reject": "Reject",
   "reject_all": "Reject all",
   "reject_change": "Reject change",
@@ -1725,6 +1728,7 @@
   "saving_20_percent_no_exclamation": "Saving 20%",
   "saving_notification_with_seconds": "Saving __docname__... (__seconds__ seconds of unsaved changes)",
   "search": "Search",
+  "search_all_project_files": "Search all project files",
   "search_bib_files": "Search by author, title, year",
   "search_by_citekey_author_year_title": "Search by citation key, author, title, year",
   "search_command_find": "Find",
@@ -1744,6 +1748,7 @@
   "search_replace_all": "Replace All",
   "search_replace_with": "Replace with",
   "search_search_for": "Search for",
+  "search_terms": "Search terms",
   "search_whole_word": "Whole word",
   "search_within_selection": "Within selection",
   "searched_path_for_lines_containing": "Searched __path__ for lines containing \"__query__\"",

services/web/package.json

@@ -298,6 +298,7 @@
     "eslint-plugin-react": "^7.32.2",
     "eslint-plugin-react-hooks": "^4.6.0",
     "events": "^3.3.0",
+    "fake-indexeddb": "^6.0.0",
     "fetch-mock": "^9.10.2",
     "formik": "^2.2.9",
     "glob": "^7.1.6",

services/web/test/frontend/bootstrap.js

@@ -92,3 +92,6 @@ addHook(() => '', { exts: ['.css'], ignoreNodeModules: false })
 globalThis.HTMLElement.prototype.scrollIntoView = () => {}
 
 globalThis.DOMParser = window.DOMParser
+
+// Polyfill for IndexedDB
+require('fake-indexeddb/auto')

services/web/test/frontend/features/file-tree/util/path.test.ts

@@ -34,6 +34,7 @@ describe('Path utils', function () {
             {
               _id: 'test-file-in-folder',
               name: 'example.png',
+              hash: '42',
             },
           ],
           folders: [
@@ -50,6 +51,7 @@ describe('Path utils', function () {
                 {
                   _id: 'test-file-in-subfolder',
                   name: 'nested-example.png',
+                  hash: '43',
                 },
               ],
               folders: [],
@@ -61,10 +63,12 @@ describe('Path utils', function () {
         {
           _id: 'test-image-file',
           name: 'frog.jpg',
+          hash: '21',
         },
         {
           _id: 'uppercase-extension-image-file',
           name: 'frog.JPG',
+          hash: '22',
         },
       ],
     }

services/web/test/frontend/features/file-view/components/file-view-image.test.jsx

@@ -7,6 +7,7 @@ describe('<FileViewImage />', function () {
   const file = {
     id: '60097ca20454610027c442a8',
     name: 'file.jpg',
+    hash: 'hash',
     linkedFileData: {
       source_entity_path: '/source-entity-path',
       provider: 'project_file',
@@ -15,12 +16,7 @@ describe('<FileViewImage />', function () {
 
   it('renders an image', function () {
     renderWithEditorContext(
-      <FileViewImage
-        fileName={file.name}
-        fileId={file.id}
-        onError={() => {}}
-        onLoad={() => {}}
-      />
+      <FileViewImage file={file} onError={() => {}} onLoad={() => {}} />
     )
     screen.getByRole('img')
   })

services/web/test/frontend/features/file-view/components/file-view-refresh-error.test.tsx

@@ -16,6 +16,7 @@ describe('<FileViewRefreshError />', function () {
       name: 'frog.jpg',
       type: 'file',
       selected: true,
+      hash: '42',
     }
 
     render(

services/web/test/frontend/features/file-view/components/file-view-text.test.jsx

@@ -6,6 +6,8 @@ import FileViewText from '../../../../../frontend/js/features/file-view/componen
 
 describe('<FileViewText/>', function () {
   const file = {
+    id: '123',
+    hash: '1234',
     name: 'example.tex',
     linkedFileData: {
       v1_source_doc_id: 'v1-source-id',

services/web/test/frontend/features/source-editor/components/codemirror-editor-autocomplete.spec.tsx

@@ -44,6 +44,7 @@ describe('autocomplete', { scrollBehavior: false }, function () {
               {
                 _id: 'test-file-in-folder',
                 name: 'example.png',
+                hash: '42',
               },
             ],
             folders: [],
@@ -53,10 +54,12 @@ describe('autocomplete', { scrollBehavior: false }, function () {
           {
             _id: 'test-image-file',
             name: 'frog.jpg',
+            hash: '21',
           },
           {
             _id: 'uppercase-extension-image-file',
             name: 'frog.JPG',
+            hash: '22',
           },
         ],
       },
@@ -194,6 +197,7 @@ describe('autocomplete', { scrollBehavior: false }, function () {
               {
                 _id: 'test-file-in-folder',
                 name: 'example.png',
+                hash: '42',
               },
             ],
             folders: [],
@@ -203,6 +207,7 @@ describe('autocomplete', { scrollBehavior: false }, function () {
           {
             _id: 'test-image-file',
             name: 'frog.jpg',
+            hash: '43',
           },
         ],
       },

services/web/test/frontend/features/source-editor/helpers/mock-scope.ts

@@ -41,10 +41,12 @@ export const mockScope = (content?: string) => {
                 {
                   _id: figureId,
                   name: 'frog.jpg',
+                  hash: '42',
                 },
                 {
                   _id: 'fake-figure-id',
                   name: 'unicorn.png',
+                  hash: '43',
                 },
               ],
             },

services/web/test/frontend/helpers/editor-providers.jsx

@@ -34,6 +34,7 @@ import { ModalsContextProvider } from '@/features/ide-react/context/modals-conte
 import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context'
 import { PermissionsProvider } from '@/features/ide-react/context/permissions-context'
 import { ReferencesProvider } from '@/features/ide-react/context/references-context'
+import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context'
 
 // these constants can be imported in tests instead of
 // using magic strings
@@ -159,6 +160,7 @@ export function EditorProviders({
     DetachProvider,
     EditorProvider,
     EditorManagerProvider,
+    SnapshotProvider,
     FileTreeDataProvider,
     FileTreeOpenProvider,
     FileTreePathProvider,
@@ -187,39 +189,41 @@ export function EditorProviders({
             <Providers.UserProvider>
               <Providers.UserSettingsProvider>
                 <Providers.ProjectProvider>
-                  <Providers.FileTreeDataProvider>
-                    <Providers.FileTreePathProvider>
-                      <Providers.ReferencesProvider>
-                        <Providers.DetachProvider>
-                          <Providers.EditorProvider>
-                            <Providers.PermissionsProvider>
-                              <Providers.ProjectSettingsProvider>
-                                <Providers.LayoutProvider>
-                                  <Providers.EditorManagerProvider>
-                                    <Providers.LocalCompileProvider>
-                                      <Providers.DetachCompileProvider>
-                                        <Providers.ChatProvider>
-                                          <Providers.FileTreeOpenProvider>
-                                            <Providers.OnlineUsersProvider>
-                                              <Providers.MetadataProvider>
-                                                <Providers.OutlineProvider>
-                                                  {children}
-                                                </Providers.OutlineProvider>
-                                              </Providers.MetadataProvider>
-                                            </Providers.OnlineUsersProvider>
-                                          </Providers.FileTreeOpenProvider>
-                                        </Providers.ChatProvider>
-                                      </Providers.DetachCompileProvider>
-                                    </Providers.LocalCompileProvider>
-                                  </Providers.EditorManagerProvider>
-                                </Providers.LayoutProvider>
-                              </Providers.ProjectSettingsProvider>
-                            </Providers.PermissionsProvider>
-                          </Providers.EditorProvider>
-                        </Providers.DetachProvider>
-                      </Providers.ReferencesProvider>
-                    </Providers.FileTreePathProvider>
-                  </Providers.FileTreeDataProvider>
+                  <Providers.SnapshotProvider>
+                    <Providers.FileTreeDataProvider>
+                      <Providers.FileTreePathProvider>
+                        <Providers.ReferencesProvider>
+                          <Providers.DetachProvider>
+                            <Providers.EditorProvider>
+                              <Providers.PermissionsProvider>
+                                <Providers.ProjectSettingsProvider>
+                                  <Providers.LayoutProvider>
+                                    <Providers.EditorManagerProvider>
+                                      <Providers.LocalCompileProvider>
+                                        <Providers.DetachCompileProvider>
+                                          <Providers.ChatProvider>
+                                            <Providers.FileTreeOpenProvider>
+                                              <Providers.OnlineUsersProvider>
+                                                <Providers.MetadataProvider>
+                                                  <Providers.OutlineProvider>
+                                                    {children}
+                                                  </Providers.OutlineProvider>
+                                                </Providers.MetadataProvider>
+                                              </Providers.OnlineUsersProvider>
+                                            </Providers.FileTreeOpenProvider>
+                                          </Providers.ChatProvider>
+                                        </Providers.DetachCompileProvider>
+                                      </Providers.LocalCompileProvider>
+                                    </Providers.EditorManagerProvider>
+                                  </Providers.LayoutProvider>
+                                </Providers.ProjectSettingsProvider>
+                              </Providers.PermissionsProvider>
+                            </Providers.EditorProvider>
+                          </Providers.DetachProvider>
+                        </Providers.ReferencesProvider>
+                      </Providers.FileTreePathProvider>
+                    </Providers.FileTreeDataProvider>
+                  </Providers.SnapshotProvider>
                 </Providers.ProjectProvider>
               </Providers.UserSettingsProvider>
             </Providers.UserProvider>

services/web/types/file-ref.ts

@@ -3,4 +3,5 @@ export type FileRef = {
   name: string
   created?: string
   linkedFileData?: any
+  hash: string
 }