Merge pull request #19151 from overleaf/em-history-ranges-quick-migration

Add quick history ranges support migration

GitOrigin-RevId: 8446beb6bcd7384c32fc1b216e4b72d8f5d91500

services/docstore/app.js

@@ -50,6 +50,7 @@ app.param('doc_id', function (req, res, next, docId) {
 app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
 app.get('/project/:project_id/doc', HttpController.getAllDocs)
 app.get('/project/:project_id/ranges', HttpController.getAllRanges)
+app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges)
 app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
 app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted)
 app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc)

services/docstore/app/js/DocManager.js

@@ -131,6 +131,25 @@ const DocManager = {
     return docs
   },
 
+  async projectHasRanges(projectId) {
+    const docs = await MongoManager.promises.getProjectsDocs(
+      projectId,
+      {},
+      { _id: 1 }
+    )
+    const docIds = docs.map(doc => doc._id)
+    for (const docId of docIds) {
+      const doc = await DocManager.peekDoc(projectId, docId)
+      if (
+        (doc.ranges?.comments != null && doc.ranges.comments.length > 0) ||
+        (doc.ranges?.changes != null && doc.ranges.changes.length > 0)
+      ) {
+        return true
+      }
+    }
+    return false
+  },
+
   async updateDoc(projectId, docId, lines, version, ranges) {
     const MAX_ATTEMPTS = 2
     for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {

services/docstore/app/js/HttpController.js

@@ -130,6 +130,16 @@ function getAllRanges(req, res, next) {
   )
 }
 
+function projectHasRanges(req, res, next) {
+  const { project_id: projectId } = req.params
+  DocManager.projectHasRanges(projectId, (err, projectHasRanges) => {
+    if (err) {
+      return next(err)
+    }
+    res.json({ projectHasRanges })
+  })
+}
+
 function updateDoc(req, res, next) {
   const { doc_id: docId, project_id: projectId } = req.params
   const lines = req.body?.lines
@@ -298,6 +308,7 @@ module.exports = {
   getAllDocs,
   getAllDeletedDocs,
   getAllRanges,
+  projectHasRanges,
   updateDoc,
   patchDoc,
   archiveAllDocs,

services/web/app/src/Features/Docstore/DocstoreManager.js

@@ -219,6 +219,31 @@ function updateDoc(projectId, docId, lines, version, ranges, callback) {
   )
 }
 
+/**
+ * Asks docstore whether any doc in the project has ranges
+ *
+ * @param {string} proejctId
+ * @param {Callback} callback
+ */
+function projectHasRanges(projectId, callback) {
+  const url = `${settings.apis.docstore.url}/project/${projectId}/has-ranges`
+  request.get({ url, timeout: TIMEOUT, json: true }, (err, res, body) => {
+    if (err) {
+      return callback(err)
+    }
+    if (res.statusCode >= 200 && res.statusCode < 300) {
+      callback(null, body.projectHasRanges)
+    } else {
+      callback(
+        new OError(
+          `docstore api responded with non-success code: ${res.statusCode}`,
+          { projectId }
+        )
+      )
+    }
+  })
+}
+
 function archiveProject(projectId, callback) {
   _operateOnProject(projectId, 'archive', callback)
 }
@@ -266,6 +291,7 @@ module.exports = {
   getDoc,
   isDocDeleted,
   updateDoc,
+  projectHasRanges,
   archiveProject,
   unarchiveProject,
   destroyProject,
@@ -277,6 +303,7 @@ module.exports = {
     getDoc: promisifyMultiResult(getDoc, ['lines', 'rev', 'version', 'ranges']),
     isDocDeleted: promisify(isDocDeleted),
     updateDoc: promisifyMultiResult(updateDoc, ['modified', 'rev']),
+    projectHasRanges: promisify(projectHasRanges),
     archiveProject: promisify(archiveProject),
     unarchiveProject: promisify(unarchiveProject),
     destroyProject: promisify(destroyProject),

services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js

@@ -247,6 +247,46 @@ function resyncProjectHistory(
   )
 }
 
+/**
+ * Block a project from being loaded in docupdater
+ *
+ * @param {string} projectId
+ * @param {Callback} callback
+ */
+function blockProject(projectId, callback) {
+  _makeRequest(
+    { path: `/project/${projectId}/block`, method: 'POST', json: true },
+    projectId,
+    'block-project',
+    (err, body) => {
+      if (err) {
+        return callback(err)
+      }
+      callback(null, body.blocked)
+    }
+  )
+}
+
+/**
+ * Unblock a previously blocked project
+ *
+ * @param {string} projectId
+ * @param {Callback} callback
+ */
+function unblockProject(projectId, callback) {
+  _makeRequest(
+    { path: `/project/${projectId}/unblock`, method: 'POST', json: true },
+    projectId,
+    'unblock-project',
+    (err, body) => {
+      if (err) {
+        return callback(err)
+      }
+      callback(null, body.wasBlocked)
+    }
+  )
+}
+
 function updateProjectStructure(
   projectId,
   projectHistoryId,
@@ -460,6 +500,8 @@ module.exports = {
   reopenThread,
   deleteThread,
   resyncProjectHistory,
+  blockProject,
+  unblockProject,
   updateProjectStructure,
   promises: {
     flushProjectToMongo: promisify(flushProjectToMongo),
@@ -481,6 +523,8 @@ module.exports = {
     reopenThread: promisify(reopenThread),
     deleteThread: promisify(deleteThread),
     resyncProjectHistory: promisify(resyncProjectHistory),
+    blockProject: promisify(blockProject),
+    unblockProject: promisify(unblockProject),
     updateProjectStructure: promisify(updateProjectStructure),
   },
 }

services/web/app/src/Features/History/HistoryRangesSupportMigration.js

@@ -2,8 +2,12 @@
 
 const { callbackify } = require('util')
 const { ObjectId } = require('mongodb')
+const OError = require('@overleaf/o-error')
 const logger = require('@overleaf/logger')
 const HistoryManager = require('../History/HistoryManager')
+const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
+const DocstoreManager = require('../Docstore/DocstoreManager')
+const ProjectOptionsHandler = require('../Project/ProjectOptionsHandler')
 const { db } = require('../../infrastructure/mongodb')
 
 /**
@@ -18,6 +22,7 @@ const { db } = require('../../infrastructure/mongodb')
  * @param {"forwards" | "backwards"} [opts.direction]
  * @param {boolean} [opts.force]
  * @param {boolean} [opts.stopOnError]
+ * @param {boolean} [opts.quickOnly]
  */
 async function migrateProjects(opts = {}) {
   const {
@@ -29,6 +34,7 @@ async function migrateProjects(opts = {}) {
     direction = 'forwards',
     force = false,
     stopOnError = false,
+    quickOnly = false,
   } = opts
 
   const clauses = []
@@ -76,11 +82,22 @@ async function migrateProjects(opts = {}) {
     }
 
     const startTimeMs = Date.now()
+    let quickMigrationSuccess
     try {
-      await migrateProject(projectId, direction)
+      quickMigrationSuccess = await quickMigration(projectId, direction)
+      if (!quickMigrationSuccess) {
+        if (quickOnly) {
+          logger.info(
+            { projectId, direction },
+            'Quick migration failed, skipping project'
+          )
+        } else {
+          await migrateProject(projectId, direction)
+        }
+      }
     } catch (err) {
       logger.error(
-        { projectId, direction, projectsProcessed },
+        { err, projectId, direction, projectsProcessed },
         'Failed to migrate history ranges support'
       )
       projectsProcessed += 1
@@ -93,12 +110,73 @@ async function migrateProjects(opts = {}) {
     const elapsedMs = Date.now() - startTimeMs
     projectsProcessed += 1
     logger.info(
-      { projectId, direction, projectsProcessed, elapsedMs },
+      {
+        projectId,
+        direction,
+        projectsProcessed,
+        elapsedMs,
+        quick: quickMigrationSuccess,
+      },
       'Migrated history ranges support'
     )
   }
 }
 
+/**
+ * Attempt a quick migration (without resync)
+ *
+ * @param {string} projectId
+ * @param {"forwards" | "backwards"} direction
+ * @return {Promise<boolean>} whether or not the quick migration was a success
+ */
+async function quickMigration(projectId, direction = 'forwards') {
+  const blockSuccess =
+    await DocumentUpdaterHandler.promises.blockProject(projectId)
+  if (!blockSuccess) {
+    return false
+  }
+
+  let projectHasRanges
+  try {
+    projectHasRanges =
+      await DocstoreManager.promises.projectHasRanges(projectId)
+  } catch (err) {
+    await DocumentUpdaterHandler.promises.unblockProject(projectId)
+    throw err
+  }
+  if (projectHasRanges) {
+    await DocumentUpdaterHandler.promises.unblockProject(projectId)
+    return false
+  }
+
+  try {
+    await ProjectOptionsHandler.promises.setHistoryRangesSupport(
+      projectId,
+      direction === 'forwards'
+    )
+  } catch (err) {
+    await DocumentUpdaterHandler.promises.unblockProject(projectId)
+    await hardResyncProject(projectId)
+    throw err
+  }
+
+  let wasBlocked
+  try {
+    wasBlocked = await DocumentUpdaterHandler.promises.unblockProject(projectId)
+  } catch (err) {
+    await hardResyncProject(projectId)
+    throw err
+  }
+  if (!wasBlocked) {
+    await hardResyncProject(projectId)
+    throw new OError('Tried to unblock project but it was not blocked', {
+      projectId,
+    })
+  }
+
+  return true
+}
+
 /**
  * Migrate a single project
  *
@@ -112,6 +190,19 @@ async function migrateProject(projectId, direction = 'forwards') {
   })
 }
 
+/**
+ * Hard resync a project
+ *
+ * This is used when something goes wrong with the quick migration after we've
+ * changed the history ranges support flag on a project.
+ *
+ * @param {string} projectId
+ */
+async function hardResyncProject(projectId) {
+  await HistoryManager.promises.flushProject(projectId)
+  await HistoryManager.promises.resyncProject(projectId, { force: true })
+}
+
 module.exports = {
   migrateProjects: callbackify(migrateProjects),
   migrateProject: callbackify(migrateProject),

services/web/scripts/history/migrate_ranges_support.js

@@ -13,6 +13,7 @@ async function main() {
     direction,
     force,
     stopOnError,
+    quickOnly,
   } = parseArgs()
   await HistoryRangesSupportMigration.promises.migrateProjects({
     projectIds,
@@ -23,6 +24,7 @@ async function main() {
     direction,
     force,
     stopOnError,
+    quickOnly,
   })
 }
 
@@ -41,12 +43,13 @@ Options:
     --backwards      Disable history ranges support for selected project ids
     --force          Migrate projects even if they were already migrated
     --stop-on-error  Stop after first migration error
+    --quick-only     Do not try a resync migration if quick migration fails
 `)
 }
 
 function parseArgs() {
   const args = minimist(process.argv.slice(2), {
-    boolean: ['backwards', 'help', 'all', 'force'],
+    boolean: ['backwards', 'help', 'all', 'force', 'quick-only'],
     string: ['owner-id', 'project-id', 'min-id', 'max-id'],
   })
 
@@ -63,6 +66,7 @@ function parseArgs() {
   const maxCount = args['max-count']
   const force = args.force
   const stopOnError = args['stop-on-error']
+  const quickOnly = args['quick-only']
   const all = args.all
 
   if (
@@ -89,6 +93,7 @@ function parseArgs() {
     direction,
     force,
     stopOnError,
+    quickOnly,
   }
 }