Merge pull request #8644 from overleaf/bg-archive-track-changes

archive track changes zip files in project when history migration fails

GitOrigin-RevId: 67f2381b0a8f2fb1de82100752031c65808af885

services/track-changes/app/js/ZipManager.js

@@ -148,6 +148,20 @@ async function generateZip(projectId, zipfile) {
     Buffer.from(JSON.stringify(manifest, null, 2)),
     'manifest.json'
   )
+  zipfile.addBuffer(
+    Buffer.from(
+      `History Migration Data
+
+This zip file contains a copy of the raw history for this project.
+
+If this file is present in a project it means the history could not
+be fully recovered or migrated.
+
+A new history should have been created starting at the datestamp of
+this file.`
+    ),
+    'README.txt'
+  )
   zipfile.end()
 }
 

services/web/app/src/Features/Project/ProjectHistoryHandler.js

@@ -125,6 +125,27 @@ const ProjectHistoryHandler = {
     )
   },
 
+  setMigrationArchiveFlag(project_id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    Project.updateOne(
+      { _id: project_id, version: { $exists: true } },
+      {
+        'overleaf.history.zipFileArchivedInProject': true,
+      },
+      function (err, result) {
+        if (err != null) {
+          return callback(err)
+        }
+        if ((result != null ? result.n : undefined) === 0) {
+          return callback(new Error('migration flag not set'))
+        }
+        return callback()
+      }
+    )
+  },
+
   ensureHistoryExistsForProject(project_id, callback) {
     // We can only set a history id for a project that doesn't have one. The
     // history id is cached in the project history service, and changing an

services/web/app/src/models/Project.js

@@ -100,6 +100,7 @@ const ProjectSchema = new Schema({
       display: { type: Boolean },
       upgradedAt: { type: Date },
       allowDowngrade: { type: Boolean },
+      zipFileArchivedInProject: { type: Boolean },
     },
   },
   collabratecUsers: [

services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js

@@ -8,6 +8,7 @@ const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
 const MAX_UPGRADES_TO_ATTEMPT =
   parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
 const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
+const ARCHIVE_ON_FAILURE = process.env.ARCHIVE_ON_FAILURE === 'true'
 // persist fallback in order to keep batchedUpdate in-sync
 process.env.BATCH_SIZE = BATCH_SIZE
 // raise mongo timeout to 1hr if otherwise unspecified
@@ -37,6 +38,7 @@ console.log({
   MAX_FAILURES,
   USE_QUERY_HINT,
   RETRY_FAILED,
+  ARCHIVE_ON_FAILURE,
   PROJECT_ID,
 })
 
@@ -86,6 +88,9 @@ async function processProject(project) {
       project.overleaf.history.conversionFailed ||
       project.overleaf.history.upgradeFailed
     ) {
+      if (project.overleaf.history.zipFileArchivedInProject) {
+        return // always give up if we have uploaded the zipfile to the project
+      }
       if (!RETRY_FAILED) {
         // we don't want to attempt upgrade on projects
         // that have been previously attempted and failed
@@ -123,7 +128,9 @@ async function doUpgradeForNoneWithConversion(project) {
           `converted ${convertedDocCount} large docs to binary files for project ${projectId}`
         )
       }
-      await ProjectHistoryController.migrateProjectHistory(projectIdString)
+      await ProjectHistoryController.migrateProjectHistory(projectIdString, {
+        archiveOnFailure: ARCHIVE_ON_FAILURE,
+      })
     } catch (err) {
       // if migrateProjectHistory fails, it cleans up by deleting
       // the history and unsetting the history id