Merge pull request #3701 from overleaf/jpa-script-back-fill-deleted-files-stage-2

[scripts] back_fill_deleted_files: optionally fix partial inserts

GitOrigin-RevId: 7d48d311db1af0b38222725ca86cdba8951b16b3

services/web/scripts/back_fill_deleted_files.js

@@ -5,7 +5,8 @@ const { promiseMapWithLimit, promisify } = require('../app/src/util/promises')
 const { db } = require('../app/src/infrastructure/mongodb')
 const sleep = promisify(setTimeout)
 
-const PERFORM_CLEANUP = process.argv.pop() === '--perform-cleanup'
+const PERFORM_CLEANUP = process.argv.includes('--perform-cleanup')
+const FIX_PARTIAL_INSERTS = process.argv.includes('--fix-partial-inserts')
 const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt(
   process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
   10
@@ -50,7 +51,12 @@ async function backFillFiles(project) {
   project.deletedFiles.forEach(file => {
     file.projectId = projectId
   })
-  await db.deletedFiles.insertMany(project.deletedFiles)
+
+  if (FIX_PARTIAL_INSERTS) {
+    await fixPartialInserts(project)
+  } else {
+    await db.deletedFiles.insertMany(project.deletedFiles)
+  }
 }
 
 function filterDuplicatesInPlace(project) {
@@ -63,6 +69,28 @@ function filterDuplicatesInPlace(project) {
   })
 }
 
+async function fixPartialInserts(project) {
+  const seenFileIds = new Set(
+    (
+      await db.deletedFiles
+        .find(
+          { _id: { $in: project.deletedFiles.map(file => file._id) } },
+          { projection: { _id: 1 } }
+        )
+        .toArray()
+    ).map(file => file._id.toString())
+  )
+  project.deletedFiles = project.deletedFiles.filter(file => {
+    const id = file._id.toString()
+    if (seenFileIds.has(id)) return false
+    seenFileIds.add(id)
+    return true
+  })
+  if (project.deletedFiles.length > 0) {
+    await db.deletedFiles.insertMany(project.deletedFiles)
+  }
+}
+
 async function cleanupProject(project) {
   await db.projects.updateOne(
     { _id: project._id },

services/web/test/acceptance/src/BackFillDeletedFilesTests.js

@@ -131,4 +131,32 @@ describe('BackFillDeletedFiles', function() {
       expect(await getDeletedFiles(projectId5)).to.deep.equal([])
     })
   })
+
+  describe('fix partial inserts and cleanup', function() {
+    beforeEach('simulate one missing insert', async function() {
+      await setDeletedFiles(projectId1, [deletedFiles1[0]])
+    })
+    beforeEach('run script with cleanup flag', async function() {
+      await runScript(['--perform-cleanup'])
+    })
+    beforeEach('add case for one missing file', async function() {
+      await setDeletedFiles(projectId1, deletedFiles1)
+    })
+    beforeEach('add cases for no more files to insert', async function() {
+      await setDeletedFiles(projectId2, deletedFiles2)
+      await setDeletedFiles(projectId5, deletedFiles3)
+    })
+
+    beforeEach('fixing partial insert and cleanup', async function() {
+      await runScript(['--fix-partial-inserts', '--perform-cleanup'])
+    })
+
+    checkAreFilesBackFilled()
+
+    it('should cleanup the deletedFiles', async function() {
+      expect(await getDeletedFiles(projectId1)).to.deep.equal([])
+      expect(await getDeletedFiles(projectId2)).to.deep.equal([])
+      expect(await getDeletedFiles(projectId5)).to.deep.equal([])
+    })
+  })
 })