Merge pull request #3656 from overleaf/jpa-script-back-fill-doc-name

[scripts] add new script for back filling the names of deleted docs

GitOrigin-RevId: c3a7ad8ba1306728bc1a433bec5dc847651bf94d

services/web/scripts/back_fill_doc_name_for_deleted_docs.js

@@ -0,0 +1,74 @@
+const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
+
+const { batchedUpdate } = require('./helpers/batchedUpdate')
+const { promiseMapWithLimit, promisify } = require('../app/src/util/promises')
+const { db } = require('../app/src/infrastructure/mongodb')
+const sleep = promisify(setTimeout)
+
+const PERFORM_CLEANUP = process.argv.pop() === '--perform-cleanup'
+const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt(
+  process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
+  10
+)
+
+async function main() {
+  await letUserDoubleCheckInputs()
+
+  await batchedUpdate(
+    'projects',
+    // array is not empty ~ array has one item
+    { 'deletedDocs.0': { $exists: true } },
+    processBatch,
+    { _id: 1, deletedDocs: 1 }
+  )
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error({ error })
+    process.exit(1)
+  })
+
+async function processBatch(_, projects) {
+  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
+}
+
+async function processProject(project) {
+  for (const doc of project.deletedDocs) {
+    await backFillDoc(doc)
+  }
+  if (PERFORM_CLEANUP) {
+    await cleanupProject(project)
+  }
+}
+
+async function backFillDoc(doc) {
+  const { name, deletedAt } = doc
+  await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
+}
+
+async function cleanupProject(project) {
+  await db.projects.updateOne(
+    { _id: project._id },
+    { $set: { deletedDocs: [] } }
+  )
+}
+
+async function letUserDoubleCheckInputs() {
+  if (PERFORM_CLEANUP) {
+    console.error('BACK FILLING AND PERFORMING CLEANUP')
+  } else {
+    console.error(
+      'BACK FILLING ONLY - You will need to rerun with --perform-cleanup'
+    )
+  }
+  console.error(
+    'Waiting for you to double check inputs for',
+    LET_USER_DOUBLE_CHECK_INPUTS_FOR,
+    'ms'
+  )
+  await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR)
+}

services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.js

@@ -0,0 +1,118 @@
+const { exec } = require('child_process')
+const { promisify } = require('util')
+const { expect } = require('chai')
+const logger = require('logger-sharelatex')
+const { db, ObjectId } = require('../../../app/src/infrastructure/mongodb')
+const User = require('./helpers/User').promises
+
+async function getDeletedDocs(projectId) {
+  return (await db.projects.findOne({ _id: projectId })).deletedDocs
+}
+
+async function setDeletedDocs(projectId, deletedDocs) {
+  await db.projects.updateOne({ _id: projectId }, { $set: { deletedDocs } })
+}
+
+describe('BackFillDocNameForDeletedDocs', function() {
+  let user, projectId1, projectId2, docId1, docId2, docId3
+  beforeEach('create projects', async function() {
+    user = new User()
+    await user.login()
+
+    projectId1 = ObjectId(await user.createProject('project1'))
+    projectId2 = ObjectId(await user.createProject('project2'))
+  })
+  beforeEach('create docs', async function() {
+    docId1 = ObjectId(
+      await user.createDocInProject(projectId1, null, 'doc1.tex')
+    )
+    docId2 = ObjectId(
+      await user.createDocInProject(projectId1, null, 'doc2.tex')
+    )
+    docId3 = ObjectId(
+      await user.createDocInProject(projectId2, null, 'doc3.tex')
+    )
+  })
+  beforeEach('deleted docs', async function() {
+    await user.deleteItemInProject(projectId1, 'doc', docId1)
+    await user.deleteItemInProject(projectId1, 'doc', docId2)
+    await user.deleteItemInProject(projectId2, 'doc', docId3)
+  })
+  beforeEach('insert doc stubs into docs collection', async function() {
+    await db.docs.insertMany([
+      { _id: docId1, deleted: true },
+      { _id: docId2, deleted: true },
+      { _id: docId3, deleted: true }
+    ])
+  })
+  let deletedDocs1, deletedDocs2
+  let deletedAt1, deletedAt2, deletedAt3
+  beforeEach('set deletedDocs details', async function() {
+    deletedAt1 = new Date()
+    deletedAt2 = new Date()
+    deletedAt3 = new Date()
+    deletedDocs1 = [
+      { _id: docId1, name: 'doc1.tex', deletedAt: deletedAt1 },
+      { _id: docId2, name: 'doc2.tex', deletedAt: deletedAt2 }
+    ]
+    deletedDocs2 = [{ _id: docId3, name: 'doc3.tex', deletedAt: deletedAt3 }]
+    await setDeletedDocs(projectId1, deletedDocs1)
+    await setDeletedDocs(projectId2, deletedDocs2)
+  })
+
+  async function runScript(args = []) {
+    let result
+    try {
+      result = await promisify(exec)(
+        ['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1']
+          .concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs'])
+          .concat(args)
+          .join(' ')
+      )
+    } catch (error) {
+      // dump details like exit code, stdErr and stdOut
+      logger.error({ error }, 'script failed')
+      throw error
+    }
+    const { stderr: stdErr, stdout: stdOut } = result
+    expect(stdOut).to.include(projectId1.toString())
+    expect(stdOut).to.include(projectId2.toString())
+
+    expect(stdErr).to.include(`Completed batch ending ${projectId2}`)
+  }
+
+  function checkDocsBackFilled() {
+    it('should back fill names and deletedAt dates into docs', async function() {
+      const docs = await db.docs.find({}).toArray()
+      expect(docs).to.deep.equal([
+        { _id: docId1, deleted: true, name: 'doc1.tex', deletedAt: deletedAt1 },
+        { _id: docId2, deleted: true, name: 'doc2.tex', deletedAt: deletedAt2 },
+        { _id: docId3, deleted: true, name: 'doc3.tex', deletedAt: deletedAt3 }
+      ])
+    })
+  }
+
+  describe('back fill only', function() {
+    beforeEach('run script', runScript)
+
+    checkDocsBackFilled()
+
+    it('should leave the deletedDocs as is', async function() {
+      expect(await getDeletedDocs(projectId1)).to.deep.equal(deletedDocs1)
+      expect(await getDeletedDocs(projectId2)).to.deep.equal(deletedDocs2)
+    })
+  })
+
+  describe('back fill and cleanup', function() {
+    beforeEach('run script with cleanup flag', async function() {
+      await runScript(['--perform-cleanup'])
+    })
+
+    checkDocsBackFilled()
+
+    it('should cleanup the deletedDocs', async function() {
+      expect(await getDeletedDocs(projectId1)).to.deep.equal([])
+      expect(await getDeletedDocs(projectId2)).to.deep.equal([])
+    })
+  })
+})