Merge pull request #2963 from overleaf/cmg-archive-state-script

Script to convert old archived state into new array state

GitOrigin-RevId: 64cd3a236dc627f922488e1fa9d8aa2e7df1eb15

services/web/scripts/convert_archived_state.js

@@ -0,0 +1,71 @@
+const _ = require('lodash')
+
+const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
+
+const { batchedUpdate } = require('./helpers/batchedUpdate')
+const { promiseMapWithLimit } = require('../app/src/util/promises')
+
+async function main() {
+  await batchedUpdate(
+    'projects',
+    { archived: false },
+    {
+      $set: { archived: [] },
+      $unset: { trashed: '' } // lest any v1 trashed bits be left behind
+    }
+  )
+
+  console.log('Done, moving to archived projects')
+
+  await batchedUpdate('projects', { archived: true }, performUpdate, {
+    _id: 1,
+    owner_ref: 1,
+    collaberator_refs: 1,
+    readOnly_refs: 1,
+    tokenAccessReadAndWrite_refs: 1,
+    tokenAccessReadOnly_refs: 1
+  })
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error({ error })
+    process.exit(1)
+  })
+
+async function performUpdate(collection, nextBatch) {
+  await promiseMapWithLimit(WRITE_CONCURRENCY, nextBatch, project =>
+    setArchived(collection, project)
+  )
+}
+
+async function setArchived(collection, project) {
+  const archived = calculateArchivedArray(project)
+
+  return collection.updateOne(
+    { _id: project._id },
+    {
+      $set: { archived: archived },
+      $unset: { trashed: '' } // lest any v1 trashed bits be left behind
+    }
+  )
+}
+
+function calculateArchivedArray(project) {
+  return _.unionWith(
+    [project.owner_ref],
+    project.collaberator_refs,
+    project.readOnly_refs,
+    project.tokenAccessReadAndWrite_refs,
+    project.tokenAccessReadOnly_refs,
+    _objectIdEquals
+  )
+}
+
+function _objectIdEquals(firstVal, secondVal) {
+  // For use as a comparator for unionWith
+  return firstVal.toString() === secondVal.toString()
+}

services/web/scripts/helpers/batchedUpdate.js

@@ -8,24 +8,27 @@ if (process.env.BATCH_LAST_ID) {
   BATCH_LAST_ID = ObjectId(process.env.BATCH_LAST_ID)
 }
 
-async function getNextBatch(collection, query, maxId) {
+async function getNextBatch(collection, query, maxId, projection) {
   if (maxId) {
     query['_id'] = { $gt: maxId }
   }
   const entries = await collection
-    .find(query, { _id: 1 })
+    .find(query, projection)
     .sort({ _id: 1 })
     .limit(BATCH_SIZE)
     .setReadPreference(ReadPreference.SECONDARY)
     .toArray()
-  return entries.map(entry => entry._id)
+  return entries
 }
 
 async function performUpdate(collection, nextBatch, update) {
-  return collection.updateMany({ _id: { $in: nextBatch } }, update)
+  return collection.updateMany(
+    { _id: { $in: nextBatch.map(entry => entry._id) } },
+    update
+  )
 }
 
-async function batchedUpdate(collectionName, query, update) {
+async function batchedUpdate(collectionName, query, update, projection) {
   // Apparently the mongo driver returns the connection too early.
   // Some secondary connections are not ready as it returns, leading to
   //  failing cursor actions with a readPreference set to 'secondary'.
@@ -35,14 +38,23 @@ async function batchedUpdate(collectionName, query, update) {
   const db = await getNativeDb()
   const collection = db.collection(collectionName)
 
+  projection = projection || { _id: 1 }
   let nextBatch
   let updated = 0
   let maxId = BATCH_LAST_ID
-  while ((nextBatch = await getNextBatch(collection, query, maxId)).length) {
-    maxId = nextBatch[nextBatch.length - 1]
+  while (
+    (nextBatch = await getNextBatch(collection, query, maxId, projection))
+      .length
+  ) {
+    maxId = nextBatch[nextBatch.length - 1]._id
     updated += nextBatch.length
     console.log(JSON.stringify(nextBatch))
-    await performUpdate(collection, nextBatch, update)
+
+    if (typeof update === 'function') {
+      await update(collection, nextBatch)
+    } else {
+      await performUpdate(collection, nextBatch, update)
+    }
   }
   return updated
 }

services/web/test/acceptance/src/ConvertArchivedState.js

@@ -0,0 +1,119 @@
+const { expect } = require('chai')
+const { exec } = require('child_process')
+const { ObjectId } = require('../../../app/src/infrastructure/mongojs')
+
+const User = require('./helpers/User').promises
+
+describe('ConvertArchivedState', function() {
+  let userOne, userTwo, userThree, userFour
+  let projectOne, projectOneId
+  let projectTwo, projectTwoId
+  let projectThree, projectThreeId
+  let projectFour, projectFourId
+
+  beforeEach(async function() {
+    userOne = new User()
+    userTwo = new User()
+    userThree = new User()
+    userFour = new User()
+    await userOne.login()
+    await userTwo.login()
+    await userThree.login()
+    await userFour.login()
+
+    projectOneId = await userOne.createProject('old-archived-1', {
+      template: 'blank'
+    })
+
+    projectOne = await userOne.getProject(projectOneId)
+    projectOne.archived = true
+    projectOne.collaberator_refs.push(userTwo._id)
+    projectOne.tokenAccessReadOnly_refs.push(userThree._id)
+
+    await userOne.saveProject(projectOne)
+
+    projectTwoId = await userOne.createProject('old-archived-2', {
+      template: 'blank'
+    })
+
+    projectTwo = await userOne.getProject(projectTwoId)
+    projectTwo.archived = true
+    projectTwo.tokenAccessReadAndWrite_refs.push(userThree._id)
+    projectTwo.tokenAccessReadOnly_refs.push(userFour._id)
+
+    await userOne.saveProject(projectTwo)
+
+    projectThreeId = await userOne.createProject('already-new-archived', {
+      template: 'blank'
+    })
+    projectThree = await userOne.getProject(projectThreeId)
+    projectThree.archived = [
+      ObjectId(userOne._id),
+      ObjectId(userTwo._id),
+      ObjectId(userFour._id)
+    ]
+    projectThree.collaberator_refs.push(userTwo._id)
+    projectThree.tokenAccessReadOnly_refs.push(userFour._id)
+
+    await userOne.saveProject(projectThree)
+
+    projectFourId = await userOne.createProject('not-archived', {
+      template: 'blank'
+    })
+    projectFour = await userOne.getProject(projectFourId)
+    projectFour.archived = false
+
+    await userOne.saveProject(projectFour)
+  })
+
+  beforeEach(function(done) {
+    exec('node scripts/convert_archived_state.js', (error, stdout, stderr) => {
+      console.log(stdout)
+      console.error(stderr)
+      if (error) {
+        return done(error)
+      }
+      done()
+    })
+  })
+
+  describe('main method', function() {
+    it('should change a project archived boolean to an array', async function() {
+      projectOne = await userOne.getProject(projectOneId)
+      projectTwo = await userOne.getProject(projectTwoId)
+      expect(convertObjectIdsToStrings(projectOne.archived)).to.deep.equal([
+        userOne._id,
+        userTwo._id,
+        userThree._id
+      ])
+
+      expect(convertObjectIdsToStrings(projectTwo.archived)).to.deep.equal([
+        userOne._id,
+        userThree._id,
+        userFour._id
+      ])
+    })
+
+    it('should not change the value of a project already archived with an array', async function() {
+      projectThree = await userOne.getProject(projectThreeId)
+      expect(convertObjectIdsToStrings(projectThree.archived)).to.deep.equal([
+        userOne._id,
+        userTwo._id,
+        userFour._id
+      ])
+    })
+
+    it('should change a none-archived project with a boolean value to an array', async function() {
+      projectFour = await userOne.getProject(projectFourId)
+      expect(convertObjectIdsToStrings(projectFour.archived)).to.deep.equal([])
+    })
+  })
+
+  function convertObjectIdsToStrings(ids) {
+    if (typeof ids === 'object') {
+      return ids.map(id => {
+        return id.toString()
+      })
+    }
+  }
+})