Merge pull request #15061 from overleaf/bg-server-pro-migrate-emails-script

add migrate emails script for server pro

GitOrigin-RevId: be5fc166554d08100de175133d413ecb1a29623a

services/web/app/src/Features/User/UserUpdater.js

@@ -234,6 +234,58 @@ async function setDefaultEmailAddress(
   }
 }
 
+/**
+ * Overwrites the primary email address of a user in the database in-place.
+ * This function is only intended for use in scripts to migrate email addresses
+ * where we do not want to trigger all the actions that happen when a user
+ * changes their own email.  It should not be used in any other circumstances.
+ */
+async function migrateDefaultEmailAddress(
+  userId,
+  oldEmail,
+  newEmail,
+  auditLog
+) {
+  oldEmail = EmailHelper.parseEmail(oldEmail)
+  if (oldEmail == null) {
+    throw new Error('invalid old email')
+  }
+  newEmail = EmailHelper.parseEmail(newEmail)
+  if (newEmail == null) {
+    throw new Error('invalid new email')
+  }
+  const reversedHostname = newEmail.split('@')[1].split('').reverse().join('')
+  const query = {
+    _id: userId,
+    email: oldEmail,
+    'emails.email': oldEmail,
+  }
+  const update = {
+    $set: {
+      email: newEmail,
+      'emails.$.email': newEmail,
+      'emails.$.reversedHostname': reversedHostname,
+    },
+  }
+  const result = await updateUser(query, update)
+  if (result.modifiedCount !== 1) {
+    throw new Error('email update error')
+  }
+  // add a user audit log entry for the email change
+  await UserAuditLogHandler.promises.addEntry(
+    userId,
+    'migrate-default-email',
+    auditLog.initiatorId,
+    auditLog.ipAddress,
+    {
+      oldEmail,
+      newEmail,
+      // Add optional extra info
+      ...(auditLog.extraInfo || {}),
+    }
+  )
+}
+
 async function confirmEmail(userId, email, affiliationOptions) {
   // used for initial email confirmation (non-SSO and SSO)
   // also used for reconfirmation of non-SSO emails
@@ -492,6 +544,7 @@ module.exports = {
   removeEmailAddress: callbackify(removeEmailAddress),
   removeReconfirmFlag: callbackify(removeReconfirmFlag),
   setDefaultEmailAddress: callbackify(setDefaultEmailAddress),
+  migrateDefaultEmailAddress: callbackify(migrateDefaultEmailAddress),
   updateUser: callbackify(updateUser),
   promises: {
     addAffiliationForNewUser,
@@ -502,6 +555,7 @@ module.exports = {
     removeEmailAddress,
     removeReconfirmFlag,
     setDefaultEmailAddress,
+    migrateDefaultEmailAddress,
     updateUser,
   },
 }

services/web/modules/server-ce-scripts/scripts/migrate-user-emails.js

@@ -0,0 +1,199 @@
+// Script to migrate user emails using a CSV file with the following format:
+//
+//  oldEmail,newEmail
+//
+// The script will iterate through the CSV file and update the user's email
+// address from oldEmail to newEmail, after checking all the email addresses
+// for duplicates.
+//
+// Intended for Server Pro customers migrating user emails from one domain to
+// another.
+
+const minimist = require('minimist')
+const { waitForDb } = require('../../../app/src/infrastructure/mongodb')
+
+const os = require('os')
+const fs = require('fs')
+const csv = require('csv/sync')
+const { parseEmail } = require('../../../app/src/Features/Helpers/EmailHelper')
+const UserGetter = require('../../../app/src/Features/User/UserGetter')
+const UserUpdater = require('../../../app/src/Features/User/UserUpdater')
+const UserSessionsManager = require('../../../app/src/Features/User/UserSessionsManager')
+
+const hostname = os.hostname()
+const scriptTimestamp = new Date().toISOString()
+
+// support command line option of --commit to actually do the migration
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['commit', 'ignore-missing'],
+  string: ['admin-id'],
+  alias: {
+    'ignore-missing': 'continue',
+  },
+  default: {
+    commit: false,
+    'ignore-missing': false,
+    'admin-id': '000000000000000000000000', // use a dummy admin ID for script audit log entries
+  },
+})
+
+// display usage if no CSV file is provided
+if (argv._.length === 0) {
+  console.log(
+    'Usage: node migrate_user_emails.js [--commit] [--continue|--ignore-missing] [--admin-id=ADMIN_USER_ID] <csv_file>'
+  )
+  console.log('  --commit: actually do the migration (default: false)')
+  console.log(
+    '  --continue|--ignore-missing: continue on missing or already-migrated users'
+  )
+  console.log('  --admin-id: admin user ID to use for audit log entries')
+  console.log('  <csv_file>: CSV file with old and new email addresses')
+  process.exit(1)
+}
+
+function filterEmails(rows) {
+  // check that emails have a valid format
+  const result = []
+  const seenOld = new Set()
+  const seenNew = new Set()
+  for (const [oldEmail, newEmail] of rows) {
+    const parsedOld = parseEmail(oldEmail)
+    const parsedNew = parseEmail(newEmail)
+    if (!parsedOld) {
+      throw new Error(`invalid old email "${oldEmail}"`)
+    }
+    if (!parsedNew) {
+      throw new Error(`invalid new email "${newEmail}"`)
+    }
+    // Check for duplicates and overlaps
+    if (seenOld.has(parsedOld)) {
+      throw new Error(`Duplicate old emails found in CSV file ${oldEmail}.`)
+    }
+    if (seenNew.has(parsedNew)) {
+      throw new Error(`Duplicate new emails found in CSV file ${newEmail}.`)
+    }
+    if (seenOld.has(parsedNew) || seenNew.has(parsedOld)) {
+      throw new Error(
+        `Old and new emails cannot overlap ${oldEmail} ${newEmail}`
+      )
+    }
+    seenOld.add(parsedOld)
+    seenNew.add(parsedNew)
+    result.push([parsedOld, parsedNew])
+  }
+  return result
+}
+
+async function checkEmailsAgainstDb(emails) {
+  const result = []
+  for (const [oldEmail, newEmail] of emails) {
+    const userWithEmail = await UserGetter.promises.getUserByMainEmail(
+      oldEmail,
+      {
+        _id: 1,
+      }
+    )
+    if (!userWithEmail) {
+      if (argv['ignore-missing']) {
+        console.log(
+          `User with email "${oldEmail}" not found, skipping update to "${newEmail}"`
+        )
+        continue
+      } else {
+        throw new Error(`no user found with email "${oldEmail}"`)
+      }
+    }
+    const userWithNewEmail = await UserGetter.promises.getUserByAnyEmail(
+      newEmail,
+      {
+        _id: 1,
+      }
+    )
+    if (userWithNewEmail) {
+      throw new Error(
+        `new email "${newEmail}" already exists for user ${userWithNewEmail._id}`
+      )
+    }
+    result.push([oldEmail, newEmail])
+  }
+  return result
+}
+
+async function doMigration(emails) {
+  let success = 0
+  let failure = 0
+  let skipped = 0
+  for (const [oldEmail, newEmail] of emails) {
+    const userWithEmail = await UserGetter.promises.getUserByMainEmail(
+      oldEmail,
+      {
+        _id: 1,
+      }
+    )
+    if (!userWithEmail) {
+      if (argv['ignore-missing']) {
+        continue
+      } else {
+        throw new Error(`no user found with email "${oldEmail}"`)
+      }
+    }
+    if (argv.commit) {
+      console.log(
+        `Updating user ${userWithEmail._id} email "${oldEmail}" to "${newEmail}"\n`
+      )
+      try {
+        await UserSessionsManager.promises.revokeAllUserSessions(
+          userWithEmail,
+          [] // log out all the user's sessions before changing the email address
+        )
+
+        await UserUpdater.promises.migrateDefaultEmailAddress(
+          userWithEmail._id,
+          oldEmail,
+          newEmail,
+          {
+            initiatorId: argv['admin-id'],
+            ipAddress: hostname,
+            extraInfo: {
+              script: 'migrate_user_emails.js',
+              runAt: scriptTimestamp,
+            },
+          }
+        )
+        success++
+      } catch (err) {
+        console.log(err)
+        failure++
+      }
+    } else {
+      console.log(`Dry run, skipping update from ${oldEmail} to ${newEmail}`)
+      skipped++
+    }
+  }
+  console.log('Success: ', success, 'Failure: ', failure, 'Skipped: ', skipped)
+  if (failure > 0) {
+    throw new Error('Some email migrations failed')
+  }
+}
+
+async function migrateEmails() {
+  console.log('Starting email migration script')
+  const csvFilePath = argv._[0]
+  const csvFile = fs.readFileSync(csvFilePath, 'utf8')
+  const rows = csv.parse(csvFile)
+  console.log('Number of users to migrate: ', rows.length)
+  await waitForDb()
+  const emails = filterEmails(rows)
+  const existingUserEmails = await checkEmailsAgainstDb(emails)
+  await doMigration(existingUserEmails)
+}
+
+migrateEmails()
+  .then(() => {
+    console.log('Done.')
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error(error)
+    process.exit(1)
+  })

services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js

@@ -3,6 +3,7 @@ const { execSync } = require('child_process')
 const { expect } = require('chai')
 const { db } = require('../../../../../app/src/infrastructure/mongodb')
 const User = require('../../../../../test/acceptance/src/helpers/User').promises
+const fs = require('fs')
 
 /**
  * @param {string} cmd
@@ -131,6 +132,131 @@ describe('ServerCEScripts', function () {
     })
   })
 
+  describe('migrate-user-emails', function () {
+    let usersToMigrate
+    let otherUsers
+    let csv
+    let csvfail
+    beforeEach(async function () {
+      // set up some users to migrate and others to leave alone
+      usersToMigrate = []
+      otherUsers = []
+      for (let i = 0; i < 2; i++) {
+        const user = new User()
+        await user.login()
+        usersToMigrate.push(user)
+      }
+      for (let i = 0; i < 2; i++) {
+        const user = new User()
+        await user.login()
+        otherUsers.push(user)
+      }
+      // write the migration csv to a temporary file
+      const id = usersToMigrate[0]._id
+      csv = `/tmp/migration-${id}.csv`
+      const rows = []
+      for (const user of usersToMigrate) {
+        rows.push(`${user.email},new-${user.email}`)
+      }
+      fs.writeFileSync(csv, rows.join('\n'))
+      // also write a csv with a user that doesn't exist
+      csvfail = `/tmp/migration-fail-${id}.csv`
+      fs.writeFileSync(
+        csvfail,
+        [
+          'nouser@example.com,nouser@other.example.com',
+          ...rows,
+          'foo@example.com,bar@example.com',
+        ].join('\n')
+      )
+    })
+
+    afterEach(function () {
+      // clean up the temporary files
+      fs.unlinkSync(csv)
+      fs.unlinkSync(csvfail)
+    })
+
+    it('should do a dry run by default', async function () {
+      run(
+        `node modules/server-ce-scripts/scripts/migrate-user-emails.js ${csv}`
+      )
+      for (const user of usersToMigrate) {
+        const dbEntry = await user.get()
+        expect(dbEntry.email).to.equal(user.email)
+      }
+      for (const user of otherUsers) {
+        const dbEntry = await user.get()
+        expect(dbEntry.email).to.equal(user.email)
+      }
+    })
+
+    it('should exit with code 0 when successfully migrating user emails', function () {
+      run(
+        `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}`
+      )
+    })
+
+    it('should migrate the user emails with the --commit option', async function () {
+      run(
+        `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}`
+      )
+      for (const user of usersToMigrate) {
+        const dbEntry = await user.get()
+        expect(dbEntry.email).to.equal(`new-${user.email}`)
+        expect(dbEntry.emails).to.have.lengthOf(1)
+        expect(dbEntry.emails[0].email).to.equal(`new-${user.email}`)
+        expect(dbEntry.emails[0].reversedHostname).to.equal('moc.elpmaxe')
+        expect(dbEntry.emails[0].createdAt).to.eql(user.emails[0].createdAt)
+      }
+    })
+
+    it('should leave other user emails unchanged', async function () {
+      run(
+        `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}`
+      )
+      for (const user of otherUsers) {
+        const dbEntry = await user.get()
+        expect(dbEntry.email).to.equal(user.email)
+      }
+    })
+
+    it('should exit with code 1 when there are failures migrating user emails', function () {
+      try {
+        run(
+          `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csvfail}`
+        )
+      } catch (e) {
+        expect(e.status).to.equal(1)
+        return
+      }
+      expect.fail('command should have failed')
+    })
+
+    it('should migrate other users when there are failures with the --continue option', async function () {
+      try {
+        run(
+          `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csvfail}`
+        )
+      } catch (e) {
+        expect(e.status).to.equal(1)
+        run(
+          `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit --continue ${csvfail}`
+        )
+        for (const user of usersToMigrate) {
+          const dbEntry = await user.get()
+          expect(dbEntry.email).to.equal(`new-${user.email}`)
+          expect(dbEntry.emails).to.have.lengthOf(1)
+          expect(dbEntry.emails[0].email).to.equal(`new-${user.email}`)
+          expect(dbEntry.emails[0].reversedHostname).to.equal('moc.elpmaxe')
+          expect(dbEntry.emails[0].createdAt).to.eql(user.emails[0].createdAt)
+        }
+        return
+      }
+      expect.fail('command should have failed')
+    })
+  })
+
   describe('rename-tag', function () {
     let user
     beforeEach(async function () {