[web] Create script to re-add wrongly deleted emails (#29550)

* Allow passing auditLog extra info in addEmailAddress

* Create script re_add_deleted_emails.mjs

* Check email validity when parsing the CSV

* Allow no IP and initiator for the log events

* Add an acceptance test on re_add_deleted_emails

* Fix remove_unconfirmed_emails.mjs

We're not planning to run this again, but this could help if some code from here ever gets copy-pasted

* Fix remove_unconfirmed_emails.mjs

Rename `emailsToRemove` to `emailsToRemoveCandidates`

* Remove console.logs of event logs:

That's unnecessary complexity and we already have the data from the previous script

* Copy `addEmailAddress` into the script

* Handle primary emails correctly

* Ignore a TS error

* Fix async function in createUsers (tests)

* Add assertion for already in use emails in ReAddDeletedEmailsTests

* Add information in admin-panel for add-email audit logs

GitOrigin-RevId: f8480426d7b323f88a268874488d9ae641bc6045

services/web/app/src/Features/User/UserAuditLogHandler.mjs

@@ -5,6 +5,7 @@ import { callbackify } from 'node:util'
 import SubscriptionLocator from '../Subscription/SubscriptionLocator.mjs'
 
 function _canHaveNoIpAddressId(operation, info) {
+  if (operation === 'add-email' && info.script) return true
   if (operation === 'join-group-subscription') return true
   if (operation === 'leave-group-subscription') return true
   if (operation === 'must-reset-password-set') return true
@@ -15,6 +16,7 @@ function _canHaveNoIpAddressId(operation, info) {
 }
 
 function _canHaveNoInitiatorId(operation, info) {
+  if (operation === 'add-email' && info.script) return true
   if (operation === 'reset-password') return true
   if (operation === 'unlink-sso' && info.providerId === 'collabratec')
     return true

services/web/app/src/Features/User/UserUpdater.mjs

@@ -101,6 +101,7 @@ async function addEmailAddress(userId, newEmail, affiliationOptions, auditLog) {
     auditLog.initiatorId,
     auditLog.ipAddress,
     {
+      ...auditLog.info,
       newSecondaryEmail: newEmail,
     }
   )

services/web/scripts/re_add_deleted_emails.mjs

@@ -0,0 +1,288 @@
+// @ts-check
+
+import minimist from 'minimist'
+import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
+import fs from 'node:fs/promises'
+import * as csv from 'csv'
+import { promisify } from 'node:util'
+import { scriptRunner } from './lib/ScriptRunner.mjs'
+import Errors from '../app/src/Features/Errors/Errors.js'
+import UserGetter from '../app/src/Features/User/UserGetter.mjs'
+import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js'
+import UserUpdater from '../app/src/Features/User/UserUpdater.mjs'
+import EmailHelper from '../app/src/Features/Helpers/EmailHelper.js'
+import AsyncLocalStorage from '../app/src/infrastructure/AsyncLocalStorage.js'
+import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.mjs'
+import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs'
+import InstitutionsAPI from '../app/src/Features/Institutions/InstitutionsAPI.mjs'
+import OError from '@overleaf/o-error'
+import EmailChangeHelper from '../app/src/Features/Analytics/EmailChangeHelper.mjs'
+import logger from '@overleaf/logger'
+
+const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
+
+/**
+ * @type {(csvString: string) => Promise<string[][]>}
+ */
+const parseAsync = promisify(csv.parse)
+
+function usage() {
+  console.log('Usage: node re_add_deleted_emails.mjs [options]')
+  console.log(
+    'fix wrongly removed emails by remove_unconfirmed_emails (see 2025-11-05 email removal)'
+  )
+  console.log(
+    'run this script with a CSV containing user IDs and emails to re-add save in ',
+    CSV_FILENAME
+  )
+  console.log('Options:')
+  console.log('  --commit       apply the changes')
+  process.exit(0)
+}
+
+const { commit, help } = minimist(process.argv.slice(2), {
+  boolean: ['commit', 'help'],
+  alias: { help: 'h' },
+  default: { commit: false },
+})
+
+/**
+ * @param {string} email
+ */
+async function isEmailUsed(email) {
+  try {
+    await UserGetter.promises.ensureUniqueEmailAddress(email)
+    return false
+  } catch (err) {
+    if (err instanceof Errors.EmailExistsError) {
+      return true
+    }
+    throw err
+  }
+}
+
+async function consumeCsvFile(trackProgress) {
+  console.time('re_add_deleted_emails')
+
+  const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
+  const rows = await parseAsync(csvContent)
+  rows.shift() // Remove header row
+
+  const emailsByUserId = {}
+  for (const [userId, email] of rows) {
+    if (!EmailHelper.parseEmail(email)) {
+      throw new Error(`invalid email ${email}`)
+    }
+
+    if (!emailsByUserId[userId]) {
+      emailsByUserId[userId] = []
+    }
+    emailsByUserId[userId].push(email)
+  }
+
+  const userIds = Object.keys(emailsByUserId)
+
+  const counts = {
+    /** @type {string[]} */
+    processedUsers: [],
+    /** @type {string[]} */
+    userNotFound: [],
+    /** @type {string[]} */
+    emailsInUse: [],
+    /** @type {string[]} */
+    alreadyOk: [],
+    /** @type {string[]} */
+    primary: [],
+    /** @type {string[]} */
+    secondary: [],
+    /** @type {string[]} */
+    addedEmails: [],
+  }
+
+  console.log('Total emails in the CSV:', rows.length)
+  console.log('Total users in the CSV:', userIds.length)
+
+  for (const userId of userIds) {
+    const candidateEmails = emailsByUserId[userId]
+
+    const user = await db.users.findOne(
+      { _id: new ObjectId(userId) },
+      { readPreference: READ_PREFERENCE_SECONDARY }
+    )
+    if (!user) {
+      counts.userNotFound.push(userId)
+      continue
+    }
+
+    for (const email of candidateEmails) {
+      if (user.emails.some(item => item.email === email)) {
+        counts.alreadyOk.push(email)
+        continue
+      }
+
+      const isUsed = await isEmailUsed(email)
+      const isOwnPrimary = user.email === email
+
+      if (isUsed && !isOwnPrimary) {
+        counts.emailsInUse.push(email)
+        continue
+      }
+
+      if (user.email === email) counts.primary.push(email)
+      else counts.secondary.push(email)
+
+      if (commit) {
+        const auditLog = {
+          initiatorId: null,
+          ipAddress: null,
+          info: {
+            script: true,
+            note: 'fix wrongly removed unconfirmed secondary email',
+          },
+        }
+        if (isOwnPrimary) {
+          // can't use addEmailAddress for primary email because ensureUniqueEmailAddress will throw
+          // using an override instead
+          await addEmailAddressOverride(user._id, email, {}, auditLog)
+        } else {
+          await UserUpdater.promises.addEmailAddress(
+            user._id,
+            email,
+            {},
+            auditLog
+          )
+        }
+        await UserUpdater.promises.confirmEmail(user._id, email)
+      }
+      counts.addedEmails.push(email)
+    }
+
+    counts.processedUsers.push(userId)
+    trackProgress(
+      `Processed users: ${counts.processedUsers.length}/${userIds.length}`
+    )
+  }
+
+  console.log()
+  if (!commit) {
+    console.log('Dry-run, use --commit to apply changes')
+    console.log('This would be the result:')
+    console.log()
+  }
+
+  console.log('Total emails in the CSV:', rows.length)
+  console.log('Total users in the CSV:', userIds.length)
+  console.log()
+  console.log('Users not found:', counts.userNotFound.length)
+  console.log('Users not found:', JSON.stringify(counts.userNotFound))
+  console.log()
+  console.log('Already OK:', counts.alreadyOk.length)
+  console.log('Already OK:', JSON.stringify(counts.alreadyOk))
+  console.log()
+  console.log('Already in use:', counts.emailsInUse.length)
+  console.log('Already in use:', JSON.stringify(counts.emailsInUse))
+  console.log()
+  console.log('Primary:', counts.primary.length)
+  console.log('Primary:', JSON.stringify(counts.primary))
+  console.log()
+  console.log('Secondary:', counts.secondary.length)
+  console.log('Secondary:', JSON.stringify(counts.secondary))
+  console.log()
+  console.log('Added emails:', counts.addedEmails.length)
+  console.log('Added emails:', JSON.stringify(counts.addedEmails))
+  console.log()
+  console.log()
+  console.timeEnd('re_add_deleted_emails')
+  console.log()
+}
+
+async function main(trackProgress) {
+  if (help) {
+    return usage()
+  }
+  await consumeCsvFile(trackProgress)
+}
+
+try {
+  await scriptRunner(main)
+  process.exit(0)
+} catch (error) {
+  console.error(error)
+  process.exit(1)
+}
+
+async function addEmailAddressOverride(
+  userId,
+  newEmail,
+  affiliationOptions,
+  auditLog
+) {
+  AsyncLocalStorage.removeItem('userFullEmails')
+  newEmail = EmailHelper.parseEmail(newEmail)
+  if (!newEmail) {
+    throw new Error('invalid email')
+  }
+
+  // Bypass ensureUniqueEmailAddress when re-adding primary emails
+  // await UserGetter.promises.ensureUniqueEmailAddress(newEmail)
+
+  AnalyticsManager.recordEventForUserInBackground(
+    userId,
+    'secondary-email-added'
+  )
+
+  await UserAuditLogHandler.promises.addEntry(
+    userId,
+    'add-email',
+    auditLog.initiatorId,
+    auditLog.ipAddress,
+    {
+      ...auditLog.info,
+      newSecondaryEmail: newEmail,
+    }
+  )
+
+  try {
+    await InstitutionsAPI.promises.addAffiliation(
+      userId,
+      newEmail,
+      affiliationOptions
+    )
+  } catch (error) {
+    throw OError.tag(error, 'problem adding affiliation while adding email')
+  }
+
+  const createdAt = new Date()
+  let res
+  try {
+    const reversedHostname = newEmail.split('@')[1].split('').reverse().join('')
+    const update = {
+      $push: {
+        emails: { email: newEmail, createdAt, reversedHostname },
+      },
+    }
+    res = await UserUpdater.promises.updateUser(
+      { _id: userId, 'emails.email': { $ne: newEmail } },
+      update
+    )
+  } catch (error) {
+    throw OError.tag(error, 'problem updating users emails')
+  }
+
+  if (res.matchedCount !== 1) {
+    return
+  }
+
+  try {
+    await EmailChangeHelper.registerEmailCreation(userId, newEmail, {
+      // @ts-expect-error - This is copied from UserUpdater.mjs
+      createdAt: new Date(),
+      emailCreatedAt: createdAt,
+    })
+  } catch (error) {
+    logger.warn(
+      { error, userId, newEmail },
+      'Error registering email creation with analytics'
+    )
+  }
+}

services/web/scripts/remove_unconfirmed_emails.mjs

@@ -146,15 +146,15 @@ async function consumeCsvFile() {
   console.log('Total users in the CSV:', userIds.length)
 
   for (const userId of userIds) {
-    const emailsToRemove = emailsByUserId[userId]
+    const emailsToRemoveCandidates = emailsByUserId[userId]
 
     const user = await db.users.findOne({ _id: new ObjectId(userId) })
     if (!user) {
-      skippedEmail.userNotFound += emailsToRemove.length
+      skippedEmail.userNotFound += emailsToRemoveCandidates.length
       continue
     }
 
-    const emailsToRemoveNow = emailsToRemove.filter(email => {
+    const emailsToRemove = emailsToRemoveCandidates.filter(email => {
       const currentEmail = user.emails.find(e => e.email === email)
       if (!currentEmail) {
         skippedEmail.nowRemoved++
@@ -171,9 +171,9 @@ async function consumeCsvFile() {
       return true
     })
 
-    removedEmailsCount += emailsToRemoveNow.length
+    removedEmailsCount += emailsToRemove.length
 
-    if (commit && emailsToRemoveNow.length > 0) {
+    if (commit && emailsToRemove.length > 0) {
       for (const email of emailsToRemove) {
         await UserAuditLogHandler.promises.addEntry(
           userId,

services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs

@@ -0,0 +1,275 @@
+import { promisify } from 'node:util'
+import { exec } from 'node:child_process'
+import { expect } from 'chai'
+import { filterOutput } from './helpers/settings.mjs'
+import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
+import fs from 'node:fs/promises'
+
+const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
+
+async function runScript(commit) {
+  const result = await promisify(exec)(
+    ['node', 'scripts/re_add_deleted_emails.mjs', commit && '--commit']
+      .filter(Boolean)
+      .join(' ')
+  )
+
+  return {
+    ...result,
+    stdout: result.stdout.split('\n').filter(filterOutput),
+  }
+}
+
+/**
+ * @param {[string, string[]][]} userEmails
+ */
+const createUsers = async userEmails =>
+  Promise.all(
+    userEmails.map(async ([email, emails]) => {
+      const _id = new ObjectId()
+      await db.users.insertOne({
+        _id,
+        email,
+        emails: emails.map(email => ({ email })),
+        features: {},
+      })
+      return _id
+    })
+  )
+
+async function generateCsv(users) {
+  const text = 'User ID,Email'
+  const userRows = users.map(user => {
+    return `${user._id.toString()},${user.email}`
+  })
+  await fs.writeFile(CSV_FILENAME, [text, ...userRows].join('\n'))
+}
+
+describe('scripts/re_add_deleted_emails', function () {
+  let userIds
+
+  afterEach(async function () {
+    try {
+      await fs.unlink(CSV_FILENAME)
+    } catch (err) {
+      // Ignore errors if file doesn't exist
+    }
+  })
+
+  describe('when user IDs dont match', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['mismatch1@xmpl.com', []]])
+      await generateCsv([{ _id: new ObjectId(), email: 'mismatch2@xmpl.com' }])
+    })
+
+    it('doesnt add new emails', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 1')
+      expect(stdout).to.include('Added emails: 0')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when email address is invalid', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['user@xmpl.com', []]])
+      await generateCsv([{ _id: userIds[0], email: 'inv@lid@xmpl.com' }])
+    })
+
+    it('throws', async function () {
+      await expect(runScript(true)).to.eventually.be.rejectedWith(
+        'invalid email'
+      )
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+    it('throws even without --commit', async function () {
+      await expect(runScript(false)).to.eventually.be.rejectedWith(
+        'invalid email'
+      )
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when new email is used by another user', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        ['user1@xmpl.com', []],
+        ['user2@xmpl.com', ['new-email@xmpl.com']],
+      ])
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+    })
+
+    it('doesnt add new emails', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Already in use: ["new-email@xmpl.com"]')
+      expect(stdout).to.include('Added emails: 0')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when the user has 0 email in the array', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['user@xmpl.com', []]])
+    })
+
+    it('adds the primary email to the user', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'user@xmpl.com' }])
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(1)
+      expect(updatedUser.emails[0].email).to.equal('user@xmpl.com')
+      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
+      const auditLogs = await db.userAuditLogEntries
+        .find({ userId: userIds[0] })
+        .toArray()
+      expect(auditLogs).to.have.length(1)
+      expect(auditLogs[0].operation).to.equal('add-email')
+      expect(auditLogs[0].info).to.deep.include({
+        script: true,
+        note: 'fix wrongly removed unconfirmed secondary email',
+        newSecondaryEmail: 'user@xmpl.com',
+      })
+    })
+
+    it('adds the secondary email to the user', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(1)
+      expect(updatedUser.emails[0].email).to.equal('new-email@xmpl.com')
+      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
+      const auditLogs = await db.userAuditLogEntries
+        .find({ userId: userIds[0] })
+        .toArray()
+      expect(auditLogs).to.have.length(1)
+      expect(auditLogs[0].operation).to.equal('add-email')
+      expect(auditLogs[0].info).to.deep.include({
+        script: true,
+        note: 'fix wrongly removed unconfirmed secondary email',
+        newSecondaryEmail: 'new-email@xmpl.com',
+      })
+    })
+
+    it('doesnt add new emails without --commit', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+      const { stdout } = await runScript(false)
+      expect(stdout).to.include('Dry-run, use --commit to apply changes')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+  describe('when the user has several emails in the array', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        [
+          'user@xmpl.com',
+          ['email1@xmpl.com', 'email2@xmpl.com', 'email3@xmpl.com'],
+        ],
+      ])
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+    })
+
+    it('adds the email to the user', async function () {
+      const { stdout } = await runScript(true)
+
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(4)
+
+      expect(updatedUser.emails[3].email).to.equal('new-email@xmpl.com')
+      expect(updatedUser.emails[3].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[3].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[3].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[3].reconfirmedAt).to.be.an.instanceof(Date)
+    })
+
+    it('doesnt add new emails without --commit', async function () {
+      const { stdout } = await runScript(false)
+      expect(stdout).to.include('Dry-run, use --commit to apply changes')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(3)
+    })
+  })
+
+  describe('all of the above', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        ['user0@xmpl.com', []],
+        ['user1@xmpl.com', []],
+        ['user2@xmpl.com', ['a@xmpl.com', 'b@xmpl.com', 'c@xmpl.com']],
+        ['user3@xmpl.com', ['d@xmpl.com', 'e@xmpl.com', 'f@xmpl.com']],
+        ['user4@xmpl.com', ['x@xmpl.com', 'y@xmpl.com', 'z@xmpl.com']],
+        ['user5@xmpl.com', ['u@xmpl.com', 'v@xmpl.com', 'w@xmpl.com']],
+      ])
+      await generateCsv([
+        { _id: userIds[1], email: 'new1@xmpl.com' },
+        { _id: userIds[1], email: 'user1@xmpl.com' },
+        { _id: userIds[1], email: 'new2@xmpl.com' },
+        { _id: userIds[2], email: 'new3@xmpl.com' },
+        { _id: userIds[2], email: 'new4@xmpl.com' },
+        { _id: userIds[2], email: 'user2@xmpl.com' },
+        { _id: userIds[2], email: 'user3@xmpl.com' },
+        { _id: userIds[3], email: 'd@xmpl.com' },
+        { _id: userIds[5], email: 'a@xmpl.com' },
+        { _id: new ObjectId(), email: 'a@xmpl.com' },
+      ])
+    })
+
+    it('updates users', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 10')
+      expect(stdout).to.include('Total users in the CSV: 5')
+      expect(stdout).to.include('Users not found: 1')
+      expect(stdout).to.include(
+        'Already in use: ["user3@xmpl.com","a@xmpl.com"]'
+      )
+      expect(stdout).to.include('Already OK: ["d@xmpl.com"]')
+      expect(stdout).to.include('Primary: ["user1@xmpl.com","user2@xmpl.com"]')
+      expect(stdout).to.include(
+        'Secondary: ["new1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com"]'
+      )
+      expect(stdout).to.include(
+        'Added emails: ["new1@xmpl.com","user1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com","user2@xmpl.com"]'
+      )
+
+      const user0 = await db.users.findOne({ _id: userIds[0] })
+      const user1 = await db.users.findOne({ _id: userIds[1] })
+      const user2 = await db.users.findOne({ _id: userIds[2] })
+      const user3 = await db.users.findOne({ _id: userIds[3] })
+      const user4 = await db.users.findOne({ _id: userIds[4] })
+      const user5 = await db.users.findOne({ _id: userIds[5] })
+      expect(user0.emails).to.have.length(0)
+      expect(user1.emails).to.have.length(3) // new1, user1, new2
+      expect(user2.emails).to.have.length(6) // a, b, c, new3, new4, user2
+      expect(user3.emails).to.have.length(3) // d, e, f
+      expect(user4.emails).to.have.length(3) // x, y, z
+      expect(user5.emails).to.have.length(3) // u, v, w
+    })
+  })
+})