Bläddra i källkod

Remove scripts about unconfirmed emails removal (#29683)

* Remove unconfirmed email deletion scripts

* Remove tests

GitOrigin-RevId: a0ef84207fced135a13074265fe5d3b38400d76f
Antoine Clausse 8 månader sedan
förälder
incheckning
1447842fbd

+ 0 - 214
services/web/scripts/check_removed_emails.mjs

@@ -1,214 +0,0 @@
-// @ts-check
-
-import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
-import fs from 'node:fs/promises'
-import * as csv from 'csv'
-import { promisify } from 'node:util'
-import { scriptRunner } from './lib/ScriptRunner.mjs'
-import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js'
-
-const CSV_FILENAME = '/tmp/unconfirmed_emails.csv'
-
-/**
- * @type {(csvString: string) => Promise<string[][]>}
- */
-const parseAsync = promisify(csv.parse)
-
-/**
- * Checks the fallout of services/web/scripts/remove_unconfirmed_emails.mjs
- * which wrongly removed some emails that have been confirmed by users
- */
-async function main(trackProgress) {
-  console.time('check_removed_emails')
-
-  const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
-  const rows = await parseAsync(csvContent)
-  rows.shift() // Remove header row
-  const emailsByUserId = {}
-
-  for (const [userId, email] of rows) {
-    if (!emailsByUserId[userId]) {
-      emailsByUserId[userId] = []
-    }
-    emailsByUserId[userId].push(email)
-  }
-
-  const userIds = Object.keys(emailsByUserId)
-  let processedUsersCount = 0
-
-  const counts = {
-    /** @type {string[]} */
-    userNotFound: [],
-    /** @type {string[]} */
-    notDeleted: [],
-    deleted: 0,
-    /** @type {string[]} */
-    wasConfirmed: [],
-    /** @type {string[]} */
-    wasConfirmedLegacy: [],
-    /** @type {string[]} */
-    madePrimary: [],
-    /** @type {string[]} */
-    madeSecondary: [],
-    /** @type {string[]} */
-    isPrimary: [],
-    /** @type {string[]} */
-    isAddedAgain: [],
-  }
-
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-
-  for (const userId of userIds) {
-    const userEmails = emailsByUserId[userId]
-
-    const user = await db.users.findOne(
-      { _id: new ObjectId(userId) },
-      { readPreference: READ_PREFERENCE_SECONDARY }
-    )
-
-    if (!user) {
-      counts.userNotFound.push(userId)
-      continue
-    }
-
-    for (const email of userEmails) {
-      const deletionLog = await db.userAuditLogEntries.findOne(
-        {
-          userId: new ObjectId(userId),
-          operation: 'remove-email',
-          'info.removedEmail': email,
-          'info.note': 'remove unconfirmed secondary emails',
-        },
-        { readPreference: READ_PREFERENCE_SECONDARY }
-      )
-      if (!deletionLog) {
-        counts.notDeleted.push(email)
-        continue
-      }
-      counts.deleted++
-
-      if (user.email === email) {
-        counts.isPrimary.push(email)
-      }
-
-      const confirmationLog = await db.userAuditLogEntries.findOne(
-        {
-          userId: new ObjectId(userId),
-          operation: 'confirm-email-via-code',
-          'info.email': email,
-          timestamp: { $gt: new Date('2025-02-25') },
-        },
-        { readPreference: READ_PREFERENCE_SECONDARY }
-      )
-      if (confirmationLog) {
-        counts.wasConfirmed.push(email)
-      }
-
-      const confirmationLegacyLog = await db.userAuditLogEntries.findOne(
-        {
-          userId: new ObjectId(userId),
-          operation: 'confirm-email',
-          'info.email': email,
-          timestamp: { $gt: new Date('2025-02-25') },
-        },
-        { readPreference: READ_PREFERENCE_SECONDARY }
-      )
-      if (confirmationLegacyLog) {
-        counts.wasConfirmedLegacy.push(email)
-      }
-
-      const madePrimaryLog = await db.userAuditLogEntries.findOne(
-        {
-          userId: new ObjectId(userId),
-          operation: 'change-primary-email',
-          'info.newPrimaryEmail': email,
-          timestamp: { $gt: new Date('2025-02-25') },
-        },
-        { readPreference: READ_PREFERENCE_SECONDARY }
-      )
-      if (madePrimaryLog) {
-        counts.madePrimary.push(email)
-      }
-
-      const madeSecondaryLog = await db.userAuditLogEntries.findOne(
-        {
-          userId: new ObjectId(userId),
-          operation: 'change-primary-email',
-          'info.oldPrimaryEmail': email,
-          timestamp: { $gt: new Date('2025-02-25') },
-        },
-        { readPreference: READ_PREFERENCE_SECONDARY }
-      )
-      if (madeSecondaryLog) {
-        counts.madeSecondary.push(email)
-      }
-
-      if (user.emails.some(item => item.email === email)) {
-        counts.isAddedAgain.push(email)
-      }
-    }
-
-    processedUsersCount++
-    if (processedUsersCount % 100 === 0) {
-      trackProgress(`Processed ${processedUsersCount} users`)
-    }
-  }
-
-  console.log()
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-  console.log('Total users processed:', processedUsersCount)
-  console.log()
-  console.log('Users not found:', JSON.stringify(counts.userNotFound))
-  console.log()
-  console.log('Emails not deleted:', counts.notDeleted.length)
-  console.log('Emails deleted:', counts.deleted)
-  console.log()
-  console.log('Emails that were confirmed:', counts.wasConfirmed.length)
-  console.log(
-    'Emails that were confirmed:',
-    JSON.stringify(counts.wasConfirmed)
-  )
-  console.log()
-  console.log(
-    'Emails that were confirmed (legacy):',
-    counts.wasConfirmedLegacy.length
-  )
-  console.log(
-    'Emails that were confirmed (legacy):',
-    JSON.stringify(counts.wasConfirmedLegacy)
-  )
-  console.log()
-  console.log('Emails that are primary:', counts.isPrimary.length)
-  console.log('Emails that are primary:', JSON.stringify(counts.isPrimary))
-  console.log()
-  console.log('Emails that were made primary:', counts.madePrimary.length)
-  console.log(
-    'Emails that were made primary:',
-    JSON.stringify(counts.madePrimary)
-  )
-  console.log()
-  console.log('Emails that were made secondary:', counts.madeSecondary.length)
-  console.log(
-    'Emails that were made secondary:',
-    JSON.stringify(counts.madeSecondary)
-  )
-  console.log()
-  console.log('Emails that were added again:', counts.isAddedAgain.length)
-  console.log(
-    'Emails that were added again:',
-    JSON.stringify(counts.isAddedAgain)
-  )
-  console.log()
-  console.timeEnd('check_removed_emails')
-  console.log()
-}
-
-try {
-  await scriptRunner(main)
-  process.exit(0)
-} catch (error) {
-  console.error(error)
-  process.exit(1)
-}

+ 0 - 288
services/web/scripts/re_add_deleted_emails.mjs

@@ -1,288 +0,0 @@
-// @ts-check
-
-import minimist from 'minimist'
-import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
-import fs from 'node:fs/promises'
-import * as csv from 'csv'
-import { promisify } from 'node:util'
-import { scriptRunner } from './lib/ScriptRunner.mjs'
-import Errors from '../app/src/Features/Errors/Errors.js'
-import UserGetter from '../app/src/Features/User/UserGetter.mjs'
-import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js'
-import UserUpdater from '../app/src/Features/User/UserUpdater.mjs'
-import EmailHelper from '../app/src/Features/Helpers/EmailHelper.mjs'
-import AsyncLocalStorage from '../app/src/infrastructure/AsyncLocalStorage.js'
-import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.mjs'
-import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs'
-import InstitutionsAPI from '../app/src/Features/Institutions/InstitutionsAPI.mjs'
-import OError from '@overleaf/o-error'
-import EmailChangeHelper from '../app/src/Features/Analytics/EmailChangeHelper.mjs'
-import logger from '@overleaf/logger'
-
-const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
-
-/**
- * @type {(csvString: string) => Promise<string[][]>}
- */
-const parseAsync = promisify(csv.parse)
-
-function usage() {
-  console.log('Usage: node re_add_deleted_emails.mjs [options]')
-  console.log(
-    'fix wrongly removed emails by remove_unconfirmed_emails (see 2025-11-05 email removal)'
-  )
-  console.log(
-    'run this script with a CSV containing user IDs and emails to re-add save in ',
-    CSV_FILENAME
-  )
-  console.log('Options:')
-  console.log('  --commit       apply the changes')
-  process.exit(0)
-}
-
-const { commit, help } = minimist(process.argv.slice(2), {
-  boolean: ['commit', 'help'],
-  alias: { help: 'h' },
-  default: { commit: false },
-})
-
-/**
- * @param {string} email
- */
-async function isEmailUsed(email) {
-  try {
-    await UserGetter.promises.ensureUniqueEmailAddress(email)
-    return false
-  } catch (err) {
-    if (err instanceof Errors.EmailExistsError) {
-      return true
-    }
-    throw err
-  }
-}
-
-async function consumeCsvFile(trackProgress) {
-  console.time('re_add_deleted_emails')
-
-  const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
-  const rows = await parseAsync(csvContent)
-  rows.shift() // Remove header row
-
-  const emailsByUserId = {}
-  for (const [userId, email] of rows) {
-    if (!EmailHelper.parseEmail(email)) {
-      throw new Error(`invalid email ${email}`)
-    }
-
-    if (!emailsByUserId[userId]) {
-      emailsByUserId[userId] = []
-    }
-    emailsByUserId[userId].push(email)
-  }
-
-  const userIds = Object.keys(emailsByUserId)
-
-  const counts = {
-    /** @type {string[]} */
-    processedUsers: [],
-    /** @type {string[]} */
-    userNotFound: [],
-    /** @type {string[]} */
-    emailsInUse: [],
-    /** @type {string[]} */
-    alreadyOk: [],
-    /** @type {string[]} */
-    primary: [],
-    /** @type {string[]} */
-    secondary: [],
-    /** @type {string[]} */
-    addedEmails: [],
-  }
-
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-
-  for (const userId of userIds) {
-    const candidateEmails = emailsByUserId[userId]
-
-    const user = await db.users.findOne(
-      { _id: new ObjectId(userId) },
-      { readPreference: READ_PREFERENCE_SECONDARY }
-    )
-    if (!user) {
-      counts.userNotFound.push(userId)
-      continue
-    }
-
-    for (const email of candidateEmails) {
-      if (user.emails.some(item => item.email === email)) {
-        counts.alreadyOk.push(email)
-        continue
-      }
-
-      const isUsed = await isEmailUsed(email)
-      const isOwnPrimary = user.email === email
-
-      if (isUsed && !isOwnPrimary) {
-        counts.emailsInUse.push(email)
-        continue
-      }
-
-      if (user.email === email) counts.primary.push(email)
-      else counts.secondary.push(email)
-
-      if (commit) {
-        const auditLog = {
-          initiatorId: null,
-          ipAddress: null,
-          info: {
-            script: true,
-            note: 'fix wrongly removed unconfirmed secondary email',
-          },
-        }
-        if (isOwnPrimary) {
-          // can't use addEmailAddress for primary email because ensureUniqueEmailAddress will throw
-          // using an override instead
-          await addEmailAddressOverride(user._id, email, {}, auditLog)
-        } else {
-          await UserUpdater.promises.addEmailAddress(
-            user._id,
-            email,
-            {},
-            auditLog
-          )
-        }
-        await UserUpdater.promises.confirmEmail(user._id, email)
-      }
-      counts.addedEmails.push(email)
-    }
-
-    counts.processedUsers.push(userId)
-    trackProgress(
-      `Processed users: ${counts.processedUsers.length}/${userIds.length}`
-    )
-  }
-
-  console.log()
-  if (!commit) {
-    console.log('Dry-run, use --commit to apply changes')
-    console.log('This would be the result:')
-    console.log()
-  }
-
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-  console.log()
-  console.log('Users not found:', counts.userNotFound.length)
-  console.log('Users not found:', JSON.stringify(counts.userNotFound))
-  console.log()
-  console.log('Already OK:', counts.alreadyOk.length)
-  console.log('Already OK:', JSON.stringify(counts.alreadyOk))
-  console.log()
-  console.log('Already in use:', counts.emailsInUse.length)
-  console.log('Already in use:', JSON.stringify(counts.emailsInUse))
-  console.log()
-  console.log('Primary:', counts.primary.length)
-  console.log('Primary:', JSON.stringify(counts.primary))
-  console.log()
-  console.log('Secondary:', counts.secondary.length)
-  console.log('Secondary:', JSON.stringify(counts.secondary))
-  console.log()
-  console.log('Added emails:', counts.addedEmails.length)
-  console.log('Added emails:', JSON.stringify(counts.addedEmails))
-  console.log()
-  console.log()
-  console.timeEnd('re_add_deleted_emails')
-  console.log()
-}
-
-async function main(trackProgress) {
-  if (help) {
-    return usage()
-  }
-  await consumeCsvFile(trackProgress)
-}
-
-try {
-  await scriptRunner(main)
-  process.exit(0)
-} catch (error) {
-  console.error(error)
-  process.exit(1)
-}
-
-async function addEmailAddressOverride(
-  userId,
-  newEmail,
-  affiliationOptions,
-  auditLog
-) {
-  AsyncLocalStorage.removeItem('userFullEmails')
-  newEmail = EmailHelper.parseEmail(newEmail)
-  if (!newEmail) {
-    throw new Error('invalid email')
-  }
-
-  // Bypass ensureUniqueEmailAddress when re-adding primary emails
-  // await UserGetter.promises.ensureUniqueEmailAddress(newEmail)
-
-  AnalyticsManager.recordEventForUserInBackground(
-    userId,
-    'secondary-email-added'
-  )
-
-  await UserAuditLogHandler.promises.addEntry(
-    userId,
-    'add-email',
-    auditLog.initiatorId,
-    auditLog.ipAddress,
-    {
-      ...auditLog.info,
-      newSecondaryEmail: newEmail,
-    }
-  )
-
-  try {
-    await InstitutionsAPI.promises.addAffiliation(
-      userId,
-      newEmail,
-      affiliationOptions
-    )
-  } catch (error) {
-    throw OError.tag(error, 'problem adding affiliation while adding email')
-  }
-
-  const createdAt = new Date()
-  let res
-  try {
-    const reversedHostname = newEmail.split('@')[1].split('').reverse().join('')
-    const update = {
-      $push: {
-        emails: { email: newEmail, createdAt, reversedHostname },
-      },
-    }
-    res = await UserUpdater.promises.updateUser(
-      { _id: userId, 'emails.email': { $ne: newEmail } },
-      update
-    )
-  } catch (error) {
-    throw OError.tag(error, 'problem updating users emails')
-  }
-
-  if (res.matchedCount !== 1) {
-    return
-  }
-
-  try {
-    await EmailChangeHelper.registerEmailCreation(userId, newEmail, {
-      // @ts-expect-error - This is copied from UserUpdater.mjs
-      createdAt: new Date(),
-      emailCreatedAt: createdAt,
-    })
-  } catch (error) {
-    logger.warn(
-      { error, userId, newEmail },
-      'Error registering email creation with analytics'
-    )
-  }
-}

+ 0 - 265
services/web/scripts/remove_unconfirmed_emails.mjs

@@ -1,265 +0,0 @@
-// @ts-check
-
-import minimist from 'minimist'
-import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
-import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
-import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs'
-import fs from 'node:fs/promises'
-import * as csv from 'csv'
-import { promisify } from 'node:util'
-import _ from 'lodash'
-import { scriptRunner } from './lib/ScriptRunner.mjs'
-
-const CSV_FILENAME = '/tmp/remove_unconfirmed_emails.csv'
-/**
- * @type {(records: string[][]) => Promise<string>}
- */
-const stringifyAsync = promisify(csv.stringify)
-/**
- * @type {(csvString: string) => Promise<string[][]>}
- */
-const parseAsync = promisify(csv.parse)
-
-function usage() {
-  console.log('Usage: node remove_unconfirmed_emails.mjs')
-  console.log('Removes unconfirmed emails from users')
-  console.log('Options:')
-  console.log(
-    '' +
-      '  --generate     generate the CSV file (remove_unconfirmed_emails.csv) containing the emails to remove\n' +
-      '  --consume      consume the CSV file (remove_unconfirmed_emails.csv) and remove the emails (by default it is a dry-run)\n' +
-      '  --commit       apply the changes (to be used with --consume)\n'
-  )
-  process.exit(0)
-}
-
-const { generate, consume, commit, help } = minimist(process.argv.slice(2), {
-  boolean: ['generate', 'consume', 'commit', 'help'],
-  alias: { help: 'h' },
-  default: { generate: false, consume: false, commit: false },
-})
-
-async function generateCsvFile(trackProgress) {
-  console.time('generate_csv')
-
-  let processedUsersCount = 0
-  let skippedUnconfirmedPrimaries = 0
-  let totalEmailsToRemove = 0
-  let totalUsersInCsv = 0
-
-  const records = [['User ID', 'Email', 'Sign Up Date']]
-
-  await batchedUpdate(
-    db.users,
-    {
-      $and: [
-        { emails: { $exists: true } },
-        { emails: { $not: { $size: 0 } } },
-        // Warning: this also matches unconfirmed primary emails
-        {
-          emails: {
-            $elemMatch: {
-              $or: [{ confirmedAt: { $exists: false } }, { confirmedAt: null }],
-            },
-          },
-        },
-      ],
-    },
-    async users => {
-      console.log('Process', users.length, 'users')
-      processedUsersCount += users.length
-
-      for (const user of users) {
-        const unconfirmedSecondaries = user.emails.filter(
-          email => !email.confirmedAt && email.email !== user.email
-        )
-
-        if (unconfirmedSecondaries.length === 0) {
-          // Users can have been selected because of their unconfirmed primary email
-          // we don't want to remove those
-          skippedUnconfirmedPrimaries++
-          continue
-        }
-
-        for (const email of unconfirmedSecondaries) {
-          records.push([
-            user._id.toString(),
-            email.email,
-            user.signUpDate.toISOString(),
-          ])
-        }
-
-        totalUsersInCsv++
-        totalEmailsToRemove += unconfirmedSecondaries.length
-      }
-    },
-    { _id: 1, signUpDate: 1, emails: 1, email: 1 },
-    undefined,
-    { trackProgress }
-  )
-
-  const csvContent = await stringifyAsync(records)
-  await fs.writeFile(CSV_FILENAME, csvContent)
-
-  console.log()
-  console.log('Processed users:', processedUsersCount)
-  console.log()
-  console.log('Generated CSV file:', CSV_FILENAME)
-  console.log('Total emails in the CSV:', totalEmailsToRemove)
-  console.log('Total users in the CSV:', totalUsersInCsv)
-  console.log(
-    'Unconfirmed primary emails (skipped):',
-    skippedUnconfirmedPrimaries
-  )
-  console.log()
-  console.timeEnd('generate_csv')
-  console.log()
-}
-
-async function consumeCsvFile() {
-  console.time('consume_csv')
-
-  const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
-  const rows = await parseAsync(csvContent)
-  rows.shift() // Remove header row
-  const emailsByUserId = {}
-
-  for (const [userId, email] of rows) {
-    if (!emailsByUserId[userId]) {
-      emailsByUserId[userId] = []
-    }
-    emailsByUserId[userId].push(email)
-  }
-
-  const userIds = Object.keys(emailsByUserId)
-  let processedUsersCount = 0
-  let removedEmailsCount = 0
-  let totalModifiedUsersCount = 0
-  const skippedEmail = {
-    userNotFound: 0,
-    nowConfirmed: 0,
-    nowPrimary: 0,
-    nowRemoved: 0,
-  }
-
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-
-  for (const userId of userIds) {
-    const emailsToRemoveCandidates = emailsByUserId[userId]
-
-    const user = await db.users.findOne({ _id: new ObjectId(userId) })
-    if (!user) {
-      skippedEmail.userNotFound += emailsToRemoveCandidates.length
-      continue
-    }
-
-    const emailsToRemove = emailsToRemoveCandidates.filter(email => {
-      const currentEmail = user.emails.find(e => e.email === email)
-      if (!currentEmail) {
-        skippedEmail.nowRemoved++
-        return false
-      }
-      if (currentEmail.confirmedAt) {
-        skippedEmail.nowConfirmed++
-        return false
-      }
-      if (currentEmail.email === user.email) {
-        skippedEmail.nowPrimary++
-        return false
-      }
-      return true
-    })
-
-    removedEmailsCount += emailsToRemove.length
-
-    if (commit && emailsToRemove.length > 0) {
-      for (const email of emailsToRemove) {
-        await UserAuditLogHandler.promises.addEntry(
-          userId,
-          'remove-email',
-          undefined,
-          undefined,
-          {
-            removedEmail: email,
-            script: true,
-            note: 'remove unconfirmed secondary emails',
-          }
-        )
-      }
-
-      const updated = await db.users.updateOne(
-        { _id: new ObjectId(userId) },
-        { $pull: { emails: { email: { $in: emailsToRemove } } } }
-      )
-      totalModifiedUsersCount += updated.modifiedCount
-    }
-
-    processedUsersCount++
-    if (processedUsersCount % 100 === 0) {
-      console.log('Processed', processedUsersCount, 'users')
-    }
-  }
-
-  console.log()
-  if (!commit) {
-    console.log('Dry-run, use --commit to apply changes')
-    console.log('This would be the result:')
-    console.log()
-  }
-
-  console.log('Total emails in the CSV:', rows.length)
-  console.log('Total users in the CSV:', userIds.length)
-  console.log('Total users processed:', processedUsersCount)
-  console.log('Total emails removed:', removedEmailsCount)
-  console.log('Skipped emails:', _.sum(Object.values(skippedEmail)))
-  console.log('  - User not found:', skippedEmail.userNotFound)
-  console.log('  - Email now confirmed:', skippedEmail.nowConfirmed)
-  console.log('  - Email now primary:', skippedEmail.nowPrimary)
-  console.log('  - Email now removed:', skippedEmail.nowRemoved)
-  console.log()
-
-  if (commit) {
-    console.log('Total users modified:', totalModifiedUsersCount)
-  } else {
-    console.log('Note: this was a dry-run. No changes were made.')
-  }
-  console.log()
-  console.timeEnd('consume_csv')
-  console.log()
-}
-
-async function main(trackProgress) {
-  if (help) {
-    return usage()
-  }
-
-  if (!generate && !consume) {
-    console.error('Error: Either --generate or --consume must be specified')
-    return usage()
-  }
-
-  if (generate && consume) {
-    console.error('Error: Cannot use both --generate and --consume together')
-    return usage()
-  }
-
-  if (commit && !consume) {
-    console.error('Error: --commit can only be used with --consume')
-    return usage()
-  }
-
-  if (generate) {
-    await generateCsvFile(trackProgress)
-  } else if (consume) {
-    await consumeCsvFile()
-  }
-}
-
-try {
-  await scriptRunner(main)
-  process.exit(0)
-} catch (error) {
-  console.error(error)
-  process.exit(1)
-}

+ 0 - 275
services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs

@@ -1,275 +0,0 @@
-import { promisify } from 'node:util'
-import { exec } from 'node:child_process'
-import { expect } from 'chai'
-import { filterOutput } from './helpers/settings.mjs'
-import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
-import fs from 'node:fs/promises'
-
-const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
-
-async function runScript(commit) {
-  const result = await promisify(exec)(
-    ['node', 'scripts/re_add_deleted_emails.mjs', commit && '--commit']
-      .filter(Boolean)
-      .join(' ')
-  )
-
-  return {
-    ...result,
-    stdout: result.stdout.split('\n').filter(filterOutput),
-  }
-}
-
-/**
- * @param {[string, string[]][]} userEmails
- */
-const createUsers = async userEmails =>
-  Promise.all(
-    userEmails.map(async ([email, emails]) => {
-      const _id = new ObjectId()
-      await db.users.insertOne({
-        _id,
-        email,
-        emails: emails.map(email => ({ email })),
-        features: {},
-      })
-      return _id
-    })
-  )
-
-async function generateCsv(users) {
-  const text = 'User ID,Email'
-  const userRows = users.map(user => {
-    return `${user._id.toString()},${user.email}`
-  })
-  await fs.writeFile(CSV_FILENAME, [text, ...userRows].join('\n'))
-}
-
-describe('scripts/re_add_deleted_emails', function () {
-  let userIds
-
-  afterEach(async function () {
-    try {
-      await fs.unlink(CSV_FILENAME)
-    } catch (err) {
-      // Ignore errors if file doesn't exist
-    }
-  })
-
-  describe('when user IDs dont match', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([['mismatch1@xmpl.com', []]])
-      await generateCsv([{ _id: new ObjectId(), email: 'mismatch2@xmpl.com' }])
-    })
-
-    it('doesnt add new emails', async function () {
-      const { stdout } = await runScript(true)
-      expect(stdout).to.include('Total emails in the CSV: 1')
-      expect(stdout).to.include('Total users in the CSV: 1')
-      expect(stdout).to.include('Users not found: 1')
-      expect(stdout).to.include('Added emails: 0')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(0)
-    })
-  })
-
-  describe('when email address is invalid', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([['user@xmpl.com', []]])
-      await generateCsv([{ _id: userIds[0], email: 'inv@lid@xmpl.com' }])
-    })
-
-    it('throws', async function () {
-      await expect(runScript(true)).to.eventually.be.rejectedWith(
-        'invalid email'
-      )
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(0)
-    })
-    it('throws even without --commit', async function () {
-      await expect(runScript(false)).to.eventually.be.rejectedWith(
-        'invalid email'
-      )
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(0)
-    })
-  })
-
-  describe('when new email is used by another user', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([
-        ['user1@xmpl.com', []],
-        ['user2@xmpl.com', ['new-email@xmpl.com']],
-      ])
-      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
-    })
-
-    it('doesnt add new emails', async function () {
-      const { stdout } = await runScript(true)
-      expect(stdout).to.include('Total emails in the CSV: 1')
-      expect(stdout).to.include('Total users in the CSV: 1')
-      expect(stdout).to.include('Users not found: 0')
-      expect(stdout).to.include('Already in use: ["new-email@xmpl.com"]')
-      expect(stdout).to.include('Added emails: 0')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(0)
-    })
-  })
-
-  describe('when the user has 0 email in the array', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([['user@xmpl.com', []]])
-    })
-
-    it('adds the primary email to the user', async function () {
-      await generateCsv([{ _id: userIds[0], email: 'user@xmpl.com' }])
-      const { stdout } = await runScript(true)
-      expect(stdout).to.include('Total emails in the CSV: 1')
-      expect(stdout).to.include('Total users in the CSV: 1')
-      expect(stdout).to.include('Users not found: 0')
-      expect(stdout).to.include('Added emails: 1')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal('user@xmpl.com')
-      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
-      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
-      const auditLogs = await db.userAuditLogEntries
-        .find({ userId: userIds[0] })
-        .toArray()
-      expect(auditLogs).to.have.length(1)
-      expect(auditLogs[0].operation).to.equal('add-email')
-      expect(auditLogs[0].info).to.deep.include({
-        script: true,
-        note: 'fix wrongly removed unconfirmed secondary email',
-        newSecondaryEmail: 'user@xmpl.com',
-      })
-    })
-
-    it('adds the secondary email to the user', async function () {
-      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
-      const { stdout } = await runScript(true)
-      expect(stdout).to.include('Total emails in the CSV: 1')
-      expect(stdout).to.include('Total users in the CSV: 1')
-      expect(stdout).to.include('Users not found: 0')
-      expect(stdout).to.include('Added emails: 1')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal('new-email@xmpl.com')
-      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
-      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
-      const auditLogs = await db.userAuditLogEntries
-        .find({ userId: userIds[0] })
-        .toArray()
-      expect(auditLogs).to.have.length(1)
-      expect(auditLogs[0].operation).to.equal('add-email')
-      expect(auditLogs[0].info).to.deep.include({
-        script: true,
-        note: 'fix wrongly removed unconfirmed secondary email',
-        newSecondaryEmail: 'new-email@xmpl.com',
-      })
-    })
-
-    it('doesnt add new emails without --commit', async function () {
-      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
-      const { stdout } = await runScript(false)
-      expect(stdout).to.include('Dry-run, use --commit to apply changes')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(0)
-    })
-  })
-  describe('when the user has several emails in the array', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([
-        [
-          'user@xmpl.com',
-          ['email1@xmpl.com', 'email2@xmpl.com', 'email3@xmpl.com'],
-        ],
-      ])
-      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
-    })
-
-    it('adds the email to the user', async function () {
-      const { stdout } = await runScript(true)
-
-      expect(stdout).to.include('Total emails in the CSV: 1')
-      expect(stdout).to.include('Total users in the CSV: 1')
-      expect(stdout).to.include('Users not found: 0')
-      expect(stdout).to.include('Added emails: 1')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(4)
-
-      expect(updatedUser.emails[3].email).to.equal('new-email@xmpl.com')
-      expect(updatedUser.emails[3].reversedHostname).to.equal('moc.lpmx')
-      expect(updatedUser.emails[3].confirmedAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[3].createdAt).to.be.an.instanceof(Date)
-      expect(updatedUser.emails[3].reconfirmedAt).to.be.an.instanceof(Date)
-    })
-
-    it('doesnt add new emails without --commit', async function () {
-      const { stdout } = await runScript(false)
-      expect(stdout).to.include('Dry-run, use --commit to apply changes')
-      const updatedUser = await db.users.findOne({ _id: userIds[0] })
-      expect(updatedUser.emails).to.have.length(3)
-    })
-  })
-
-  describe('all of the above', function () {
-    beforeEach(async function () {
-      userIds = await createUsers([
-        ['user0@xmpl.com', []],
-        ['user1@xmpl.com', []],
-        ['user2@xmpl.com', ['a@xmpl.com', 'b@xmpl.com', 'c@xmpl.com']],
-        ['user3@xmpl.com', ['d@xmpl.com', 'e@xmpl.com', 'f@xmpl.com']],
-        ['user4@xmpl.com', ['x@xmpl.com', 'y@xmpl.com', 'z@xmpl.com']],
-        ['user5@xmpl.com', ['u@xmpl.com', 'v@xmpl.com', 'w@xmpl.com']],
-      ])
-      await generateCsv([
-        { _id: userIds[1], email: 'new1@xmpl.com' },
-        { _id: userIds[1], email: 'user1@xmpl.com' },
-        { _id: userIds[1], email: 'new2@xmpl.com' },
-        { _id: userIds[2], email: 'new3@xmpl.com' },
-        { _id: userIds[2], email: 'new4@xmpl.com' },
-        { _id: userIds[2], email: 'user2@xmpl.com' },
-        { _id: userIds[2], email: 'user3@xmpl.com' },
-        { _id: userIds[3], email: 'd@xmpl.com' },
-        { _id: userIds[5], email: 'a@xmpl.com' },
-        { _id: new ObjectId(), email: 'a@xmpl.com' },
-      ])
-    })
-
-    it('updates users', async function () {
-      const { stdout } = await runScript(true)
-      expect(stdout).to.include('Total emails in the CSV: 10')
-      expect(stdout).to.include('Total users in the CSV: 5')
-      expect(stdout).to.include('Users not found: 1')
-      expect(stdout).to.include(
-        'Already in use: ["user3@xmpl.com","a@xmpl.com"]'
-      )
-      expect(stdout).to.include('Already OK: ["d@xmpl.com"]')
-      expect(stdout).to.include('Primary: ["user1@xmpl.com","user2@xmpl.com"]')
-      expect(stdout).to.include(
-        'Secondary: ["new1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com"]'
-      )
-      expect(stdout).to.include(
-        'Added emails: ["new1@xmpl.com","user1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com","user2@xmpl.com"]'
-      )
-
-      const user0 = await db.users.findOne({ _id: userIds[0] })
-      const user1 = await db.users.findOne({ _id: userIds[1] })
-      const user2 = await db.users.findOne({ _id: userIds[2] })
-      const user3 = await db.users.findOne({ _id: userIds[3] })
-      const user4 = await db.users.findOne({ _id: userIds[4] })
-      const user5 = await db.users.findOne({ _id: userIds[5] })
-      expect(user0.emails).to.have.length(0)
-      expect(user1.emails).to.have.length(3) // new1, user1, new2
-      expect(user2.emails).to.have.length(6) // a, b, c, new3, new4, user2
-      expect(user3.emails).to.have.length(3) // d, e, f
-      expect(user4.emails).to.have.length(3) // x, y, z
-      expect(user5.emails).to.have.length(3) // u, v, w
-    })
-  })
-})

+ 0 - 290
services/web/test/acceptance/src/RemoveUnconfirmedEmailsScriptTests.mjs

@@ -1,290 +0,0 @@
-import { promisify } from 'node:util'
-import { exec } from 'node:child_process'
-import { expect } from 'chai'
-import { filterOutput } from './helpers/settings.mjs'
-import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
-import fs from 'node:fs/promises'
-
-const CSV_FILENAME = '/tmp/remove_unconfirmed_emails.csv'
-
-async function runScript(mode, commit) {
-  const result = await promisify(exec)(
-    [
-      'node',
-      'scripts/remove_unconfirmed_emails.mjs',
-      mode === 'generate' ? '--generate' : '--consume',
-      commit && '--commit',
-    ]
-      .filter(Boolean)
-      .join(' ')
-  )
-  return {
-    ...result,
-    stdout: result.stdout.split('\n').filter(filterOutput),
-  }
-}
-
-function createUser(signUpDate, emails, userIdx) {
-  const email = `primary${userIdx ?? ''}@overleaf.com`
-  return {
-    _id: new ObjectId(),
-    email,
-    emails,
-    signUpDate,
-  }
-}
-
-describe('scripts/remove_unconfirmed_emails', function () {
-  let user
-
-  afterEach(async function () {
-    try {
-      await fs.unlink(CSV_FILENAME)
-    } catch (err) {
-      // Ignore errors if file doesn't exist
-    }
-  })
-
-  describe('when removing unconfirmed secondary emails', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com', confirmedAt: new Date() },
-        { email: 'unconfirmed1@overleaf.com' },
-        { email: 'unconfirmed-special-,\'"@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should remove all unconfirmed secondary emails', async function () {
-      await runScript('generate')
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 2')
-      expect(r.stdout).to.include('Total users processed: 1')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal(user.email)
-    })
-
-    it('should not modify anything in dry run mode', async function () {
-      await runScript('generate')
-      const r = await runScript('consume', false)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 2')
-      expect(r.stdout).to.include('Total users processed: 1')
-      expect(r.stdout).to.include(
-        'Note: this was a dry-run. No changes were made.'
-      )
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(3)
-    })
-  })
-
-  describe('when handling confirmed secondary emails', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com', confirmedAt: new Date() },
-        { email: 'confirmed@overleaf.com', confirmedAt: new Date() },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should preserve confirmed secondary emails', async function () {
-      await runScript('generate')
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 0')
-      expect(r.stdout).to.include('Total users processed: 0')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(2)
-      expect(updatedUser.emails[1].confirmedAt).to.exist
-    })
-  })
-
-  describe('when handling unconfirmed primary emails', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should not remove unconfirmed primary emails', async function () {
-      await runScript('generate')
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 0')
-      expect(r.stdout).to.include('Total users processed: 0')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal('primary@overleaf.com')
-    })
-  })
-
-  describe('when users confirmed their email in between', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com' },
-        { email: 'secondary@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should not remove emails from users who confirmed their email in between', async function () {
-      await runScript('generate')
-
-      await db.users.updateOne(
-        { _id: user._id },
-        { $set: { 'emails.1.confirmedAt': new Date() } }
-      )
-
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 1')
-      expect(r.stdout).to.include('Skipped emails: 1')
-      expect(r.stdout).to.include('  - Email now confirmed: 1')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(2)
-    })
-  })
-
-  describe('when users changed their primary email in between', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com' },
-        { email: 'secondary@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should not remove emails from users who changed their primary email in between', async function () {
-      await runScript('generate')
-
-      await db.users.updateOne(
-        { _id: user._id },
-        { $set: { email: 'secondary@overleaf.com' } }
-      )
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 1')
-      expect(r.stdout).to.include('Skipped emails: 1')
-      expect(r.stdout).to.include('  - Email now primary: 1')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(2)
-    })
-  })
-
-  describe('when users account was deleted in between', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com' },
-        { email: 'secondary@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should skip emails from users whose account was deleted', async function () {
-      await runScript('generate')
-
-      // Delete the user
-      await db.users.deleteOne({ _id: user._id })
-
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 1')
-      expect(r.stdout).to.include('Skipped emails: 1')
-      expect(r.stdout).to.include('  - User not found: 1')
-    })
-  })
-
-  describe('when users email was deleted in between', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com' },
-        { email: 'secondary@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should skip emails that were already removed', async function () {
-      await runScript('generate')
-
-      // Remove the secondary email
-      await db.users.updateOne(
-        { _id: user._id },
-        { $pull: { emails: { email: 'secondary@overleaf.com' } } }
-      )
-
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 1')
-      expect(r.stdout).to.include('Skipped emails: 1')
-      expect(r.stdout).to.include('  - Email now removed: 1')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal('primary@overleaf.com')
-    })
-  })
-
-  describe('when handling confirmation field edge cases', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com', confirmedAt: new Date() },
-        { email: 'secondary1@overleaf.com', confirmedAt: null },
-        { email: 'secondary2@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should remove emails with both missing and null confirmedAt', async function () {
-      await runScript('generate')
-      const r = await runScript('consume', true)
-
-      expect(r.stdout).to.include('Total emails in the CSV: 2')
-      expect(r.stdout).to.include('Total users processed: 1')
-
-      const updatedUser = await db.users.findOne({ _id: user._id })
-      expect(updatedUser.emails).to.have.length(1)
-      expect(updatedUser.emails[0].email).to.equal(user.email)
-    })
-  })
-
-  describe('CSV file generation', function () {
-    beforeEach(async function () {
-      user = createUser(new Date('2000-01-01'), [
-        { email: 'primary@overleaf.com', confirmedAt: new Date() },
-        { email: 'unconfirmed1@overleaf.com' },
-        { email: 'confirmed1@overleaf.com', confirmedAt: new Date() },
-        { email: 'unconfirmed2@overleaf.com' },
-        { email: '!,@overleaf.com' },
-        { email: "!'@overleaf.com" },
-        { email: '!,\'"@overleaf.com' },
-      ])
-      await db.users.insertOne(user)
-    })
-
-    it('should generate a valid CSV file', async function () {
-      const r = await runScript('generate')
-
-      expect(r.stdout).to.include(
-        'Generated CSV file: /tmp/remove_unconfirmed_emails.csv'
-      )
-      expect(r.stdout).to.include('Total emails in the CSV: 5')
-      const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
-      expect(csvContent).to.equal(`User ID,Email,Sign Up Date
-${user._id},unconfirmed1@overleaf.com,2000-01-01T00:00:00.000Z
-${user._id},unconfirmed2@overleaf.com,2000-01-01T00:00:00.000Z
-${user._id},"!,@overleaf.com",2000-01-01T00:00:00.000Z
-${user._id},!'@overleaf.com,2000-01-01T00:00:00.000Z
-${user._id},"!,'""@overleaf.com",2000-01-01T00:00:00.000Z
-`)
-    })
-  })
-})