[monorepo] enable mongo notablescan for all services but web (#29497)

* [migrations] make old migrations compatible with --notablescan in mongo

Re-running the migration for db.users.emails.reversedHostname on all
Server Pro/CE users will not be harmfull.

* [monorepo] enable mongo notablescan for all services but web

GitOrigin-RevId: e2a710c9de08de9ac2aa9df1f3082d5f1c33ea05

services/chat/docker-compose.ci.yml

@@ -52,7 +52,7 @@ services:
     user: root
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/chat/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/chat
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -50,7 +51,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/contacts/docker-compose.ci.yml

@@ -52,7 +52,7 @@ services:
     user: root
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/contacts/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/contacts
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -50,7 +51,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/contacts/test/unit/js/ContactsManagerTests.js

@@ -5,6 +5,7 @@ import { expect } from 'chai'
 import { ObjectId } from 'mongodb'
 import { cleanupTestDatabase } from '../../../app/js/mongodb.js'
 import * as ContactManager from '../../../app/js/ContactManager.js'
+import '../../acceptance/js/MongoHelper.js'
 
 describe('ContactManager', function () {
   beforeEach(cleanupTestDatabase)

services/docstore/docker-compose.ci.yml

@@ -57,7 +57,7 @@ services:
     user: root
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/docstore/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/docstore
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -55,7 +56,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/document-updater/docker-compose.ci.yml

@@ -71,7 +71,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/document-updater/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/document-updater
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -69,7 +70,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/history-v1/docker-compose.ci.yml

@@ -95,7 +95,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/history-v1/docker-compose.yml

@@ -16,6 +16,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/history-v1
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -99,7 +100,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/history-v1/test/setup.js

@@ -20,7 +20,8 @@ async function setupPostgresDatabase() {
 async function setupMongoDatabase() {
   this.timeout(60_000)
   await promisify(exec)(
-    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+    // Run saas migrations for backup indexes
+    `cd ../../tools/migrations && npm run migrations -- migrate -t saas`
   )
 }
 

services/notifications/docker-compose.ci.yml

@@ -55,7 +55,7 @@ services:
     user: root
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/notifications/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
       - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
     working_dir: /overleaf/services/notifications
     environment:
@@ -52,7 +53,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/project-history/docker-compose.ci.yml

@@ -71,7 +71,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

services/project-history/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/project-history
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
@@ -69,7 +70,7 @@ services:
 
   mongo:
     image: mongo:8.0.11
-    command: --replSet overleaf
+    command: --replSet overleaf --notablescan
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
       - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh

tools/migrations/20230512100122_ensure_history_migration.mjs

@@ -6,6 +6,10 @@ const tags = ['server-ce', 'server-pro']
 
 const migrate = async client => {
   const { db } = client
+  const totalProjects = await db.projects.estimatedDocumentCount()
+  if (totalProjects === 0) return
+
+  // Does not use an index.
   const count = await db.projects.countDocuments({
     'overleaf.history.display': { $ne: true },
   })

tools/migrations/202407131109055_admin_reversed_hostname.mjs

@@ -1,66 +0,0 @@
-const tags = ['server-ce', 'server-pro']
-
-const migrate = async client => {
-  const { db } = client
-
-  const adminsWithEmails = await db.users
-    .find(
-      {
-        isAdmin: true,
-        emails: { $exists: true },
-      },
-      { _id: 1, emails: 1 }
-    )
-    .toArray()
-
-  for (const { _id, emails } of adminsWithEmails) {
-    let shouldUpdateEmails = false
-    for (const emailObj of emails) {
-      if (!emailObj.reversedHostname) {
-        shouldUpdateEmails = true
-        emailObj.reversedHostname = emailObj.email
-          .split('@')[1]
-          .split('')
-          .reverse()
-          .join('')
-      }
-    }
-    if (shouldUpdateEmails) {
-      await db.users.updateOne({ _id }, { $set: { emails } })
-    }
-  }
-
-  const adminsNoEmails = await db.users
-    .find(
-      {
-        isAdmin: true,
-        emails: { $exists: false },
-      },
-      { _id: 1, email: 1 }
-    )
-    .toArray()
-
-  for (const { _id, email } of adminsNoEmails) {
-    const reversedHostname = email.split('@')[1].split('').reverse().join('')
-    await db.users.updateOne(
-      { _id },
-      {
-        emails: [
-          {
-            email,
-            reversedHostname,
-            createdAt: new Date(),
-          },
-        ],
-      }
-    )
-  }
-}
-
-const rollback = async () => {}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

tools/migrations/20240713110905_emails_reversed_hostname.mjs

@@ -0,0 +1,40 @@
+const tags = ['server-ce', 'server-pro']
+
+function fixEmails(user) {
+  let emails = user.emails
+  if (!emails || emails.length === 0) {
+    emails = [{ email: user.email, createdAt: new Date() }]
+  }
+  for (let i = 0; i < emails.length; i++) {
+    const reversedHostname = emails[i].email
+      .split('@')[1]
+      .split('')
+      .reverse()
+      .join('')
+    if (emails[i].reversedHostname !== reversedHostname) {
+      emails = emails.slice()
+      emails[i].reversedHostname = reversedHostname
+    }
+  }
+  return emails
+}
+
+const migrate = async client => {
+  const { db } = client
+
+  const cursor = db.users.find({}, { projection: { email: 1, emails: 1 } })
+  for await (const user of cursor) {
+    const emails = fixEmails(user)
+    if (user.emails !== emails) {
+      await db.users.updateOne({ _id: user._id }, { $set: { emails } })
+    }
+  }
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}