Merge pull request #10998 from overleaf/em-upgrade-mongo-document-updater

Upgrade Mongo driver in document updater

GitOrigin-RevId: e3c4d86bbcf86dcf16fdfdbfab2aaf0d8eb1037c

package-lock.json

@@ -35782,7 +35782,7 @@
         "diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz",
         "express": "4.17.1",
         "lodash": "^4.17.21",
-        "mongodb": "^3.6.6",
+        "mongodb": "^4.11.0",
         "request": "^2.88.2",
         "requestretry": "^4.1.2"
       },
@@ -35816,6 +35816,40 @@
         "node": ">= 0.8"
       }
     },
+    "services/document-updater/node_modules/bson": {
+      "version": "4.7.1",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz",
+      "integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==",
+      "dependencies": {
+        "buffer": "^5.6.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "services/document-updater/node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "services/document-updater/node_modules/bytes": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@@ -35927,6 +35961,23 @@
         "node": ">=4"
       }
     },
+    "services/document-updater/node_modules/mongodb": {
+      "version": "4.13.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
+      "integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==",
+      "dependencies": {
+        "bson": "^4.7.0",
+        "mongodb-connection-string-url": "^2.5.4",
+        "socks": "^2.7.1"
+      },
+      "engines": {
+        "node": ">=12.9.0"
+      },
+      "optionalDependencies": {
+        "@aws-sdk/credential-providers": "^3.186.0",
+        "saslprep": "^1.0.3"
+      }
+    },
     "services/document-updater/node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -46853,7 +46904,7 @@
         "express": "4.17.1",
         "lodash": "^4.17.21",
         "mocha": "^8.4.0",
-        "mongodb": "^3.6.6",
+        "mongodb": "^4.11.0",
         "request": "^2.88.2",
         "requestretry": "^4.1.2",
         "sandboxed-module": "^2.0.4",
@@ -46878,6 +46929,23 @@
             "type-is": "~1.6.17"
           }
         },
+        "bson": {
+          "version": "4.7.1",
+          "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz",
+          "integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==",
+          "requires": {
+            "buffer": "^5.6.0"
+          }
+        },
+        "buffer": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+          "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+          "requires": {
+            "base64-js": "^1.3.1",
+            "ieee754": "^1.1.13"
+          }
+        },
         "bytes": {
           "version": "3.1.0",
           "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@@ -46968,6 +47036,18 @@
           "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
           "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
         },
+        "mongodb": {
+          "version": "4.13.0",
+          "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
+          "integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==",
+          "requires": {
+            "@aws-sdk/credential-providers": "^3.186.0",
+            "bson": "^4.7.0",
+            "mongodb-connection-string-url": "^2.5.4",
+            "saslprep": "^1.0.3",
+            "socks": "^2.7.1"
+          }
+        },
         "ms": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",

services/document-updater/app.js

@@ -225,8 +225,8 @@ const host = Settings.internal.documentupdater.host || 'localhost'
 
 if (!module.parent) {
   // Called directly
-  mongodb
-    .waitForDb()
+  mongodb.mongoClient
+    .connect()
     .then(() => {
       app.listen(port, host, function (err) {
         if (err) {

services/document-updater/app/js/mongodb.js

@@ -1,37 +1,25 @@
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
 
-const clientPromise = MongoClient.connect(
-  Settings.mongo.url,
-  Settings.mongo.options
-)
+const mongoClient = new MongoClient(Settings.mongo.url)
+const mongoDb = mongoClient.db()
+
+const db = {
+  docs: mongoDb.collection('docs'),
+  docSnapshots: mongoDb.collection('docSnapshots'),
+  projects: mongoDb.collection('projects'),
+}
 
 async function healthCheck() {
-  const internalDb = (await clientPromise).db()
-  const res = await internalDb.command({ ping: 1 })
+  const res = await mongoDb.command({ ping: 1 })
   if (!res.ok) {
     throw new Error('failed mongo ping')
   }
 }
 
-let setupDbPromise
-async function waitForDb() {
-  if (!setupDbPromise) {
-    setupDbPromise = setupDb()
-  }
-  await setupDbPromise
-}
-
-const db = {}
-async function setupDb() {
-  const internalDb = (await clientPromise).db()
-
-  db.docSnapshots = internalDb.collection('docSnapshots')
-}
-
 module.exports = {
   db,
   ObjectId,
+  mongoClient,
   healthCheck: require('util').callbackify(healthCheck),
-  waitForDb,
 }

services/document-updater/config/settings.defaults.js

@@ -166,10 +166,6 @@ module.exports = {
   dispatcherCount: parseInt(process.env.DISPATCHER_COUNT || 10, 10),
 
   mongo: {
-    options: {
-      useUnifiedTopology:
-        (process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
-    },
     url:
       process.env.MONGO_CONNECTION_STRING ||
       `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`,

services/document-updater/package.json

@@ -28,7 +28,7 @@
     "diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz",
     "express": "4.17.1",
     "lodash": "^4.17.21",
-    "mongodb": "^3.6.6",
+    "mongodb": "^4.11.0",
     "request": "^2.88.2",
     "requestretry": "^4.1.2"
   },

services/document-updater/scripts/fix_docs_with_missing_project.js

@@ -4,25 +4,14 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
   Settings.redis.documentupdater
 )
 const keys = Settings.redis.documentupdater.key_schema
-const ProjectFlusher = require('app/js/ProjectFlusher')
-const DocumentManager = require('app/js/DocumentManager')
+const ProjectFlusher = require('../app/js/ProjectFlusher')
+const DocumentManager = require('../app/js/DocumentManager')
+const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
 const util = require('util')
 const flushAndDeleteDocWithLock = util.promisify(
   DocumentManager.flushAndDeleteDocWithLock
 )
 
-const { MongoClient, ObjectId } = require('mongodb')
-
-const clientPromise = MongoClient.connect(
-  Settings.mongo.url,
-  Settings.mongo.options
-)
-
-const db = {}
-clientPromise.then(client => {
-  db.docs = client.db().collection('docs')
-})
-
 async function fixDocsWithMissingProjectIds(dockeys, options) {
   const docIds = ProjectFlusher._extractIds(dockeys)
   for (const docId of docIds) {
@@ -86,15 +75,13 @@ async function findAndProcessDocs(options) {
   } while (cursor !== '0')
 }
 
-clientPromise.then(client => {
-  findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
-    .then(result => {
-      rclient.quit()
-      client.close()
-      console.log('DONE')
-    })
-    .catch(function (error) {
-      console.error(error)
-      process.exit(1)
-    })
-})
+findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
+  .then(result => {
+    rclient.quit()
+    mongoClient.close()
+    console.log('DONE')
+  })
+  .catch(function (error) {
+    console.error(error)
+    process.exit(1)
+  })

services/document-updater/scripts/remove_deleted_docs.js

@@ -4,28 +4,17 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
   Settings.redis.documentupdater
 )
 const keys = Settings.redis.documentupdater.key_schema
-const ProjectFlusher = require('app/js/ProjectFlusher')
-const RedisManager = require('app/js/RedisManager')
+const ProjectFlusher = require('../app/js/ProjectFlusher')
+const RedisManager = require('../app/js/RedisManager')
+const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
 const util = require('util')
 const getDoc = util.promisify((projectId, docId, cb) =>
   RedisManager.getDoc(projectId, docId, (err, ...args) => cb(err, args))
 )
 const removeDocFromMemory = util.promisify(RedisManager.removeDocFromMemory)
-const { MongoClient, ObjectId } = require('mongodb')
-
-const clientPromise = MongoClient.connect(
-  Settings.mongo.url,
-  Settings.mongo.options
-)
 
 const summary = { totalDocs: 0, deletedDocs: 0, skippedDocs: 0 }
 
-const db = {}
-clientPromise.then(client => {
-  db.docs = client.db().collection('docs')
-  db.projects = client.db().collection('projects')
-})
-
 async function removeDeletedDocs(dockeys, options) {
   const docIds = ProjectFlusher._extractIds(dockeys)
   for (const docId of docIds) {
@@ -159,15 +148,13 @@ async function findAndProcessDocs(options) {
   } while (cursor !== '0')
 }
 
-clientPromise.then(client => {
-  findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
-    .then(result => {
-      rclient.quit()
-      client.close()
-      console.log('DONE')
-    })
-    .catch(function (error) {
-      console.error(error)
-      process.exit(1)
-    })
-})
+findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
+  .then(result => {
+    rclient.quit()
+    mongoClient.close()
+    console.log('DONE')
+  })
+  .catch(function (error) {
+    console.error(error)
+    process.exit(1)
+  })

services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js

@@ -9,7 +9,6 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 const app = require('../../../../app')
-const { waitForDb } = require('../../../../app/js/mongodb')
 require('@overleaf/logger').logger.level('fatal')
 
 module.exports = {
@@ -27,20 +26,18 @@ module.exports = {
     }
     this.initing = true
     this.callbacks.push(callback)
-    waitForDb().then(() => {
-      return app.listen(3003, 'localhost', error => {
-        if (error != null) {
-          throw error
+    app.listen(3003, 'localhost', error => {
+      if (error != null) {
+        throw error
+      }
+      this.running = true
+      return (() => {
+        const result = []
+        for (callback of Array.from(this.callbacks)) {
+          result.push(callback())
         }
-        this.running = true
-        return (() => {
-          const result = []
-          for (callback of Array.from(this.callbacks)) {
-            result.push(callback())
-          }
-          return result
-        })()
-      })
+        return result
+      })()
     })
   },
 }