Merge pull request #10911 from overleaf/em-upgrade-mongo-docstore

Upgrade the Mongo driver in docstore

GitOrigin-RevId: 35e5de558e7900fcb455f1044703912077273d47

package-lock.json

@@ -36209,11 +36209,11 @@
         "bunyan": "^1.8.15",
         "celebrate": "^13.0.4",
         "express": "^4.17.1",
-        "mongodb": "^3.6.0",
+        "lodash": "^4.17.21",
+        "mongodb": "^4.11.0",
         "p-map": "^4.0.0",
         "request": "^2.88.2",
-        "streamifier": "^0.1.1",
-        "underscore": "~1.12.1"
+        "streamifier": "^0.1.1"
       },
       "devDependencies": {
         "@google-cloud/storage": "^5.1.2",
@@ -36258,6 +36258,29 @@
         "node": ">=10"
       }
     },
+    "services/docstore/node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "services/docstore/node_modules/celebrate": {
       "version": "13.0.4",
       "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@@ -36274,6 +36297,14 @@
       "integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
       "dev": true
     },
+    "services/docstore/node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "services/docstore/node_modules/diff": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -36335,6 +36366,35 @@
         "node": ">=10.0.0"
       }
     },
+    "services/docstore/node_modules/mongodb": {
+      "version": "4.11.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
+      "integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
+      "dependencies": {
+        "bson": "^4.7.0",
+        "denque": "^2.1.0",
+        "mongodb-connection-string-url": "^2.5.4",
+        "socks": "^2.7.1"
+      },
+      "engines": {
+        "node": ">=12.9.0"
+      },
+      "optionalDependencies": {
+        "@aws-sdk/credential-providers": "^3.186.0",
+        "saslprep": "^1.0.3"
+      }
+    },
+    "services/docstore/node_modules/mongodb/node_modules/bson": {
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
+      "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
+      "dependencies": {
+        "buffer": "^5.6.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "services/docstore/node_modules/sinon": {
       "version": "9.0.3",
       "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@@ -36366,11 +36426,6 @@
         "node": ">=8"
       }
     },
-    "services/docstore/node_modules/underscore": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
-      "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
-    },
     "services/document-updater": {
       "name": "@overleaf/document-updater",
       "dependencies": {
@@ -46560,15 +46615,15 @@
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "express": "^4.17.1",
+        "lodash": "^4.17.21",
         "mocha": "^8.4.0",
-        "mongodb": "^3.6.0",
+        "mongodb": "^4.11.0",
         "p-map": "^4.0.0",
         "request": "^2.88.2",
         "sandboxed-module": "~2.0.4",
         "sinon": "~9.0.2",
         "sinon-chai": "^3.7.0",
-        "streamifier": "^0.1.1",
-        "underscore": "~1.12.1"
+        "streamifier": "^0.1.1"
       },
       "dependencies": {
         "@google-cloud/storage": {
@@ -46601,6 +46656,15 @@
             "xdg-basedir": "^4.0.0"
           }
         },
+        "buffer": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+          "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+          "requires": {
+            "base64-js": "^1.3.1",
+            "ieee754": "^1.1.13"
+          }
+        },
         "celebrate": {
           "version": "13.0.4",
           "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@@ -46617,6 +46681,11 @@
           "integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
           "dev": true
         },
+        "denque": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+          "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
+        },
         "diff": {
           "version": "4.0.2",
           "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -46660,6 +46729,29 @@
           "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
           "dev": true
         },
+        "mongodb": {
+          "version": "4.11.0",
+          "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
+          "integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
+          "requires": {
+            "@aws-sdk/credential-providers": "^3.186.0",
+            "bson": "^4.7.0",
+            "denque": "^2.1.0",
+            "mongodb-connection-string-url": "^2.5.4",
+            "saslprep": "^1.0.3",
+            "socks": "^2.7.1"
+          },
+          "dependencies": {
+            "bson": {
+              "version": "4.7.0",
+              "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
+              "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
+              "requires": {
+                "buffer": "^5.6.0"
+              }
+            }
+          }
+        },
         "sinon": {
           "version": "9.0.3",
           "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@@ -46683,11 +46775,6 @@
           "requires": {
             "has-flag": "^4.0.0"
           }
-        },
-        "underscore": {
-          "version": "1.12.1",
-          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
-          "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
         }
       }
     },

services/docstore/app.js

@@ -15,7 +15,7 @@ const {
   Joi,
   errors: handleValidationErrors,
 } = require('celebrate')
-const mongodb = require('./app/js/mongodb')
+const { mongoClient } = require('./app/js/mongodb')
 const Errors = require('./app/js/Errors')
 const HttpController = require('./app/js/HttpController')
 
@@ -101,8 +101,8 @@ const { host } = Settings.internal.docstore
 
 if (!module.parent) {
   // Called directly
-  mongodb
-    .waitForDb()
+  mongoClient
+    .connect()
     .then(() => {
       const server = app.listen(port, host, function (err) {
         if (err) {

services/docstore/app/js/DocManager.js

@@ -15,7 +15,7 @@ let DocManager
 const MongoManager = require('./MongoManager')
 const Errors = require('./Errors')
 const logger = require('@overleaf/logger')
-const _ = require('underscore')
+const _ = require('lodash')
 const DocArchive = require('./DocArchiveManager')
 const RangeManager = require('./RangeManager')
 const Settings = require('@overleaf/settings')
@@ -157,7 +157,7 @@ module.exports = DocManager = {
             )
             return callback(err)
           }
-          doc = _.extend(doc, archivedDoc)
+          Object.assign(doc, archivedDoc)
           callback(null, doc)
         })
       }

services/docstore/app/js/HealthChecker.js

@@ -12,7 +12,7 @@
 const { db, ObjectId } = require('./mongodb')
 const request = require('request')
 const async = require('async')
-const _ = require('underscore')
+const _ = require('lodash')
 const crypto = require('crypto')
 const settings = require('@overleaf/settings')
 const { port } = settings.internal.docstore

services/docstore/app/js/RangeManager.js

@@ -12,7 +12,7 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 let RangeManager
-const _ = require('underscore')
+const _ = require('lodash')
 const { ObjectId } = require('./mongodb')
 
 module.exports = RangeManager = {

services/docstore/app/js/mongodb.js

@@ -1,36 +1,16 @@
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
 
-const clientPromise = MongoClient.connect(
-  Settings.mongo.url,
-  Settings.mongo.options
-)
+const mongoClient = new MongoClient(Settings.mongo.url)
+const mongoDb = mongoClient.db()
 
-let setupDbPromise
-async function waitForDb() {
-  if (!setupDbPromise) {
-    setupDbPromise = setupDb()
-  }
-  await setupDbPromise
-}
-
-const db = {}
-async function setupDb() {
-  const internalDb = (await clientPromise).db()
-
-  db.docs = internalDb.collection('docs')
-  db.docOps = internalDb.collection('docOps')
-}
-async function addCollection(name) {
-  await waitForDb()
-  const internalDb = (await clientPromise).db()
-
-  db[name] = internalDb.collection(name)
+const db = {
+  docs: mongoDb.collection('docs'),
+  docOps: mongoDb.collection('docOps'),
 }
 
 module.exports = {
   db,
+  mongoClient,
   ObjectId,
-  addCollection,
-  waitForDb,
 }

services/docstore/config/settings.defaults.js

@@ -9,12 +9,7 @@ const Settings = {
     },
   },
 
-  mongo: {
-    options: {
-      useUnifiedTopology:
-        (process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
-    },
-  },
+  mongo: {},
 
   docstore: {
     archiveOnSoftDelete: process.env.ARCHIVE_ON_SOFT_DELETE === 'true',

services/docstore/package.json

@@ -27,11 +27,11 @@
     "bunyan": "^1.8.15",
     "celebrate": "^13.0.4",
     "express": "^4.17.1",
-    "mongodb": "^3.6.0",
+    "lodash": "^4.17.21",
+    "mongodb": "^4.11.0",
     "p-map": "^4.0.0",
     "request": "^2.88.2",
-    "streamifier": "^0.1.1",
-    "underscore": "~1.12.1"
+    "streamifier": "^0.1.1"
   },
   "devDependencies": {
     "@google-cloud/storage": "^5.1.2",

services/docstore/test/acceptance/js/helpers/DocstoreApp.js

@@ -1,5 +1,4 @@
 const app = require('../../../../app')
-const { waitForDb } = require('../../../../app/js/mongodb')
 require('@overleaf/logger').logger.level('error')
 const settings = require('@overleaf/settings')
 
@@ -8,9 +7,6 @@ module.exports = {
   initing: false,
   callbacks: [],
   ensureRunning(callback) {
-    if (callback == null) {
-      callback = function () {}
-    }
     if (this.running) {
       return callback()
     } else if (this.initing) {
@@ -18,20 +14,14 @@ module.exports = {
     }
     this.initing = true
     this.callbacks.push(callback)
-    waitForDb().then(() => {
-      return app.listen(settings.internal.docstore.port, 'localhost', error => {
-        if (error != null) {
-          throw error
-        }
-        this.running = true
-        return (() => {
-          const result = []
-          for (callback of Array.from(this.callbacks)) {
-            result.push(callback())
-          }
-          return result
-        })()
-      })
+    app.listen(settings.internal.docstore.port, 'localhost', error => {
+      if (error != null) {
+        throw error
+      }
+      this.running = true
+      for (callback of Array.from(this.callbacks)) {
+        callback()
+      }
     })
   },
 }

services/docstore/test/setup.js

@@ -30,7 +30,7 @@ SandboxedModule.configure({
   requires: {
     '@overleaf/logger': stubs.logger,
   },
-  globals: { Buffer, JSON, console, process },
+  globals: { Buffer, JSON, Math, console, process },
 })
 
 exports.mochaHooks = {

services/docstore/test/unit/js/RangeManagerTests.js

@@ -18,7 +18,6 @@ const modulePath = require('path').join(
   '../../../app/js/RangeManager'
 )
 const { ObjectId } = require('mongodb')
-const _ = require('underscore')
 
 describe('RangeManager', function () {
   beforeEach(function () {