[misc] migrate the app to the native mongo driver

acceptance tests to follow in a separate commit.

services/track-changes/app.js

@@ -44,6 +44,7 @@ Metrics.memory.monitor(logger)
 
 const childProcess = require('child_process')
 
+const mongodb = require('./app/js/mongodb')
 const HttpController = require('./app/js/HttpController')
 const express = require('express')
 const bodyParser = require('body-parser')
@@ -128,18 +129,26 @@ const host =
 
 if (!module.parent) {
   // Called directly
-  app.listen(port, host, function (error) {
-    if (error != null) {
-      return logger.error(
-        { err: error },
-        'could not start track-changes server'
-      )
-    } else {
-      return logger.info(
-        `trackchanges starting up, listening on ${host}:${port}`
-      )
-    }
-  })
+  mongodb
+    .waitForDb()
+    .then(() => {
+      app.listen(port, host, function (error) {
+        if (error != null) {
+          return logger.error(
+            { err: error },
+            'could not start track-changes server'
+          )
+        } else {
+          return logger.info(
+            `trackchanges starting up, listening on ${host}:${port}`
+          )
+        }
+      })
+    })
+    .catch((err) => {
+      logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')
+      process.exit(1)
+    })
 }
 
 module.exports = app

services/track-changes/app/js/HealthChecker.js

@@ -10,7 +10,7 @@
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
-const { ObjectId } = require('mongojs')
+const { ObjectId } = require('./mongodb')
 const request = require('request')
 const async = require('async')
 const settings = require('settings-sharelatex')

services/track-changes/app/js/MongoAWS.js

@@ -18,7 +18,7 @@ const settings = require('settings-sharelatex')
 const logger = require('logger-sharelatex')
 const AWS = require('aws-sdk')
 const S3S = require('s3-streams')
-const { db, ObjectId } = require('./mongojs')
+const { db, ObjectId } = require('./mongodb')
 const JSONStream = require('JSONStream')
 const ReadlineStream = require('byline')
 const zlib = require('zlib')
@@ -187,7 +187,11 @@ module.exports = MongoAWS = {
       // allow the object to expire, we can always retrieve it again
       object.expiresAt = new Date(Date.now() + 7 * DAYS)
       logger.log({ project_id, doc_id, pack_id }, 'inserting object from s3')
-      return db.docHistory.insert(object, callback)
+      return db.docHistory.insertOne(object, (err, confirmation) => {
+        if (err) return callback(err)
+        object._id = confirmation.insertedId
+        callback(null, object)
+      })
     })
   }
 }

services/track-changes/app/js/MongoManager.js

@@ -12,7 +12,7 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 let MongoManager
-const { db, ObjectId } = require('./mongojs')
+const { db, ObjectId } = require('./mongodb')
 const PackManager = require('./PackManager')
 const async = require('async')
 const _ = require('underscore')
@@ -25,7 +25,11 @@ module.exports = MongoManager = {
       callback = function (error, update) {}
     }
     return db.docHistory
-      .find({ doc_id: ObjectId(doc_id.toString()) }, { pack: { $slice: -1 } }) // only return the last entry in a pack
+      .find(
+        { doc_id: ObjectId(doc_id.toString()) },
+        // only return the last entry in a pack
+        { projection: { pack: { $slice: -1 } } }
+      )
       .sort({ v: -1 })
       .limit(1)
       .toArray(function (error, compressedUpdates) {
@@ -96,7 +100,7 @@ module.exports = MongoManager = {
     if (callback == null) {
       callback = function (error) {}
     }
-    return db.docHistory.update(
+    return db.docHistory.updateMany(
       {
         doc_id: ObjectId(doc_id.toString()),
         project_id: { $exists: false }
@@ -104,9 +108,6 @@ module.exports = MongoManager = {
       {
         $set: { project_id: ObjectId(project_id.toString()) }
       },
-      {
-        multi: true
-      },
       callback
     )
   },
@@ -127,7 +128,7 @@ module.exports = MongoManager = {
     if (callback == null) {
       callback = function (error) {}
     }
-    return db.projectHistoryMetaData.update(
+    return db.projectHistoryMetaData.updateOne(
       {
         project_id: ObjectId(project_id)
       },
@@ -146,7 +147,7 @@ module.exports = MongoManager = {
     if (callback == null) {
       callback = function (error) {}
     }
-    return db.docHistory.update(
+    return db.docHistory.updateMany(
       {
         project_id: ObjectId(project_id),
         temporary: true,
@@ -156,9 +157,6 @@ module.exports = MongoManager = {
         $set: { temporary: false },
         $unset: { expiresAt: '' }
       },
-      {
-        multi: true
-      },
       callback
     )
   },

services/track-changes/app/js/PackManager.js

@@ -18,7 +18,7 @@ const async = require('async')
 const _ = require('underscore')
 const Bson = require('bson')
 const BSON = new Bson()
-const { db, ObjectId } = require('./mongojs')
+const { db, ObjectId } = require('./mongodb')
 const logger = require('logger-sharelatex')
 const LockManager = require('./LockManager')
 const MongoAWS = require('./MongoAWS')
@@ -220,7 +220,7 @@ module.exports = PackManager = {
       { project_id, doc_id, newUpdates },
       'inserting updates into new pack'
     )
-    return db.docHistory.save(newPack, function (err, result) {
+    return db.docHistory.insertOne(newPack, function (err) {
       if (err != null) {
         return callback(err)
       }
@@ -381,7 +381,10 @@ module.exports = PackManager = {
   fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
     let id
     return db.docHistory
-      .find({ _id: { $in: pack_ids.map(ObjectId) } }, { _id: 1 })
+      .find(
+        { _id: { $in: pack_ids.map(ObjectId) } },
+        { projection: { _id: 1 } }
+      )
       .toArray(function (err, loadedPacks) {
         if (err != null) {
           return callback(err)
@@ -425,7 +428,10 @@ module.exports = PackManager = {
   makeProjectIterator(project_id, before, callback) {
     // get all the docHistory Entries
     return db.docHistory
-      .find({ project_id: ObjectId(project_id) }, { pack: false })
+      .find(
+        { project_id: ObjectId(project_id) },
+        { projection: { pack: false } }
+      )
       .sort({ 'meta.end_ts': -1 })
       .toArray(function (err, packs) {
         let pack
@@ -507,7 +513,7 @@ module.exports = PackManager = {
   getPackFromIndex(doc_id, pack_id, callback) {
     return db.docHistoryIndex.findOne(
       { _id: ObjectId(doc_id.toString()), 'packs._id': pack_id },
-      { 'packs.$': 1 },
+      { projection: { 'packs.$': 1 } },
       callback
     )
   },
@@ -515,7 +521,7 @@ module.exports = PackManager = {
   getLastPackFromIndex(doc_id, callback) {
     return db.docHistoryIndex.findOne(
       { _id: ObjectId(doc_id.toString()) },
-      { packs: { $slice: -1 } },
+      { projection: { packs: { $slice: -1 } } },
       function (err, indexPack) {
         if (err != null) {
           return callback(err)
@@ -602,7 +608,7 @@ module.exports = PackManager = {
       expiresAt: { $exists: false }
     }
     return db.docHistory
-      .find(query, { pack: false })
+      .find(query, { projection: { pack: false } })
       .sort({ v: 1 })
       .toArray(function (err, packs) {
         if (err != null) {
@@ -628,7 +634,7 @@ module.exports = PackManager = {
       expiresAt: { $exists: false }
     }
     return db.docHistory
-      .find(query, { pack: false })
+      .find(query, { projection: { pack: false } })
       .sort({ v: 1 })
       .toArray(function (err, packs) {
         if (err != null) {
@@ -1069,20 +1075,18 @@ module.exports = PackManager = {
       { project_id, doc_id },
       'marking pack as archive in progress status'
     )
-    return db.docHistoryIndex.findAndModify(
+    return db.docHistoryIndex.findOneAndUpdate(
       {
-        query: {
-          _id: ObjectId(doc_id.toString()),
-          packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
-        },
-        fields: { 'packs.$': 1 },
-        update: { $set: { 'packs.$.inS3': false } }
+        _id: ObjectId(doc_id.toString()),
+        packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
       },
+      { $set: { 'packs.$.inS3': false } },
+      { projection: { 'packs.$': 1 } },
       function (err, result) {
         if (err != null) {
           return callback(err)
         }
-        if (result == null) {
+        if (!result.value) {
           return callback(new Error('archive is already in progress'))
         }
         logger.log(
@@ -1111,20 +1115,18 @@ module.exports = PackManager = {
 
   markPackAsArchived(project_id, doc_id, pack_id, callback) {
     logger.log({ project_id, doc_id, pack_id }, 'marking pack as archived')
-    return db.docHistoryIndex.findAndModify(
+    return db.docHistoryIndex.findOneAndUpdate(
       {
-        query: {
-          _id: ObjectId(doc_id.toString()),
-          packs: { $elemMatch: { _id: pack_id, inS3: false } }
-        },
-        fields: { 'packs.$': 1 },
-        update: { $set: { 'packs.$.inS3': true } }
+        _id: ObjectId(doc_id.toString()),
+        packs: { $elemMatch: { _id: pack_id, inS3: false } }
       },
+      { $set: { 'packs.$.inS3': true } },
+      { projection: { 'packs.$': 1 } },
       function (err, result) {
         if (err != null) {
           return callback(err)
         }
-        if (result == null) {
+        if (!result.value) {
           return callback(new Error('archive is not marked as progress'))
         }
         logger.log({ project_id, doc_id, pack_id }, 'marked as archived')

services/track-changes/app/js/PackWorker.js

@@ -18,7 +18,7 @@ let project_id, doc_id
 const Settings = require('settings-sharelatex')
 const async = require('async')
 const _ = require('underscore')
-const { db, ObjectId } = require('./mongojs')
+const { db, ObjectId } = require('./mongodb')
 const fs = require('fs')
 const Metrics = require('metrics-sharelatex')
 Metrics.initialize('track-changes')
@@ -174,7 +174,7 @@ if (pending != null) {
         _id: { $lt: ObjectIdFromDate(oneWeekAgo) },
         last_checked: { $lt: oneWeekAgo }
       },
-      { _id: 1, doc_id: 1, project_id: 1 }
+      { projection: { _id: 1, doc_id: 1, project_id: 1 } }
     )
     .sort({
       last_checked: 1

services/track-changes/app/js/mongodb.js

@@ -0,0 +1,30 @@
+const Settings = require('settings-sharelatex')
+const { MongoClient, ObjectId } = require('mongodb')
+
+const clientPromise = MongoClient.connect(
+  Settings.mongo.url,
+  Settings.mongo.options
+)
+
+let setupDbPromise
+async function waitForDb() {
+  if (!setupDbPromise) {
+    setupDbPromise = setupDb()
+  }
+  await setupDbPromise
+}
+
+const db = {}
+async function setupDb() {
+  const internalDb = (await clientPromise).db()
+
+  db.docHistory = internalDb.collection('docHistory')
+  db.docHistoryIndex = internalDb.collection('docHistoryIndex')
+  db.projectHistoryMetaData = internalDb.collection('projectHistoryMetaData')
+}
+
+module.exports = {
+  db,
+  ObjectId,
+  waitForDb
+}

services/track-changes/config/settings.defaults.js

@@ -4,6 +4,10 @@ const TMP_DIR =
 
 module.exports = {
   mongo: {
+    options: {
+      useUnifiedTopology:
+        (process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true'
+    },
     url:
       process.env.MONGO_CONNECTION_STRING ||
       `mongodb://${process.env.MONGO_HOST || 'localhost'}/sharelatex`

services/track-changes/package-lock.json

@@ -1496,9 +1496,9 @@
       "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g=="
     },
     "bl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
-      "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
+      "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
       "requires": {
         "readable-stream": "^2.3.5",
         "safe-buffer": "^5.1.1"
@@ -4717,12 +4717,12 @@
       "integrity": "sha512-FehPVi2Dv7VPvAkLnN9haM1aarj1E9w08rkn2MAbbQJF5EbcOckdOHRAD9T35yUkfLVcs0YzYluNX4/+G8HaIw=="
     },
     "mongodb": {
-      "version": "3.5.5",
-      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.5.tgz",
-      "integrity": "sha512-GCjDxR3UOltDq00Zcpzql6dQo1sVry60OXJY3TDmFc2SWFY6c8Gn1Ardidc5jDirvJrx2GC3knGOImKphbSL3A==",
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.1.tgz",
+      "integrity": "sha512-uH76Zzr5wPptnjEKJRQnwTsomtFOU/kQEU8a9hKHr2M7y9qVk7Q4Pkv0EQVp88742z9+RwvsdTw6dRjDZCNu1g==",
       "requires": {
         "bl": "^2.2.0",
-        "bson": "^1.1.1",
+        "bson": "^1.1.4",
         "denque": "^1.4.1",
         "require_optional": "^1.0.1",
         "safe-buffer": "^5.1.2",

services/track-changes/package.json

@@ -30,6 +30,7 @@
     "logger-sharelatex": "^2.2.0",
     "metrics-sharelatex": "^2.6.2",
     "mongo-uri": "^0.1.2",
+    "mongodb": "^3.6.0",
     "mongojs": "3.1.0",
     "redis": "~0.10.1",
     "redis-sharelatex": "^1.0.13",

services/track-changes/test/acceptance/js/helpers/TrackChangesApp.js

@@ -13,6 +13,7 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 const app = require('../../../../app')
+const { waitForDb } = require('../../../../app/js/mongodb')
 require('logger-sharelatex')
 const logger = require('logger-sharelatex')
 const Settings = require('settings-sharelatex')
@@ -29,9 +30,10 @@ module.exports = {
       return callback()
     } else if (this.initing) {
       return this.callbacks.push(callback)
-    } else {
-      this.initing = true
-      this.callbacks.push(callback)
+    }
+    this.initing = true
+    this.callbacks.push(callback)
+    waitForDb().then(() => {
       return app.listen(
         __guard__(
           Settings.internal != null
@@ -56,7 +58,7 @@ module.exports = {
           })()
         }
       )
-    }
+    })
   }
 }
 function __guard__(value, transform) {

services/track-changes/test/unit/js/DocArchive/MongoAWS.js

@@ -14,7 +14,7 @@ chai.should()
 const sinon = require('sinon')
 const modulePath = '../../../../app/js/MongoAWS.js'
 const SandboxedModule = require('sandboxed-module')
-const { ObjectId } = require('mongojs')
+const { ObjectId } = require('mongodb')
 const MemoryStream = require('memorystream')
 const zlib = require('zlib')
 
@@ -44,7 +44,7 @@ describe('MongoAWS', function () {
         'aws-sdk': (this.awssdk = {}),
         fs: (this.fs = {}),
         's3-streams': (this.S3S = {}),
-        './mongojs': { db: (this.db = {}), ObjectId },
+        './mongodb': { db: (this.db = {}), ObjectId },
         JSONStream: (this.JSONStream = {}),
         'readline-stream': (this.readline = sinon.stub()),
         'metrics-sharelatex': { inc() {} }
@@ -92,7 +92,9 @@ describe('MongoAWS', function () {
         this.S3S.ReadStream = () =>
           MemoryStream.createReadStream(zbuf, { readable: true })
         this.db.docHistory = {}
-        this.db.docHistory.insert = sinon.stub().callsArgWith(1, null, 'pack')
+        this.db.docHistory.insertOne = sinon
+          .stub()
+          .yields(null, { insertedId: ObjectId() })
 
         return this.MongoAWS.unArchivePack(
           this.project_id,
@@ -107,7 +109,7 @@ describe('MongoAWS', function () {
     })
 
     return it('should call db.docHistory.insert', function () {
-      return this.db.docHistory.insert.called.should.equal(true)
+      return this.db.docHistory.insertOne.called.should.equal(true)
     })
   })
 })

services/track-changes/test/unit/js/MongoManager/MongoManagerTests.js

@@ -16,7 +16,7 @@ const { expect } = chai
 const modulePath = '../../../../app/js/MongoManager.js'
 const packModulePath = '../../../../app/js/PackManager.js'
 const SandboxedModule = require('sandboxed-module')
-const { ObjectId } = require('mongojs')
+const { ObjectId } = require('mongodb')
 const tk = require('timekeeper')
 
 describe('MongoManager', function () {
@@ -24,7 +24,7 @@ describe('MongoManager', function () {
     tk.freeze(new Date())
     this.MongoManager = SandboxedModule.require(modulePath, {
       requires: {
-        './mongojs': { db: (this.db = {}), ObjectId },
+        './mongodb': { db: (this.db = {}), ObjectId },
         './PackManager': (this.PackManager = {}),
         'metrics-sharelatex': { timeAsyncMethod() {} },
         'logger-sharelatex': { log() {} }
@@ -156,7 +156,7 @@ describe('MongoManager', function () {
 
   describe('backportProjectId', function () {
     beforeEach(function () {
-      this.db.docHistory = { update: sinon.stub().callsArg(3) }
+      this.db.docHistory = { updateMany: sinon.stub().yields() }
       return this.MongoManager.backportProjectId(
         this.project_id,
         this.doc_id,
@@ -165,7 +165,7 @@ describe('MongoManager', function () {
     })
 
     it("should insert the project_id into all entries for the doc_id which don't have it set", function () {
-      return this.db.docHistory.update
+      return this.db.docHistory.updateMany
         .calledWith(
           {
             doc_id: ObjectId(this.doc_id),
@@ -173,9 +173,6 @@ describe('MongoManager', function () {
           },
           {
             $set: { project_id: ObjectId(this.project_id) }
-          },
-          {
-            multi: true
           }
         )
         .should.equal(true)
@@ -213,7 +210,7 @@ describe('MongoManager', function () {
     beforeEach(function () {
       this.metadata = { mock: 'metadata' }
       this.db.projectHistoryMetaData = {
-        update: sinon.stub().callsArgWith(3, null, [this.metadata])
+        updateOne: sinon.stub().yields()
       }
       return this.MongoManager.setProjectMetaData(
         this.project_id,
@@ -223,7 +220,7 @@ describe('MongoManager', function () {
     })
 
     it('should upsert the metadata into the DB', function () {
-      return this.db.projectHistoryMetaData.update
+      return this.db.projectHistoryMetaData.updateOne
         .calledWith(
           {
             project_id: ObjectId(this.project_id)

services/track-changes/test/unit/js/PackManager/PackManagerTests.js

@@ -17,7 +17,7 @@ const should = chai.should()
 const { expect } = chai
 const modulePath = '../../../../app/js/PackManager.js'
 const SandboxedModule = require('sandboxed-module')
-const { ObjectId } = require('mongojs')
+const { ObjectId } = require('mongodb')
 const _ = require('underscore')
 
 const tk = require('timekeeper')
@@ -28,7 +28,7 @@ describe('PackManager', function () {
     this.PackManager = SandboxedModule.require(modulePath, {
       requires: {
         bson: require('bson'),
-        './mongojs': { db: (this.db = {}), ObjectId },
+        './mongodb': { db: (this.db = {}), ObjectId },
         './LockManager': {},
         './MongoAWS': {},
         'logger-sharelatex': { log: sinon.stub(), error: sinon.stub() },
@@ -65,7 +65,7 @@ describe('PackManager', function () {
         { op: 'op-4', meta: 'meta-4', v: 4 }
       ]
       return (this.db.docHistory = {
-        save: sinon.stub().callsArg(1),
+        insertOne: sinon.stub().yields(),
         insert: sinon.stub().callsArg(1),
         updateOne: sinon.stub().yields(),
         findAndModify: sinon.stub().callsArg(1)
@@ -390,7 +390,7 @@ describe('PackManager', function () {
 
         return describe('for a small update  that will expire', function () {
           it('should insert the update into mongo', function () {
-            return this.db.docHistory.save
+            return this.db.docHistory.insertOne
               .calledWithMatch({
                 pack: this.newUpdates,
                 project_id: ObjectId(this.project_id),
@@ -403,7 +403,7 @@ describe('PackManager', function () {
           })
 
           it('should set an expiry time in the future', function () {
-            return this.db.docHistory.save
+            return this.db.docHistory.insertOne
               .calledWithMatch({
                 expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
               })
@@ -496,7 +496,7 @@ describe('PackManager', function () {
 
       return describe('for a small update that will not expire', function () {
         it('should insert the update into mongo', function () {
-          return this.db.docHistory.save
+          return this.db.docHistory.insertOne
             .calledWithMatch({
               pack: this.newUpdates,
               project_id: ObjectId(this.project_id),
@@ -509,7 +509,7 @@ describe('PackManager', function () {
         })
 
         it('should not set any expiry time', function () {
-          return this.db.docHistory.save
+          return this.db.docHistory.insertOne
             .neverCalledWithMatch(sinon.match.has('expiresAt'))
             .should.equal(true)
         })
@@ -546,7 +546,7 @@ describe('PackManager', function () {
 
       return describe('for a small update that will expire', function () {
         it('should insert the update into mongo', function () {
-          return this.db.docHistory.save
+          return this.db.docHistory.insertOne
             .calledWithMatch({
               pack: this.newUpdates,
               project_id: ObjectId(this.project_id),
@@ -559,7 +559,7 @@ describe('PackManager', function () {
         })
 
         it('should set an expiry time in the future', function () {
-          return this.db.docHistory.save
+          return this.db.docHistory.insertOne
             .calledWithMatch({
               expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
             })

services/track-changes/test/unit/js/UpdatesManager/UpdatesManagerTests.js

@@ -17,6 +17,7 @@ const sinon = require('sinon')
 const chai = require('chai')
 const should = chai.should()
 const { expect } = chai
+const { ObjectId } = require('mongodb')
 const modulePath = '../../../../app/js/UpdatesManager.js'
 const SandboxedModule = require('sandboxed-module')
 
@@ -864,7 +865,6 @@ describe('UpdatesManager', function () {
   describe('fillUserInfo', function () {
     describe('with valid users', function () {
       beforeEach(function (done) {
-        const { ObjectId } = require('mongojs')
         this.user_id_1 = ObjectId().toString()
         this.user_id_2 = ObjectId().toString()
         this.updates = [