Update unit tests for object-persistor

services/docstore/test/unit/js/DocArchiveManagerTests.js

@@ -1,47 +1,59 @@
-/* eslint-disable
-    camelcase,
-    handle-callback-err,
-    no-return-assign,
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-const { assert } = require('chai')
 const sinon = require('sinon')
 const chai = require('chai')
-const should = chai.should()
 const { expect } = chai
 const modulePath = '../../../app/js/DocArchiveManager.js'
 const SandboxedModule = require('sandboxed-module')
 const { ObjectId } = require('mongojs')
 const Errors = require('../../../app/js/Errors')
-const crypto = require('crypto')
+
+chai.use(require('chai-as-promised'))
+chai.use(require('sinon-chai'))
 
 describe('DocArchiveManager', function () {
+  let DocArchiveManager,
+    PersistorManager,
+    MongoManager,
+    RangeManager,
+    Settings,
+    Logger,
+    Crypto,
+    Streamifier,
+    HashDigest,
+    HashUpdate,
+    archivedDocs,
+    mongoDocs,
+    docJson,
+    md5Sum,
+    projectId,
+    readStream,
+    stream
+
   beforeEach(function () {
-    this.settings = {
+    md5Sum = 'decafbad'
+
+    RangeManager = {
+      jsonRangesToMongo: sinon.stub().returns({ mongo: 'ranges' })
+    }
+    Settings = {
       docstore: {
-        s3: {
-          secret: 'secret',
-          key: 'this_key',
-          bucket: 'doc-archive-unit-test'
-        }
+        bucket: 'wombat'
       }
     }
-
-    this.request = {
-      put: {},
-      get: {},
-      del: {}
+    Logger = {
+      log: sinon.stub(),
+      err: sinon.stub()
+    }
+    HashDigest = sinon.stub().returns(md5Sum)
+    HashUpdate = sinon.stub().returns({ digest: HashDigest })
+    Crypto = {
+      createHash: sinon.stub().returns({ update: HashUpdate })
+    }
+    Streamifier = {
+      createReadStream: sinon.stub().returns({ stream: 'readStream' })
     }
 
-    this.archivedDocs = [
+    projectId = ObjectId()
+    archivedDocs = [
       {
         _id: ObjectId(),
         inS3: true,
@@ -58,8 +70,7 @@ describe('DocArchiveManager', function () {
         rev: 6
       }
     ]
-
-    this.mongoDocs = [
+    mongoDocs = [
       {
         _id: ObjectId(),
         lines: ['one', 'two', 'three'],
@@ -87,488 +98,429 @@ describe('DocArchiveManager', function () {
       }
     ]
 
-    this.unarchivedDocs = [
-      {
-        _id: ObjectId(),
-        lines: ['wombat', 'potato', 'banana'],
-        rev: 2
-      },
-      {
-        _id: ObjectId(),
-        lines: ['llama', 'turnip', 'apple'],
-        rev: 4
-      },
-      {
-        _id: ObjectId(),
-        lines: ['elephant', 'swede', 'nectarine'],
-        rev: 6
-      }
-    ]
+    docJson = JSON.stringify({
+      lines: mongoDocs[0].lines,
+      ranges: mongoDocs[0].ranges,
+      schema_v: 1
+    })
 
-    this.mixedDocs = this.archivedDocs.concat(this.unarchivedDocs)
+    stream = {
+      on: sinon.stub(),
+      resume: sinon.stub()
+    }
+    stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+    stream.on.withArgs('end').yields()
 
-    this.MongoManager = {
-      markDocAsArchived: sinon.stub().callsArgWith(2, null),
-      upsertIntoDocCollection: sinon.stub().callsArgWith(3, null),
-      getProjectsDocs: sinon.stub().callsArgWith(3, null, this.mongoDocs),
-      getArchivedProjectDocs: sinon.stub().callsArgWith(2, null, this.mongoDocs)
+    readStream = {
+      stream: 'readStream'
     }
 
-    this.requires = {
-      'settings-sharelatex': this.settings,
-      './MongoManager': this.MongoManager,
-      request: this.request,
-      './RangeManager': (this.RangeManager = {}),
-      'logger-sharelatex': {
-        log() {},
-        err() {}
-      }
+    PersistorManager = {
+      getObjectStream: sinon.stub().resolves(stream),
+      sendStream: sinon.stub().resolves(),
+      getObjectMd5Hash: sinon.stub().resolves(md5Sum),
+      deleteObject: sinon.stub().resolves()
+    }
+
+    MongoManager = {
+      markDocAsArchived: sinon.stub().yields(),
+      upsertIntoDocCollection: sinon.stub().yields(),
+      getProjectsDocs: sinon.stub().yields(null, mongoDocs),
+      getArchivedProjectDocs: sinon.stub().yields(null, archivedDocs),
+      findDoc: sinon.stub().yields(),
+      destroyDoc: sinon.stub().yields()
+    }
+    for (const mongoDoc of mongoDocs) {
+      MongoManager.findDoc
+        .withArgs(projectId, mongoDoc._id, sinon.match.any)
+        .yields(null, mongoDoc)
     }
-    this.globals = { JSON }
-
-    this.error = 'my errror'
-    this.project_id = ObjectId().toString()
-    this.stubbedError = new Errors.NotFoundError('Error in S3 request')
-    return (this.DocArchiveManager = SandboxedModule.require(modulePath, {
-      requires: this.requires,
-      globals: this.globals
-    }))
+
+    DocArchiveManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'settings-sharelatex': Settings,
+        'logger-sharelatex': Logger,
+        crypto: Crypto,
+        streamifier: Streamifier,
+        './MongoManager': MongoManager,
+        './RangeManager': RangeManager,
+        './PersistorManager': PersistorManager,
+        './Errors': Errors
+      },
+      globals: {
+        console,
+        JSON
+      }
+    })
   })
 
   describe('archiveDoc', function () {
-    it('should use correct options', function (done) {
-      this.request.put = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 200, headers: { etag: '' } })
-      return this.DocArchiveManager.archiveDoc(
-        this.project_id,
-        this.mongoDocs[0],
-        (err) => {
-          const opts = this.request.put.args[0][0]
-          assert.deepEqual(opts.aws, {
-            key: this.settings.docstore.s3.key,
-            secret: this.settings.docstore.s3.secret,
-            bucket: this.settings.docstore.s3.bucket
-          })
-          opts.body.should.equal(
-            JSON.stringify({
-              lines: this.mongoDocs[0].lines,
-              ranges: this.mongoDocs[0].ranges,
-              schema_v: 1
-            })
-          )
-          opts.timeout.should.equal(30 * 1000)
-          opts.uri.should.equal(
-            `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
-          )
-          return done()
-        }
+    it('should resolve when passed a valid document', async function () {
+      await expect(
+        DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+      ).to.eventually.be.fulfilled
+    })
+
+    it('should throw an error if the doc has no lines', async function () {
+      const doc = mongoDocs[0]
+      doc.lines = null
+
+      await expect(
+        DocArchiveManager.promises.archiveDoc(projectId, doc)
+      ).to.eventually.be.rejectedWith('doc has no lines')
+    })
+
+    it('should add the schema version', async function () {
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[1])
+      expect(Streamifier.createReadStream).to.have.been.calledWith(
+        sinon.match(/"schema_v":1/)
       )
     })
 
-    it('should return no md5 error', function (done) {
-      const data = JSON.stringify({
-        lines: this.mongoDocs[0].lines,
-        ranges: this.mongoDocs[0].ranges,
+    it('should calculate the hex md5 sum of the content', async function () {
+      const json = JSON.stringify({
+        lines: mongoDocs[0].lines,
+        ranges: mongoDocs[0].ranges,
         schema_v: 1
       })
-      this.md5 = crypto.createHash('md5').update(data).digest('hex')
-      this.request.put = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 200, headers: { etag: this.md5 } })
-      return this.DocArchiveManager.archiveDoc(
-        this.project_id,
-        this.mongoDocs[0],
-        (err) => {
-          should.not.exist(err)
-          return done()
-        }
-      )
+
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+      expect(Crypto.createHash).to.have.been.calledWith('md5')
+      expect(HashUpdate).to.have.been.calledWith(json)
+      expect(HashDigest).to.have.been.calledWith('hex')
     })
 
-    return it('should return the error', function (done) {
-      this.request.put = sinon.stub().callsArgWith(1, this.stubbedError, {
-        statusCode: 400,
-        headers: { etag: '' }
-      })
-      return this.DocArchiveManager.archiveDoc(
-        this.project_id,
-        this.mongoDocs[0],
-        (err) => {
-          should.exist(err)
-          return done()
-        }
+    it('should pass the md5 hash to the object persistor for verification', async function () {
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+      expect(PersistorManager.sendStream).to.have.been.calledWith(
+        sinon.match.any,
+        sinon.match.any,
+        sinon.match.any,
+        { sourceMd5: md5Sum }
       )
     })
-  })
 
-  describe('unarchiveDoc', function () {
-    it('should use correct options', function (done) {
-      this.request.get = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 200 }, this.mongoDocs[0].lines)
-      this.request.del = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 204 }, {})
-      return this.DocArchiveManager.unarchiveDoc(
-        this.project_id,
-        this.mongoDocs[0]._id,
-        (err) => {
-          const opts = this.request.get.args[0][0]
-          assert.deepEqual(opts.aws, {
-            key: this.settings.docstore.s3.key,
-            secret: this.settings.docstore.s3.secret,
-            bucket: this.settings.docstore.s3.bucket
-          })
-          opts.json.should.equal(true)
-          opts.timeout.should.equal(30 * 1000)
-          opts.uri.should.equal(
-            `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
-          )
-          return done()
-        }
+    it('should pass the correct bucket and key to the persistor', async function () {
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+      expect(PersistorManager.sendStream).to.have.been.calledWith(
+        Settings.docstore.bucket,
+        `${projectId}/${mongoDocs[0]._id}`
       )
     })
 
-    it('should return the error', function (done) {
-      this.request.get = sinon.stub().callsArgWith(1, this.stubbedError, {}, {})
-      return this.DocArchiveManager.unarchiveDoc(
-        this.project_id,
-        this.mongoDocs[0],
-        (err) => {
-          should.exist(err)
-          return done()
-        }
+    it('should create a stream from the encoded json and send it', async function () {
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+      expect(Streamifier.createReadStream).to.have.been.calledWith(docJson)
+      expect(PersistorManager.sendStream).to.have.been.calledWith(
+        sinon.match.any,
+        sinon.match.any,
+        readStream
       )
     })
 
-    return it('should error if the doc lines are a string not an array', function (done) {
-      this.request.get = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 200 }, 'this is a string')
-      this.request.del = sinon.stub()
-      return this.DocArchiveManager.unarchiveDoc(
-        this.project_id,
-        this.mongoDocs[0],
-        (err) => {
-          should.exist(err)
-          this.request.del.called.should.equal(false)
-          return done()
-        }
+    it('should mark the doc as archived', async function () {
+      await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+      expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+        mongoDocs[0]._id,
+        mongoDocs[0].rev
       )
     })
-  })
 
-  describe('archiveAllDocs', function () {
-    it('should archive all project docs which are not in s3', function (done) {
-      this.MongoManager.getProjectsDocs = sinon
-        .stub()
-        .callsArgWith(3, null, this.mongoDocs)
-      this.DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)
-
-      return this.DocArchiveManager.archiveAllDocs(this.project_id, (err) => {
-        this.DocArchiveManager.archiveDoc
-          .calledWith(this.project_id, this.mongoDocs[0])
-          .should.equal(true)
-        this.DocArchiveManager.archiveDoc
-          .calledWith(this.project_id, this.mongoDocs[1])
-          .should.equal(true)
-        this.DocArchiveManager.archiveDoc
-          .calledWith(this.project_id, this.mongoDocs[4])
-          .should.equal(true)
-
-        this.DocArchiveManager.archiveDoc
-          .calledWith(this.project_id, this.mongoDocs[2])
-          .should.equal(false)
-        this.DocArchiveManager.archiveDoc
-          .calledWith(this.project_id, this.mongoDocs[3])
-          .should.equal(false)
-
-        should.not.exist(err)
-        return done()
+    describe('with null bytes in the result', function () {
+      const _stringify = JSON.stringify
+
+      beforeEach(function () {
+        JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}')
       })
-    })
 
-    it('should return error if have no docs', function (done) {
-      this.MongoManager.getProjectsDocs = sinon
-        .stub()
-        .callsArgWith(3, null, null)
+      afterEach(function () {
+        JSON.stringify = _stringify
+      })
 
-      return this.DocArchiveManager.archiveAllDocs(this.project_id, (err) => {
-        should.exist(err)
-        return done()
+      it('should return an error', async function () {
+        await expect(
+          DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+        ).to.eventually.be.rejectedWith('null bytes detected')
       })
     })
+  })
 
-    it('should return the error', function (done) {
-      this.MongoManager.getProjectsDocs = sinon
-        .stub()
-        .callsArgWith(3, this.error, null)
+  describe('unarchiveDoc', function () {
+    let docId
 
-      return this.DocArchiveManager.archiveAllDocs(this.project_id, (err) => {
-        err.should.equal(this.error)
-        return done()
-      })
+    beforeEach(function () {
+      docId = mongoDocs[0]._id
     })
 
-    return describe('when most have been already put in s3', function () {
-      beforeEach(function () {
-        let numberOfDocs = 10 * 1000
-        this.mongoDocs = []
-        while (--numberOfDocs !== 0) {
-          this.mongoDocs.push({ inS3: true, _id: ObjectId() })
-        }
-
-        this.MongoManager.getProjectsDocs = sinon
-          .stub()
-          .callsArgWith(3, null, this.mongoDocs)
-        return (this.DocArchiveManager.archiveDoc = sinon
-          .stub()
-          .callsArgWith(2, null))
-      })
+    it('should resolve when passed a valid document', async function () {
+      await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId)).to
+        .eventually.be.fulfilled
+    })
+
+    it('should throw an error if the md5 does not match', async function () {
+      PersistorManager.getObjectMd5Hash.resolves('badf00d')
+      await expect(
+        DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+      ).to.eventually.be.rejected.and.be.instanceof(Errors.Md5MismatchError)
+    })
+
+    it('should update the doc lines in mongo', async function () {
+      await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+      expect(
+        MongoManager.upsertIntoDocCollection
+      ).to.have.been.calledWith(projectId, docId, { lines: mongoDocs[0].lines })
+    })
 
-      return it('should not throw and error', function (done) {
-        return this.DocArchiveManager.archiveAllDocs(this.project_id, (err) => {
-          should.not.exist(err)
-          return done()
+    it('should delete the doc in s3', async function () {
+      await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+      expect(PersistorManager.deleteObject).to.have.been.calledWith(
+        Settings.docstore.bucket,
+        `${projectId}/${docId}`
+      )
+    })
+
+    describe('doc contents', function () {
+      let mongoDoc, s3Doc
+
+      describe('when the doc has the old schema', function () {
+        beforeEach(function () {
+          mongoDoc = {
+            lines: ['doc', 'lines']
+          }
+          s3Doc = ['doc', 'lines']
+          docJson = JSON.stringify(s3Doc)
+          stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+        })
+
+        it('should return the docs lines', async function () {
+          await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+          expect(MongoManager.upsertIntoDocCollection).to.have.been.calledWith(
+            projectId,
+            docId,
+            mongoDoc
+          )
         })
       })
-    })
-  })
 
-  describe('unArchiveAllDocs', function () {
-    it('should unarchive all inS3 docs', function (done) {
-      this.MongoManager.getArchivedProjectDocs = sinon
-        .stub()
-        .callsArgWith(1, null, this.archivedDocs)
-      this.DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null)
-      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, (err) => {
-        for (const doc of Array.from(this.archivedDocs)) {
-          this.DocArchiveManager.unarchiveDoc
-            .calledWith(this.project_id, doc._id)
-            .should.equal(true)
-        }
-        should.not.exist(err)
-        return done()
+      describe('with the new schema and ranges', function () {
+        beforeEach(function () {
+          s3Doc = {
+            lines: ['doc', 'lines'],
+            ranges: { json: 'ranges' },
+            schema_v: 1
+          }
+          mongoDoc = {
+            lines: ['doc', 'lines'],
+            ranges: { mongo: 'ranges' }
+          }
+          docJson = JSON.stringify(s3Doc)
+          stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+        })
+
+        it('should return the doc lines and ranges', async function () {
+          await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+          expect(MongoManager.upsertIntoDocCollection).to.have.been.calledWith(
+            projectId,
+            docId,
+            mongoDoc
+          )
+        })
       })
-    })
 
-    it('should return error if have no docs', function (done) {
-      this.MongoManager.getArchivedProjectDocs = sinon
-        .stub()
-        .callsArgWith(1, null, null)
-      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, (err) => {
-        should.exist(err)
-        return done()
+      describe('with the new schema and no ranges', function () {
+        beforeEach(function () {
+          s3Doc = {
+            lines: ['doc', 'lines'],
+            schema_v: 1
+          }
+          mongoDoc = {
+            lines: ['doc', 'lines']
+          }
+          docJson = JSON.stringify(s3Doc)
+          stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+        })
+
+        it('should return only the doc lines', async function () {
+          await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+          expect(MongoManager.upsertIntoDocCollection).to.have.been.calledWith(
+            projectId,
+            docId,
+            mongoDoc
+          )
+        })
       })
-    })
 
-    return it('should return the error', function (done) {
-      this.MongoManager.getArchivedProjectDocs = sinon
-        .stub()
-        .callsArgWith(1, this.error, null)
-      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, (err) => {
-        err.should.equal(this.error)
-        return done()
+      describe('with an unrecognised schema', function () {
+        beforeEach(function () {
+          s3Doc = {
+            lines: ['doc', 'lines'],
+            schema_v: 2
+          }
+          docJson = JSON.stringify(s3Doc)
+          stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+        })
+
+        it('should throw an error', async function () {
+          await expect(
+            DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+          ).to.eventually.be.rejectedWith(
+            "I don't understand the doc format in s3"
+          )
+        })
       })
     })
   })
 
-  describe('destroyAllDocs', function () {
+  describe('destroyDoc', function () {
+    let docId
+
     beforeEach(function () {
-      this.request.del = sinon
-        .stub()
-        .callsArgWith(1, null, { statusCode: 204 }, {})
-      this.MongoManager.getProjectsDocs = sinon
-        .stub()
-        .callsArgWith(3, null, this.mixedDocs)
-      this.MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
-      this.MongoManager.destroyDoc = sinon.stub().yields()
-      return Array.from(this.mixedDocs).map((doc) =>
-        this.MongoManager.findDoc
-          .withArgs(this.project_id, doc._id)
-          .callsArgWith(3, null, doc)
-      )
+      docId = mongoDocs[0]._id
     })
 
-    it('should destroy all the docs', function (done) {
-      this.DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
-      return this.DocArchiveManager.destroyAllDocs(this.project_id, (err) => {
-        for (const doc of Array.from(this.mixedDocs)) {
-          this.DocArchiveManager.destroyDoc
-            .calledWith(this.project_id, doc._id)
-            .should.equal(true)
-        }
-        should.not.exist(err)
-        return done()
-      })
+    it('should resolve when passed a valid document', async function () {
+      await expect(DocArchiveManager.promises.destroyDoc(projectId, docId)).to
+        .eventually.be.fulfilled
     })
 
-    it('should only the s3 docs from s3', function (done) {
-      const docOpts = (doc) => {
-        return JSON.parse(
-          JSON.stringify({
-            aws: {
-              key: this.settings.docstore.s3.key,
-              secret: this.settings.docstore.s3.secret,
-              bucket: this.settings.docstore.s3.bucket
-            },
-            json: true,
-            timeout: 30 * 1000,
-            uri: `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${doc._id}`
-          })
-        )
-      }
+    it('should throw a not found error when there is no document', async function () {
+      await expect(
+        DocArchiveManager.promises.destroyDoc(projectId, 'wombat')
+      ).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
+    })
 
-      return this.DocArchiveManager.destroyAllDocs(this.project_id, (err) => {
-        let doc
-        expect(err).not.to.exist
+    describe('when the doc is in s3', function () {
+      beforeEach(function () {
+        mongoDocs[0].inS3 = true
+      })
 
-        for (doc of Array.from(this.archivedDocs)) {
-          sinon.assert.calledWith(this.request.del, docOpts(doc))
-        }
-        for (doc of Array.from(this.unarchivedDocs)) {
-          expect(this.request.del.calledWith(docOpts(doc))).to.equal(false)
-        } // no notCalledWith
+      it('should delete the document from s3, if it is in s3', async function () {
+        await DocArchiveManager.promises.destroyDoc(projectId, docId)
+        expect(PersistorManager.deleteObject).to.have.been.calledWith(
+          Settings.docstore.bucket,
+          `${projectId}/${docId}`
+        )
+      })
 
-        return done()
+      it('should delete the doc in mongo', async function () {
+        await DocArchiveManager.promises.destroyDoc(projectId, docId)
       })
     })
 
-    return it('should remove the docs from mongo', function (done) {
-      this.DocArchiveManager.destroyAllDocs(this.project_id, (err) => {
-        return expect(err).not.to.exist
+    describe('when the doc is not in s3', function () {
+      beforeEach(function () {
+        mongoDocs[0].inS3 = false
       })
 
-      for (const doc of Array.from(this.mixedDocs)) {
-        sinon.assert.calledWith(this.MongoManager.destroyDoc, doc._id)
-      }
+      it('should not delete the document from s3, if it is not in s3', async function () {
+        await DocArchiveManager.promises.destroyDoc(projectId, docId)
+        expect(PersistorManager.deleteObject).not.to.have.been.called
+      })
 
-      return done()
+      it('should delete the doc in mongo', async function () {
+        await DocArchiveManager.promises.destroyDoc(projectId, docId)
+      })
     })
   })
 
-  describe('_s3DocToMongoDoc', function () {
-    describe('with the old schema', function () {
-      return it('should return the docs lines', function (done) {
-        return this.DocArchiveManager._s3DocToMongoDoc(
-          ['doc', 'lines'],
-          (error, doc) => {
-            expect(doc).to.deep.equal({
-              lines: ['doc', 'lines']
-            })
-            return done()
-          }
-        )
-      })
+  describe('archiveAllDocs', function () {
+    it('should resolve with valid arguments', async function () {
+      await expect(DocArchiveManager.promises.archiveAllDocs(projectId)).to
+        .eventually.be.fulfilled
     })
 
-    describe('with the new schema', function () {
-      it('should return the doc lines and ranges', function (done) {
-        this.RangeManager.jsonRangesToMongo = sinon
-          .stub()
-          .returns({ mongo: 'ranges' })
-        return this.DocArchiveManager._s3DocToMongoDoc(
-          {
-            lines: ['doc', 'lines'],
-            ranges: { json: 'ranges' },
-            schema_v: 1
-          },
-          (error, doc) => {
-            expect(doc).to.deep.equal({
-              lines: ['doc', 'lines'],
-              ranges: { mongo: 'ranges' }
-            })
-            return done()
-          }
-        )
-      })
+    it('should archive all project docs which are not in s3', async function () {
+      await DocArchiveManager.promises.archiveAllDocs(projectId)
+      // not inS3
+      expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+        mongoDocs[0]._id
+      )
+      expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+        mongoDocs[1]._id
+      )
+      expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+        mongoDocs[4]._id
+      )
 
-      return it('should return just the doc lines when there are no ranges', function (done) {
-        return this.DocArchiveManager._s3DocToMongoDoc(
-          {
-            lines: ['doc', 'lines'],
-            schema_v: 1
-          },
-          (error, doc) => {
-            expect(doc).to.deep.equal({
-              lines: ['doc', 'lines']
-            })
-            return done()
-          }
-        )
-      })
+      // inS3
+      expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
+        mongoDocs[2]._id
+      )
+      expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
+        mongoDocs[3]._id
+      )
     })
 
-    return describe('with an unrecognised schema', function () {
-      return it('should return an error', function (done) {
-        return this.DocArchiveManager._s3DocToMongoDoc(
-          {
-            schema_v: 2
-          },
-          (error, doc) => {
-            expect(error).to.exist
-            return done()
-          }
-        )
-      })
+    it('should return error if the project has no docs', async function () {
+      MongoManager.getProjectsDocs.yields(null, null)
+
+      await expect(
+        DocArchiveManager.promises.archiveAllDocs(projectId)
+      ).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
     })
   })
 
-  return describe('_mongoDocToS3Doc', function () {
-    describe('with a valid doc', function () {
-      return it('should return the json version', function (done) {
-        let doc
-        return this.DocArchiveManager._mongoDocToS3Doc(
-          (doc = {
-            lines: ['doc', 'lines'],
-            ranges: { mock: 'ranges' }
-          }),
-          (err, s3_doc) => {
-            expect(s3_doc).to.equal(
-              JSON.stringify({
-                lines: ['doc', 'lines'],
-                ranges: { mock: 'ranges' },
-                schema_v: 1
-              })
-            )
-            return done()
-          }
+  describe('unArchiveAllDocs', function () {
+    it('should resolve with valid arguments', async function () {
+      await expect(DocArchiveManager.promises.unArchiveAllDocs(projectId)).to
+        .eventually.be.fulfilled
+    })
+
+    it('should unarchive all inS3 docs', async function () {
+      await DocArchiveManager.promises.unArchiveAllDocs(projectId)
+
+      for (const doc of archivedDocs) {
+        expect(PersistorManager.getObjectStream).to.have.been.calledWith(
+          Settings.docstore.bucket,
+          `${projectId}/${doc._id}`
         )
-      })
+      }
     })
 
-    describe('with null bytes in the result', function () {
-      beforeEach(function () {
-        this._stringify = JSON.stringify
-        return (JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}'))
-      })
+    it('should return error if the project has no docs', async function () {
+      MongoManager.getArchivedProjectDocs.yields(null, null)
 
-      afterEach(function () {
-        return (JSON.stringify = this._stringify)
-      })
+      await expect(
+        DocArchiveManager.promises.unArchiveAllDocs(projectId)
+      ).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
+    })
+  })
 
-      return it('should return an error', function (done) {
-        return this.DocArchiveManager._mongoDocToS3Doc(
-          {
-            lines: ['doc', 'lines'],
-            ranges: { mock: 'ranges' }
-          },
-          (err, s3_doc) => {
-            expect(err).to.exist
-            return done()
-          }
+  describe('destroyAllDocs', function () {
+    it('should resolve with valid arguments', async function () {
+      await expect(DocArchiveManager.promises.destroyAllDocs(projectId)).to
+        .eventually.be.fulfilled
+    })
+
+    it('should delete all docs that are in s3 from s3', async function () {
+      await DocArchiveManager.promises.destroyAllDocs(projectId)
+
+      // not inS3
+      for (const index of [0, 1, 4]) {
+        expect(PersistorManager.deleteObject).not.to.have.been.calledWith(
+          Settings.docstore.bucket,
+          `${projectId}/${mongoDocs[index]._id}`
         )
-      })
+      }
+
+      // inS3
+      for (const index of [2, 3]) {
+        expect(PersistorManager.deleteObject).to.have.been.calledWith(
+          Settings.docstore.bucket,
+          `${projectId}/${mongoDocs[index]._id}`
+        )
+      }
     })
 
-    return describe('without doc lines', function () {
-      return it('should return an error', function (done) {
-        return this.DocArchiveManager._mongoDocToS3Doc({}, (err, s3_doc) => {
-          expect(err).to.exist
-          return done()
-        })
-      })
+    it('should destroy all docs in mongo', async function () {
+      await DocArchiveManager.promises.destroyAllDocs(projectId)
+
+      for (const mongoDoc of mongoDocs) {
+        expect(MongoManager.destroyDoc).to.have.been.calledWith(mongoDoc._id)
+      }
     })
   })
 })

services/docstore/test/unit/js/DocManagerTests.js

@@ -38,7 +38,8 @@ describe('DocManager', function () {
           log: sinon.stub(),
           warn() {},
           err() {}
-        })
+        }),
+        './Errors': Errors
       }
     })
     this.doc_id = ObjectId().toString()