Merge pull request #52 from overleaf/spd-decaf

Decaffienate docstore

services/docstore/.dockerignore

@@ -5,5 +5,3 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
-app.js
-**/js/*

services/docstore/.eslintrc

@@ -0,0 +1,65 @@
+// this file was auto-generated, do not edit it directly.
+// instead run bin/update_build_scripts from
+// https://github.com/sharelatex/sharelatex-dev-environment
+// Version: 1.3.5
+{
+  "extends": [
+    "standard",
+    "prettier",
+    "prettier/standard"
+  ],
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
+  "plugins": [
+    "mocha",
+    "chai-expect",
+    "chai-friendly"
+  ],
+  "env": {
+    "node": true,
+    "mocha": true
+  },
+  "rules": {
+    // Swap the no-unused-expressions rule with a more chai-friendly one
+    "no-unused-expressions": 0,
+    "chai-friendly/no-unused-expressions": "error"
+  },
+  "overrides": [
+    {
+      // Test specific rules
+      "files": ["test/**/*.js"],
+      "globals": {
+        "expect": true
+      },
+      "rules": {
+        // mocha-specific rules
+        "mocha/handle-done-callback": "error",
+        "mocha/no-exclusive-tests": "error",
+        "mocha/no-global-tests": "error",
+        "mocha/no-identical-title": "error",
+        "mocha/no-nested-tests": "error",
+        "mocha/no-pending-tests": "error",
+        "mocha/no-skipped-tests": "error",
+        "mocha/no-mocha-arrows": "error",
+
+        // chai-specific rules
+        "chai-expect/missing-assertion": "error",
+        "chai-expect/terminating-properties": "error",
+
+        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
+        // we don't enforce this at the top-level - just in tests to manage `this` scope
+        // based on mocha's context mechanism
+        "mocha/prefer-arrow-callback": "error"
+      }
+    },
+    {
+      // Backend specific rules
+      "files": ["app/**/*.js", "app.js", "index.js"],
+      "rules": {
+        // don't allow console.log in backend code
+        "no-console": "error"
+      }
+    }
+  ]
+}

services/docstore/.gitignore

@@ -1,7 +1,2 @@
 node_modules
-app/js/
-test/unit/js
-test/acceptance/js
-app.js
-**/*.map
 forever

services/docstore/.prettierrc

@@ -0,0 +1,8 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3.5
+{
+  "semi": false,
+  "singleQuote": true
+}

services/docstore/Dockerfile

@@ -17,7 +17,6 @@ RUN npm install --quiet
 COPY . /app
 
 
-RUN npm run compile:all
 
 FROM base
 

services/docstore/Jenkinsfile

@@ -37,6 +37,13 @@ pipeline {
       }
     }
 
+    stage('Linting') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
+      }
+    }
+
     stage('Unit Tests') {
       steps {
         sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

services/docstore/Makefile

@@ -19,12 +19,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 clean:
 	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	rm -f app.js
-	rm -rf app/js
-	rm -rf test/unit/js
-	rm -rf test/acceptance/js
 
-test: test_unit test_acceptance
+format:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+
+format_fix:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+
+lint:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+
+test: format lint test_unit test_acceptance
 
 test_unit:
 	@[ ! -d test/unit ] && echo "docstore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit

services/docstore/app.coffee

@@ -1,67 +0,0 @@
-Metrics    = require "metrics-sharelatex"
-Metrics.initialize("docstore")
-Settings   = require "settings-sharelatex"
-logger     = require "logger-sharelatex"
-express    = require "express"
-bodyParser = require "body-parser"
-Errors     = require "./app/js/Errors"
-HttpController = require "./app/js/HttpController"
-Path       = require "path"
-
-
-logger.initialize("docstore")
-Metrics.event_loop?.monitor(logger)
-
-app = express()
-
-app.use Metrics.http.monitor(logger)
-
-Metrics.injectMetricsRoute(app)
-
-app.param 'project_id', (req, res, next, project_id) ->
-	if project_id?.match /^[0-9a-f]{24}$/
-		next()
-	else
-		next new Error("invalid project id")
-
-app.param 'doc_id', (req, res, next, doc_id) ->
-	if doc_id?.match /^[0-9a-f]{24}$/
-		next()
-	else
-		next new Error("invalid doc id")
-
-Metrics.injectMetricsRoute(app)
-
-app.get  '/project/:project_id/doc', HttpController.getAllDocs
-app.get  '/project/:project_id/ranges', HttpController.getAllRanges
-app.get  '/project/:project_id/doc/:doc_id', HttpController.getDoc
-app.get  '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc
-# Add 64kb overhead for the JSON encoding, and double the size to allow for ranges in the json payload
-app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: (Settings.max_doc_length + 64 * 1024) * 2), HttpController.updateDoc
-app.del  '/project/:project_id/doc/:doc_id', HttpController.deleteDoc
-
-app.post  '/project/:project_id/archive', HttpController.archiveAllDocs
-app.post  '/project/:project_id/unarchive', HttpController.unArchiveAllDocs
-app.post  '/project/:project_id/destroy', HttpController.destroyAllDocs
-
-app.get "/health_check",  HttpController.healthCheck
-
-app.get '/status', (req, res)->
-	res.send('docstore is alive')
-
-app.use (error, req, res, next) ->
-	logger.error err: error, req:req, "request errored"
-	if error instanceof Errors.NotFoundError
-		res.send 404
-	else
-		res.send(500, "Oops, something went wrong")
-
-port = Settings.internal.docstore.port
-host = Settings.internal.docstore.host
-
-if !module.parent # Called directly
-	app.listen port, host, (error) ->
-		throw error if error?
-		logger.info "Docstore starting up, listening on #{host}:#{port}"
-
-module.exports = app

services/docstore/app.js

@@ -0,0 +1,87 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const Metrics = require('metrics-sharelatex')
+Metrics.initialize('docstore')
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const express = require('express')
+const bodyParser = require('body-parser')
+const Errors = require('./app/js/Errors')
+const HttpController = require('./app/js/HttpController')
+
+logger.initialize('docstore')
+if (Metrics.event_loop != null) {
+  Metrics.event_loop.monitor(logger)
+}
+
+const app = express()
+
+app.use(Metrics.http.monitor(logger))
+
+Metrics.injectMetricsRoute(app)
+
+app.param('project_id', function(req, res, next, projectId) {
+  if (projectId != null ? projectId.match(/^[0-9a-f]{24}$/) : undefined) {
+    return next()
+  } else {
+    return next(new Error('invalid project id'))
+  }
+})
+
+app.param('doc_id', function(req, res, next, docId) {
+  if (docId != null ? docId.match(/^[0-9a-f]{24}$/) : undefined) {
+    return next()
+  } else {
+    return next(new Error('invalid doc id'))
+  }
+})
+
+Metrics.injectMetricsRoute(app)
+
+app.get('/project/:project_id/doc', HttpController.getAllDocs)
+app.get('/project/:project_id/ranges', HttpController.getAllRanges)
+app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
+app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc)
+// Add 64kb overhead for the JSON encoding, and double the size to allow for ranges in the json payload
+app.post(
+  '/project/:project_id/doc/:doc_id',
+  bodyParser.json({ limit: (Settings.max_doc_length + 64 * 1024) * 2 }),
+  HttpController.updateDoc
+)
+app.del('/project/:project_id/doc/:doc_id', HttpController.deleteDoc)
+
+app.post('/project/:project_id/archive', HttpController.archiveAllDocs)
+app.post('/project/:project_id/unarchive', HttpController.unArchiveAllDocs)
+app.post('/project/:project_id/destroy', HttpController.destroyAllDocs)
+
+app.get('/health_check', HttpController.healthCheck)
+
+app.get('/status', (req, res) => res.send('docstore is alive'))
+
+app.use(function(error, req, res, next) {
+  logger.error({ err: error, req }, 'request errored')
+  if (error instanceof Errors.NotFoundError) {
+    return res.send(404)
+  } else {
+    return res.send(500, 'Oops, something went wrong')
+  }
+})
+
+const { port } = Settings.internal.docstore
+const { host } = Settings.internal.docstore
+
+if (!module.parent) {
+  // Called directly
+  app.listen(port, host, function(error) {
+    if (error != null) {
+      throw error
+    }
+    return logger.info(`Docstore starting up, listening on ${host}:${port}`)
+  })
+}
+
+module.exports = app

services/docstore/app/coffee/DocArchiveManager.coffee

@@ -1,156 +0,0 @@
-MongoManager = require "./MongoManager"
-Errors = require "./Errors"
-logger = require "logger-sharelatex"
-_ = require "underscore"
-async = require "async"
-settings = require("settings-sharelatex")
-request = require("request")
-crypto = require("crypto")
-RangeManager = require("./RangeManager")
-thirtySeconds = 30 * 1000
-
-module.exports = DocArchive =
-
-	archiveAllDocs: (project_id, callback = (err, docs) ->) ->
-		MongoManager.getProjectsDocs project_id, {include_deleted: true}, {lines: true, ranges: true, rev: true, inS3: true}, (err, docs) ->
-			if err?
-				return callback(err)
-			else if !docs?
-				return callback new Errors.NotFoundError("No docs for project #{project_id}")
-			docs = _.filter docs, (doc)-> doc.inS3 != true
-			jobs = _.map docs, (doc) ->
-				(cb)->
-					DocArchive.archiveDoc project_id, doc, cb
-			async.parallelLimit jobs, 5, callback
-
-
-	archiveDoc: (project_id, doc, callback)->
-		logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3"
-		try
-			options = DocArchive.buildS3Options(project_id+"/"+doc._id)
-		catch e
-			return callback e
-		DocArchive._mongoDocToS3Doc doc, (error, json_doc) ->
-			return callback(error) if error?
-			options.body = json_doc
-			options.headers =
-				'Content-Type': "application/json"
-			request.put options, (err, res) ->
-				if err? || res.statusCode != 200
-					logger.err err:err, res:res, project_id:project_id, doc_id: doc._id, statusCode: res?.statusCode, "something went wrong archiving doc in aws"
-					return callback new Error("Error in S3 request")
-				md5lines = crypto.createHash("md5").update(json_doc, "utf8").digest("hex")
-				md5response = res.headers.etag.toString().replace(/\"/g, '')
-				if md5lines != md5response
-					logger.err responseMD5:md5response, linesMD5:md5lines,  project_id:project_id, doc_id: doc?._id, "err in response md5 from s3"
-					return callback new Error("Error in S3 md5 response")
-				MongoManager.markDocAsArchived doc._id, doc.rev, (err) ->
-					return callback(err) if err?
-					callback()
-
-	unArchiveAllDocs: (project_id, callback = (err) ->) ->
-		MongoManager.getArchivedProjectDocs project_id, (err, docs) ->
-			if err?
-				logger.err err:err, project_id:project_id, "error unarchiving all docs"
-				return callback(err)
-			else if !docs?
-				return callback new Errors.NotFoundError("No docs for project #{project_id}")
-			jobs = _.map docs, (doc) ->
-				(cb)->
-					if !doc.inS3?
-						return cb()
-					else
-						DocArchive.unarchiveDoc project_id, doc._id, cb
-			async.parallelLimit jobs, 5, callback
-
-	unarchiveDoc: (project_id, doc_id, callback)->
-		logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3"
-		try
-			options = DocArchive.buildS3Options(project_id+"/"+doc_id)
-		catch e
-			return callback e
-		options.json = true
-		request.get options, (err, res, doc)->
-			if err? || res.statusCode != 200
-				logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong unarchiving doc from aws"
-				return callback new Errors.NotFoundError("Error in S3 request")
-			DocArchive._s3DocToMongoDoc doc, (error, mongo_doc) ->
-				return callback(error) if error?
-				MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), mongo_doc, (err) ->
-					return callback(err) if err?
-					logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3"
-					DocArchive._deleteDocFromS3 project_id, doc_id, callback
-
-	destroyAllDocs: (project_id, callback = (err) ->) ->
-		MongoManager.getProjectsDocs project_id, {include_deleted: true}, {_id: 1}, (err, docs) ->
-			if err?
-				logger.err err:err, project_id:project_id, "error getting project's docs"
-				return callback(err)
-			else if !docs?
-				return callback()
-			jobs = _.map docs, (doc) ->
-				(cb)->
-					DocArchive.destroyDoc(project_id, doc._id, cb)
-			async.parallelLimit jobs, 5, callback
-
-	destroyDoc: (project_id, doc_id, callback)->
-		logger.log project_id: project_id, doc_id: doc_id, "removing doc from mongo and s3"
-		MongoManager.findDoc project_id, doc_id, {inS3: 1}, (error, doc) ->
-			return callback error if error?
-			return callback new Errors.NotFoundError("Doc not found in Mongo") unless doc?
-			if doc.inS3 == true
-				DocArchive._deleteDocFromS3 project_id, doc_id, (err) ->
-					return err if err?
-					MongoManager.destroyDoc doc_id, callback
-			else
-				MongoManager.destroyDoc doc_id, callback
-
-	_deleteDocFromS3: (project_id, doc_id, callback) ->
-		try
-			options = DocArchive.buildS3Options(project_id+"/"+doc_id)
-		catch e
-			return callback e
-		options.json = true
-		request.del options, (err, res, body)->
-			if err? || res.statusCode != 204
-				logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws"
-				return callback new Error("Error in S3 request")
-			callback()
-
-	_s3DocToMongoDoc: (doc, callback = (error, mongo_doc) ->) ->
-		mongo_doc = {}
-		if doc.schema_v == 1 and doc.lines?
-			mongo_doc.lines = doc.lines
-			if doc.ranges?
-				mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges)
-		else if doc instanceof Array
-			mongo_doc.lines = doc
-		else
-			return callback(new Error("I don't understand the doc format in s3"))
-		return callback null, mongo_doc
-
-	_mongoDocToS3Doc: (doc, callback = (error, s3_doc) ->) ->
-		if !doc.lines?
-			return callback(new Error("doc has no lines"))
-		json = JSON.stringify({
-			lines: doc.lines
-			ranges: doc.ranges
-			schema_v: 1
-		})
-		if json.indexOf("\u0000") != -1
-			error = new Error("null bytes detected")
-			logger.err {err: error, doc, json}, error.message
-			return callback(error)
-		return callback null, json
-
-	buildS3Options: (key)->
-		if !settings.docstore.s3?
-			throw new Error("S3 settings are not configured")
-		return {
-				aws:
-					key: settings.docstore.s3.key
-					secret: settings.docstore.s3.secret
-					bucket: settings.docstore.s3.bucket
-				timeout: thirtySeconds
-				uri:"https://#{settings.docstore.s3.bucket}.s3.amazonaws.com/#{key}"
-		}

services/docstore/app/coffee/DocManager.coffee

@@ -1,130 +0,0 @@
-MongoManager = require "./MongoManager"
-Errors = require "./Errors"
-logger = require "logger-sharelatex"
-_ = require "underscore"
-DocArchive = require "./DocArchiveManager"
-RangeManager = require "./RangeManager"
-
-module.exports = DocManager =
-	
-
-	# TODO: For historical reasons, the doc version is currently stored in the docOps
-	# collection (which is all that this collection contains). In future, we should
-	# migrate this version property to be part of the docs collection, to guarantee
-	# consitency between lines and version when writing/reading, and for a simpler schema.
-	_getDoc: (project_id, doc_id, filter = {}, callback = (error, doc) ->) ->
-		if filter.inS3 != true
-			return callback("must include inS3 when getting doc")
-
-		MongoManager.findDoc project_id, doc_id, filter, (err, doc)->
-			if err?
-				return callback(err)
-			else if !doc?
-				return callback new Errors.NotFoundError("No such doc: #{doc_id} in project #{project_id}")
-			else if doc?.inS3
-				DocArchive.unarchiveDoc project_id, doc_id, (err)->
-					if err?
-						logger.err err:err, project_id:project_id, doc_id:doc_id, "error unarchiving doc"
-						return callback(err)
-					DocManager._getDoc project_id, doc_id, filter, callback
-			else
-				if filter.version
-					MongoManager.getDocVersion doc_id, (error, version) ->
-						return callback(error) if error?
-						doc.version = version
-						callback err, doc
-				else
-					callback err, doc
-
-	checkDocExists: (project_id, doc_id, callback = (err, exists)->)->
-		DocManager._getDoc project_id, doc_id, {_id:1, inS3:true}, (err, doc)->
-			if err?
-				return callback(err)
-			callback(err, doc?)
-
-	getFullDoc: (project_id, doc_id, callback = (err, doc)->)->
-		DocManager._getDoc project_id, doc_id, {lines: true, rev: true, deleted: true, version: true, ranges: true, inS3:true}, (err, doc)->
-			if err? 
-				return callback(err)
-			callback(err, doc)
-
-
-	getDocLines: (project_id, doc_id, callback = (err, doc)->)->
-		DocManager._getDoc project_id, doc_id, {lines:true, inS3:true}, (err, doc)->
-			if err?
-				return callback(err)
-			callback(err, doc)
-
-	getAllNonDeletedDocs: (project_id, filter, callback = (error, docs) ->) ->
-		DocArchive.unArchiveAllDocs project_id, (error) ->
-			if error?
-				return callback(error)
-			MongoManager.getProjectsDocs project_id, {include_deleted: false}, filter, (error, docs) ->
-				if err?
-					return callback(error)
-				else if !docs?
-					return callback new Errors.NotFoundError("No docs for project #{project_id}")
-				else
-					return callback(null, docs)
-
-	updateDoc: (project_id, doc_id, lines, version, ranges, callback = (error, modified, rev) ->) ->
-		if !lines? or !version? or !ranges?
-			return callback(new Error("no lines, version or ranges provided"))
-	
-		DocManager._getDoc project_id, doc_id, {version: true, rev: true, lines: true, version: true, ranges: true, inS3:true}, (err, doc)->
-			if err? and !(err instanceof Errors.NotFoundError)
-				logger.err project_id: project_id, doc_id: doc_id, err:err, "error getting document for update"
-				return callback(err)
-			
-			ranges = RangeManager.jsonRangesToMongo(ranges)
-
-			if !doc?
-				# If the document doesn't exist, we'll make sure to create/update all parts of it.
-				updateLines = true
-				updateVersion = true
-				updateRanges = true
-			else
-				updateLines = not _.isEqual(doc.lines, lines)
-				updateVersion = (doc.version != version)
-				updateRanges = RangeManager.shouldUpdateRanges(doc.ranges, ranges)
-			
-			modified = false
-			rev = doc?.rev || 0
-
-			updateLinesAndRangesIfNeeded = (cb) ->
-				if updateLines or updateRanges
-					update = {}
-					if updateLines
-						update.lines = lines
-					if updateRanges
-						update.ranges = ranges
-					logger.log { project_id, doc_id }, "updating doc lines and ranges"
-					
-					modified = true
-					rev += 1 # rev will be incremented in mongo by MongoManager.upsertIntoDocCollection
-					MongoManager.upsertIntoDocCollection project_id, doc_id, update, cb
-				else
-					logger.log { project_id, doc_id, }, "doc lines have not changed - not updating"
-					cb()
-			
-			updateVersionIfNeeded = (cb) ->
-				if updateVersion
-					logger.log { project_id, doc_id, oldVersion: doc?.version, newVersion: version }, "updating doc version"
-					modified = true
-					MongoManager.setDocVersion doc_id, version, cb
-				else
-					logger.log { project_id, doc_id, version }, "doc version has not changed - not updating"
-					cb()
-			
-			updateLinesAndRangesIfNeeded (error) ->
-				return callback(error) if error?
-				updateVersionIfNeeded (error) ->
-					return callback(error) if error?
-					callback null, modified, rev
-
-	deleteDoc: (project_id, doc_id, callback = (error) ->) ->
-		DocManager.checkDocExists project_id, doc_id, (error, exists) ->
-			return callback(error) if error?
-			return callback new Errors.NotFoundError("No such project/doc to delete: #{project_id}/#{doc_id}") if !exists
-			MongoManager.markDocAsDeleted project_id, doc_id, callback
-

services/docstore/app/coffee/Errors.coffee

@@ -1,10 +0,0 @@
-NotFoundError = (message) ->
-	error = new Error(message)
-	error.name = "NotFoundError"
-	error.__proto__ = NotFoundError.prototype
-	return error
-NotFoundError.prototype.__proto__ = Error.prototype
-
-module.exports = Errors =
-	NotFoundError: NotFoundError
-

services/docstore/app/coffee/HealthChecker.coffee

@@ -1,44 +0,0 @@
-{db, ObjectId} = require "./mongojs"
-request = require("request")
-async = require("async")
-_ = require("underscore")
-crypto = require("crypto")	
-settings = require("settings-sharelatex")
-port = settings.internal.docstore.port
-logger = require "logger-sharelatex"
-
-
-module.exports = 
-	check : (callback)->
-		doc_id = ObjectId()
-		project_id = ObjectId(settings.docstore.healthCheck.project_id)
-		url = "http://localhost:#{port}/project/#{project_id}/doc/#{doc_id}"
-		lines = ["smoke test - delete me", "#{crypto.randomBytes(32).toString("hex")}"]
-		getOpts = -> {url:url, timeout:3000}
-		logger.log lines:lines, url:url, doc_id:doc_id, project_id:project_id, "running health check"
-		jobs = [
-			(cb)->
-				opts = getOpts()
-				opts.json = {lines: lines, version: 42, ranges: {}}
-				request.post(opts, cb)
-			(cb)->
-				opts = getOpts()
-				opts.json = true
-				request.get opts, (err, res, body)->
-					if err?
-						logger.err err:err, "docstore returned a error in health check get"
-						cb(err)
-					else if !res?
-						cb("no response from docstore with get check")
-					else if res?.statusCode != 200
-						cb("status code not 200, its #{res.statusCode}")
-					else if _.isEqual(body?.lines, lines) and body?._id == doc_id.toString()
-						cb()
-					else
-						cb("health check lines not equal #{body.lines} != #{lines}")
-			(cb)-> 
-				db.docs.remove {_id: doc_id, project_id: project_id}, cb
-			(cb)-> 
-				db.docOps.remove {doc_id: doc_id}, cb
-		]
-		async.series jobs, callback

services/docstore/app/coffee/HttpController.coffee

@@ -1,144 +0,0 @@
-DocManager = require "./DocManager"
-logger = require "logger-sharelatex"
-DocArchive = require "./DocArchiveManager"
-HealthChecker = require "./HealthChecker"
-Settings   = require "settings-sharelatex"
-
-
-module.exports = HttpController =
-	getDoc: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id     = req.params.doc_id
-		include_deleted = req.query?.include_deleted == "true"
-		logger.log project_id: project_id, doc_id: doc_id, "getting doc"
-		DocManager.getFullDoc project_id, doc_id, (error, doc) ->
-			return next(error) if error?
-			logger.log {doc_id, project_id}, "got doc"
-			if !doc?
-				res.send 404
-			else if doc.deleted && !include_deleted
-				res.send 404
-			else
-				res.json HttpController._buildDocView(doc)
-
-	getRawDoc: (req, res, next = (error)->)->
-		project_id = req.params.project_id
-		doc_id     = req.params.doc_id
-		logger.log project_id: project_id, doc_id: doc_id, "getting raw doc"
-		DocManager.getDocLines project_id, doc_id, (error, doc) ->
-			return next(error) if error?
-			if !doc?
-				res.send 404
-			else
-				res.setHeader('content-type', 'text/plain')
-				res.send HttpController._buildRawDocView(doc)
-
-	getAllDocs: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "getting all docs"
-		DocManager.getAllNonDeletedDocs project_id, {lines: true, rev: true}, (error, docs = []) ->
-			return next(error) if error?
-			res.json HttpController._buildDocsArrayView(project_id, docs)
-	
-	getAllRanges: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log {project_id}, "getting all ranges"
-		DocManager.getAllNonDeletedDocs project_id, {ranges: true}, (error, docs = []) ->
-			return next(error) if error?
-			res.json HttpController._buildDocsArrayView(project_id, docs)
-
-	updateDoc: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id     = req.params.doc_id
-		lines      = req.body?.lines
-		version    = req.body?.version
-		ranges     = req.body?.ranges
-
-		if !lines? or lines not instanceof Array
-			logger.error project_id: project_id, doc_id: doc_id, "no doc lines provided"
-			res.send 400 # Bad Request
-			return
-		
-		if !version? or typeof version is not "number"
-			logger.error project_id: project_id, doc_id: doc_id, "no doc version provided"
-			res.send 400 # Bad Request
-			return
-		
-		if !ranges?
-			logger.error project_id: project_id, doc_id: doc_id, "no doc ranges provided"
-			res.send 400 # Bad Request
-			return
-
-		bodyLength = lines.reduce(
-			(len, line) => line.length + len
-			0
-		)
-		if bodyLength > Settings.max_doc_length
-			logger.error project_id: project_id, doc_id: doc_id, bodyLength: bodyLength, "document body too large"
-			res.status(413).send("document body too large")
-			return
-
-		logger.log project_id: project_id, doc_id: doc_id, "got http request to update doc"
-		DocManager.updateDoc project_id, doc_id, lines, version, ranges, (error, modified, rev) ->
-			return next(error) if error?
-			res.json {
-				modified: modified
-				rev: rev
-			}
-
-	deleteDoc: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id     = req.params.doc_id
-		logger.log project_id: project_id, doc_id: doc_id, "deleting doc"
-		DocManager.deleteDoc project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	_buildDocView: (doc) -> 
-		doc_view = { _id: doc._id?.toString() }
-		for attribute in ["lines", "rev", "version", "ranges", "deleted"]
-			if doc[attribute]?
-				doc_view[attribute] = doc[attribute]
-		return doc_view
-
-	_buildRawDocView: (doc)->
-		return (doc?.lines or []).join("\n")
-	
-	_buildDocsArrayView: (project_id, docs) ->
-		docViews = []
-		for doc in docs
-			if doc? # There can end up being null docs for some reason :( (probably a race condition)
-				docViews.push HttpController._buildDocView(doc)
-			else
-				logger.error err: new Error("null doc"), project_id: project_id, "encountered null doc"
-		return docViews
-
-	archiveAllDocs: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "archiving all docs"
-		DocArchive.archiveAllDocs project_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	unArchiveAllDocs: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "unarchiving all docs"
-		DocArchive.unArchiveAllDocs project_id, (error) ->
-			return next(error) if error?
-			res.send 200
-
-	destroyAllDocs: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "destroying all docs"
-		DocArchive.destroyAllDocs project_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	healthCheck: (req, res)->
-		HealthChecker.check (err)->
-			if err?
-				logger.err err:err, "error performing health check"
-				res.send 500
-			else
-				res.send 200
-

services/docstore/app/coffee/MongoManager.coffee

@@ -1,93 +0,0 @@
-{db, ObjectId} = require "./mongojs"
-logger = require 'logger-sharelatex'
-metrics = require 'metrics-sharelatex'
-
-module.exports = MongoManager =
-
-	findDoc: (project_id, doc_id, filter, callback = (error, doc) ->) ->
-		db.docs.find {_id: ObjectId(doc_id.toString()), project_id: ObjectId(project_id.toString())}, filter, (error, docs = []) ->
-			callback error, docs[0]
-
-	getProjectsDocs: (project_id, options = {include_deleted: true}, filter, callback)->
-		query = {project_id: ObjectId(project_id.toString())}
-		if !options.include_deleted
-			query.deleted = { $ne: true }
-		db.docs.find query, filter, callback
-
-	getArchivedProjectDocs: (project_id, callback)->
-		query =
-			project_id: ObjectId(project_id.toString())
-			inS3: true
-		db.docs.find query, {}, callback
-
-	upsertIntoDocCollection: (project_id, doc_id, updates, callback)->
-		update =
-			$set: updates
-			$inc:
-				rev: 1
-			$unset:
-				inS3: true
-		update.$set["project_id"] = ObjectId(project_id)
-		db.docs.update _id: ObjectId(doc_id), update, {upsert: true}, callback
-
-	markDocAsDeleted: (project_id, doc_id, callback)->
-		db.docs.update {
-			_id: ObjectId(doc_id),
-			project_id: ObjectId(project_id) 
-		}, {
-			$set: { deleted: true }
-		}, callback
-
-	markDocAsArchived: (doc_id, rev, callback)->
-		update =
-			$set: {}
-			$unset: {}
-		update.$set["inS3"] = true
-		update.$unset["lines"] = true
-		update.$unset["ranges"] = true
-		query =
-			_id: doc_id
-			rev: rev
-		db.docs.update query, update, (err)->
-			callback(err)
-	
-	getDocVersion: (doc_id, callback = (error, version) ->) ->
-		db.docOps.find {
-			doc_id: ObjectId(doc_id)
-		}, {
-			version: 1
-		}, (error, docs) ->
-			return callback(error) if error?
-			if docs.length < 1 or !docs[0].version?
-				return callback null, 0
-			else
-				return callback null, docs[0].version
-
-	setDocVersion: (doc_id, version, callback = (error) ->) ->
-		db.docOps.update {
-			doc_id: ObjectId(doc_id)
-		}, {
-			$set: version: version
-		}, {
-			upsert: true
-		}, callback
-
-	destroyDoc: (doc_id, callback) ->
-		db.docs.remove {
-			_id: ObjectId(doc_id)
-		}, (err) ->
-			return callback(err) if err?
-			db.docOps.remove {
-				doc_id: ObjectId(doc_id)
-			}, callback
-
-[
-	'findDoc',
-	'getProjectsDocs',
-	'getArchivedProjectDocs',
-	'upsertIntoDocCollection',
-	'markDocAsArchived',
-	'getDocVersion',
-	'setDocVersion'
-].map (method) ->
-	metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger)

services/docstore/app/coffee/RangeManager.coffee

@@ -1,40 +0,0 @@
-_ = require "underscore"
-{ObjectId} = require("./mongojs")
-
-module.exports = RangeManager =
-	shouldUpdateRanges: (doc_ranges, incoming_ranges) ->
-		if !incoming_ranges?
-			throw new Error("expected incoming_ranges")
-
-		# If the ranges are empty, we don't store them in the DB, so set
-		# doc_ranges to an empty object as default, since this is was the 
-		# incoming_ranges will be for an empty range set.
-		if !doc_ranges?
-			doc_ranges = {}
-
-		return not _.isEqual(doc_ranges, incoming_ranges)
-	
-	jsonRangesToMongo: (ranges) ->
-		return null if !ranges?
-		
-		updateMetadata = (metadata) ->
-			if metadata?.ts?
-				metadata.ts = new Date(metadata.ts)
-			if metadata?.user_id?
-				metadata.user_id = RangeManager._safeObjectId(metadata.user_id)
-		
-		for change in ranges.changes or []
-			change.id = RangeManager._safeObjectId(change.id)
-			updateMetadata(change.metadata)
-		for comment in ranges.comments or []
-			comment.id = RangeManager._safeObjectId(comment.id)
-			if comment.op?.t?
-				comment.op.t = RangeManager._safeObjectId(comment.op.t)
-			updateMetadata(comment.metadata)
-		return ranges
-	
-	_safeObjectId: (data) ->
-		try
-			return ObjectId(data)
-		catch error
-			return data

services/docstore/app/coffee/mongojs.coffee

@@ -1,7 +0,0 @@
-Settings = require "settings-sharelatex"
-mongojs = require "mongojs"
-db = mongojs(Settings.mongo.url, ["docs", "docOps"])
-module.exports =
-	db: db
-	ObjectId: mongojs.ObjectId
-

services/docstore/app/js/DocArchiveManager.js

@@ -0,0 +1,292 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-useless-escape,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocArchive
+const MongoManager = require('./MongoManager')
+const Errors = require('./Errors')
+const logger = require('logger-sharelatex')
+const _ = require('underscore')
+const async = require('async')
+const settings = require('settings-sharelatex')
+const request = require('request')
+const crypto = require('crypto')
+const RangeManager = require('./RangeManager')
+const thirtySeconds = 30 * 1000
+
+module.exports = DocArchive = {
+  archiveAllDocs(project_id, callback) {
+    if (callback == null) {
+      callback = function(err, docs) {}
+    }
+    return MongoManager.getProjectsDocs(
+      project_id,
+      { include_deleted: true },
+      { lines: true, ranges: true, rev: true, inS3: true },
+      function(err, docs) {
+        if (err != null) {
+          return callback(err)
+        } else if (docs == null) {
+          return callback(
+            new Errors.NotFoundError(`No docs for project ${project_id}`)
+          )
+        }
+        docs = _.filter(docs, doc => doc.inS3 !== true)
+        const jobs = _.map(docs, doc => cb =>
+          DocArchive.archiveDoc(project_id, doc, cb)
+        )
+        return async.parallelLimit(jobs, 5, callback)
+      }
+    )
+  },
+
+  archiveDoc(project_id, doc, callback) {
+    let options
+    logger.log({ project_id, doc_id: doc._id }, 'sending doc to s3')
+    try {
+      options = DocArchive.buildS3Options(project_id + '/' + doc._id)
+    } catch (e) {
+      return callback(e)
+    }
+    return DocArchive._mongoDocToS3Doc(doc, function(error, json_doc) {
+      if (error != null) {
+        return callback(error)
+      }
+      options.body = json_doc
+      options.headers = { 'Content-Type': 'application/json' }
+      return request.put(options, function(err, res) {
+        if (err != null || res.statusCode !== 200) {
+          logger.err(
+            {
+              err,
+              res,
+              project_id,
+              doc_id: doc._id,
+              statusCode: res != null ? res.statusCode : undefined
+            },
+            'something went wrong archiving doc in aws'
+          )
+          return callback(new Error('Error in S3 request'))
+        }
+        const md5lines = crypto
+          .createHash('md5')
+          .update(json_doc, 'utf8')
+          .digest('hex')
+        const md5response = res.headers.etag.toString().replace(/\"/g, '')
+        if (md5lines !== md5response) {
+          logger.err(
+            {
+              responseMD5: md5response,
+              linesMD5: md5lines,
+              project_id,
+              doc_id: doc != null ? doc._id : undefined
+            },
+            'err in response md5 from s3'
+          )
+          return callback(new Error('Error in S3 md5 response'))
+        }
+        return MongoManager.markDocAsArchived(doc._id, doc.rev, function(err) {
+          if (err != null) {
+            return callback(err)
+          }
+          return callback()
+        })
+      })
+    })
+  },
+
+  unArchiveAllDocs(project_id, callback) {
+    if (callback == null) {
+      callback = function(err) {}
+    }
+    return MongoManager.getArchivedProjectDocs(project_id, function(err, docs) {
+      if (err != null) {
+        logger.err({ err, project_id }, 'error unarchiving all docs')
+        return callback(err)
+      } else if (docs == null) {
+        return callback(
+          new Errors.NotFoundError(`No docs for project ${project_id}`)
+        )
+      }
+      const jobs = _.map(
+        docs,
+        doc =>
+          function(cb) {
+            if (doc.inS3 == null) {
+              return cb()
+            } else {
+              return DocArchive.unarchiveDoc(project_id, doc._id, cb)
+            }
+          }
+      )
+      return async.parallelLimit(jobs, 5, callback)
+    })
+  },
+
+  unarchiveDoc(project_id, doc_id, callback) {
+    let options
+    logger.log({ project_id, doc_id }, 'getting doc from s3')
+    try {
+      options = DocArchive.buildS3Options(project_id + '/' + doc_id)
+    } catch (e) {
+      return callback(e)
+    }
+    options.json = true
+    return request.get(options, function(err, res, doc) {
+      if (err != null || res.statusCode !== 200) {
+        logger.err(
+          { err, res, project_id, doc_id },
+          'something went wrong unarchiving doc from aws'
+        )
+        return callback(new Errors.NotFoundError('Error in S3 request'))
+      }
+      return DocArchive._s3DocToMongoDoc(doc, function(error, mongo_doc) {
+        if (error != null) {
+          return callback(error)
+        }
+        return MongoManager.upsertIntoDocCollection(
+          project_id,
+          doc_id.toString(),
+          mongo_doc,
+          function(err) {
+            if (err != null) {
+              return callback(err)
+            }
+            logger.log({ project_id, doc_id }, 'deleting doc from s3')
+            return DocArchive._deleteDocFromS3(project_id, doc_id, callback)
+          }
+        )
+      })
+    })
+  },
+
+  destroyAllDocs(project_id, callback) {
+    if (callback == null) {
+      callback = function(err) {}
+    }
+    return MongoManager.getProjectsDocs(
+      project_id,
+      { include_deleted: true },
+      { _id: 1 },
+      function(err, docs) {
+        if (err != null) {
+          logger.err({ err, project_id }, "error getting project's docs")
+          return callback(err)
+        } else if (docs == null) {
+          return callback()
+        }
+        const jobs = _.map(docs, doc => cb =>
+          DocArchive.destroyDoc(project_id, doc._id, cb)
+        )
+        return async.parallelLimit(jobs, 5, callback)
+      }
+    )
+  },
+
+  destroyDoc(project_id, doc_id, callback) {
+    logger.log({ project_id, doc_id }, 'removing doc from mongo and s3')
+    return MongoManager.findDoc(project_id, doc_id, { inS3: 1 }, function(
+      error,
+      doc
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (doc == null) {
+        return callback(new Errors.NotFoundError('Doc not found in Mongo'))
+      }
+      if (doc.inS3 === true) {
+        return DocArchive._deleteDocFromS3(project_id, doc_id, function(err) {
+          if (err != null) {
+            return err
+          }
+          return MongoManager.destroyDoc(doc_id, callback)
+        })
+      } else {
+        return MongoManager.destroyDoc(doc_id, callback)
+      }
+    })
+  },
+
+  _deleteDocFromS3(project_id, doc_id, callback) {
+    let options
+    try {
+      options = DocArchive.buildS3Options(project_id + '/' + doc_id)
+    } catch (e) {
+      return callback(e)
+    }
+    options.json = true
+    return request.del(options, function(err, res, body) {
+      if (err != null || res.statusCode !== 204) {
+        logger.err(
+          { err, res, project_id, doc_id },
+          'something went wrong deleting doc from aws'
+        )
+        return callback(new Error('Error in S3 request'))
+      }
+      return callback()
+    })
+  },
+
+  _s3DocToMongoDoc(doc, callback) {
+    if (callback == null) {
+      callback = function(error, mongo_doc) {}
+    }
+    const mongo_doc = {}
+    if (doc.schema_v === 1 && doc.lines != null) {
+      mongo_doc.lines = doc.lines
+      if (doc.ranges != null) {
+        mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges)
+      }
+    } else if (doc instanceof Array) {
+      mongo_doc.lines = doc
+    } else {
+      return callback(new Error("I don't understand the doc format in s3"))
+    }
+    return callback(null, mongo_doc)
+  },
+
+  _mongoDocToS3Doc(doc, callback) {
+    if (callback == null) {
+      callback = function(error, s3_doc) {}
+    }
+    if (doc.lines == null) {
+      return callback(new Error('doc has no lines'))
+    }
+    const json = JSON.stringify({
+      lines: doc.lines,
+      ranges: doc.ranges,
+      schema_v: 1
+    })
+    if (json.indexOf('\u0000') !== -1) {
+      const error = new Error('null bytes detected')
+      logger.err({ err: error, doc, json }, error.message)
+      return callback(error)
+    }
+    return callback(null, json)
+  },
+
+  buildS3Options(key) {
+    if (settings.docstore.s3 == null) {
+      throw new Error('S3 settings are not configured')
+    }
+    return {
+      aws: {
+        key: settings.docstore.s3.key,
+        secret: settings.docstore.s3.secret,
+        bucket: settings.docstore.s3.bucket
+      },
+      timeout: thirtySeconds,
+      uri: `https://${settings.docstore.s3.bucket}.s3.amazonaws.com/${key}`
+    }
+  }
+}

services/docstore/app/js/DocManager.js

@@ -0,0 +1,289 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-dupe-keys,
+    no-undef,
+    standard/no-callback-literal,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocManager
+const MongoManager = require('./MongoManager')
+const Errors = require('./Errors')
+const logger = require('logger-sharelatex')
+const _ = require('underscore')
+const DocArchive = require('./DocArchiveManager')
+const RangeManager = require('./RangeManager')
+
+module.exports = DocManager = {
+  // TODO: For historical reasons, the doc version is currently stored in the docOps
+  // collection (which is all that this collection contains). In future, we should
+  // migrate this version property to be part of the docs collection, to guarantee
+  // consitency between lines and version when writing/reading, and for a simpler schema.
+  _getDoc(project_id, doc_id, filter, callback) {
+    if (filter == null) {
+      filter = {}
+    }
+    if (callback == null) {
+      callback = function(error, doc) {}
+    }
+    if (filter.inS3 !== true) {
+      return callback('must include inS3 when getting doc')
+    }
+
+    return MongoManager.findDoc(project_id, doc_id, filter, function(err, doc) {
+      if (err != null) {
+        return callback(err)
+      } else if (doc == null) {
+        return callback(
+          new Errors.NotFoundError(
+            `No such doc: ${doc_id} in project ${project_id}`
+          )
+        )
+      } else if (doc != null ? doc.inS3 : undefined) {
+        return DocArchive.unarchiveDoc(project_id, doc_id, function(err) {
+          if (err != null) {
+            logger.err({ err, project_id, doc_id }, 'error unarchiving doc')
+            return callback(err)
+          }
+          return DocManager._getDoc(project_id, doc_id, filter, callback)
+        })
+      } else {
+        if (filter.version) {
+          return MongoManager.getDocVersion(doc_id, function(error, version) {
+            if (error != null) {
+              return callback(error)
+            }
+            doc.version = version
+            return callback(err, doc)
+          })
+        } else {
+          return callback(err, doc)
+        }
+      }
+    })
+  },
+
+  checkDocExists(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(err, exists) {}
+    }
+    return DocManager._getDoc(
+      project_id,
+      doc_id,
+      { _id: 1, inS3: true },
+      function(err, doc) {
+        if (err != null) {
+          return callback(err)
+        }
+        return callback(err, doc != null)
+      }
+    )
+  },
+
+  getFullDoc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(err, doc) {}
+    }
+    return DocManager._getDoc(
+      project_id,
+      doc_id,
+      {
+        lines: true,
+        rev: true,
+        deleted: true,
+        version: true,
+        ranges: true,
+        inS3: true
+      },
+      function(err, doc) {
+        if (err != null) {
+          return callback(err)
+        }
+        return callback(err, doc)
+      }
+    )
+  },
+
+  getDocLines(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(err, doc) {}
+    }
+    return DocManager._getDoc(
+      project_id,
+      doc_id,
+      { lines: true, inS3: true },
+      function(err, doc) {
+        if (err != null) {
+          return callback(err)
+        }
+        return callback(err, doc)
+      }
+    )
+  },
+
+  getAllNonDeletedDocs(project_id, filter, callback) {
+    if (callback == null) {
+      callback = function(error, docs) {}
+    }
+    return DocArchive.unArchiveAllDocs(project_id, function(error) {
+      if (error != null) {
+        return callback(error)
+      }
+      return MongoManager.getProjectsDocs(
+        project_id,
+        { include_deleted: false },
+        filter,
+        function(error, docs) {
+          if (typeof err !== 'undefined' && err !== null) {
+            return callback(error)
+          } else if (docs == null) {
+            return callback(
+              new Errors.NotFoundError(`No docs for project ${project_id}`)
+            )
+          } else {
+            return callback(null, docs)
+          }
+        }
+      )
+    })
+  },
+
+  updateDoc(project_id, doc_id, lines, version, ranges, callback) {
+    if (callback == null) {
+      callback = function(error, modified, rev) {}
+    }
+    if (lines == null || version == null || ranges == null) {
+      return callback(new Error('no lines, version or ranges provided'))
+    }
+
+    return DocManager._getDoc(
+      project_id,
+      doc_id,
+      {
+        version: true,
+        rev: true,
+        lines: true,
+        version: true,
+        ranges: true,
+        inS3: true
+      },
+      function(err, doc) {
+        let updateLines, updateRanges, updateVersion
+        if (err != null && !(err instanceof Errors.NotFoundError)) {
+          logger.err(
+            { project_id, doc_id, err },
+            'error getting document for update'
+          )
+          return callback(err)
+        }
+
+        ranges = RangeManager.jsonRangesToMongo(ranges)
+
+        if (doc == null) {
+          // If the document doesn't exist, we'll make sure to create/update all parts of it.
+          updateLines = true
+          updateVersion = true
+          updateRanges = true
+        } else {
+          updateLines = !_.isEqual(doc.lines, lines)
+          updateVersion = doc.version !== version
+          updateRanges = RangeManager.shouldUpdateRanges(doc.ranges, ranges)
+        }
+
+        let modified = false
+        let rev = (doc != null ? doc.rev : undefined) || 0
+
+        const updateLinesAndRangesIfNeeded = function(cb) {
+          if (updateLines || updateRanges) {
+            const update = {}
+            if (updateLines) {
+              update.lines = lines
+            }
+            if (updateRanges) {
+              update.ranges = ranges
+            }
+            logger.log({ project_id, doc_id }, 'updating doc lines and ranges')
+
+            modified = true
+            rev += 1 // rev will be incremented in mongo by MongoManager.upsertIntoDocCollection
+            return MongoManager.upsertIntoDocCollection(
+              project_id,
+              doc_id,
+              update,
+              cb
+            )
+          } else {
+            logger.log(
+              { project_id, doc_id },
+              'doc lines have not changed - not updating'
+            )
+            return cb()
+          }
+        }
+
+        const updateVersionIfNeeded = function(cb) {
+          if (updateVersion) {
+            logger.log(
+              {
+                project_id,
+                doc_id,
+                oldVersion: doc != null ? doc.version : undefined,
+                newVersion: version
+              },
+              'updating doc version'
+            )
+            modified = true
+            return MongoManager.setDocVersion(doc_id, version, cb)
+          } else {
+            logger.log(
+              { project_id, doc_id, version },
+              'doc version has not changed - not updating'
+            )
+            return cb()
+          }
+        }
+
+        return updateLinesAndRangesIfNeeded(function(error) {
+          if (error != null) {
+            return callback(error)
+          }
+          return updateVersionIfNeeded(function(error) {
+            if (error != null) {
+              return callback(error)
+            }
+            return callback(null, modified, rev)
+          })
+        })
+      }
+    )
+  },
+
+  deleteDoc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return DocManager.checkDocExists(project_id, doc_id, function(
+      error,
+      exists
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (!exists) {
+        return callback(
+          new Errors.NotFoundError(
+            `No such project/doc to delete: ${project_id}/${doc_id}`
+          )
+        )
+      }
+      return MongoManager.markDocAsDeleted(project_id, doc_id, callback)
+    })
+  }
+}

services/docstore/app/js/Errors.js

@@ -0,0 +1,16 @@
+/* eslint-disable
+    no-proto,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+let Errors
+var NotFoundError = function(message) {
+  const error = new Error(message)
+  error.name = 'NotFoundError'
+  error.__proto__ = NotFoundError.prototype
+  return error
+}
+NotFoundError.prototype.__proto__ = Error.prototype
+
+module.exports = Errors = { NotFoundError }

services/docstore/app/js/HealthChecker.js

@@ -0,0 +1,68 @@
+/* eslint-disable
+    camelcase,
+    standard/no-callback-literal,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { db, ObjectId } = require('./mongojs')
+const request = require('request')
+const async = require('async')
+const _ = require('underscore')
+const crypto = require('crypto')
+const settings = require('settings-sharelatex')
+const { port } = settings.internal.docstore
+const logger = require('logger-sharelatex')
+
+module.exports = {
+  check(callback) {
+    const doc_id = ObjectId()
+    const project_id = ObjectId(settings.docstore.healthCheck.project_id)
+    const url = `http://localhost:${port}/project/${project_id}/doc/${doc_id}`
+    const lines = [
+      'smoke test - delete me',
+      `${crypto.randomBytes(32).toString('hex')}`
+    ]
+    const getOpts = () => ({
+      url,
+      timeout: 3000
+    })
+    logger.log({ lines, url, doc_id, project_id }, 'running health check')
+    const jobs = [
+      function(cb) {
+        const opts = getOpts()
+        opts.json = { lines, version: 42, ranges: {} }
+        return request.post(opts, cb)
+      },
+      function(cb) {
+        const opts = getOpts()
+        opts.json = true
+        return request.get(opts, function(err, res, body) {
+          if (err != null) {
+            logger.err({ err }, 'docstore returned a error in health check get')
+            return cb(err)
+          } else if (res == null) {
+            return cb('no response from docstore with get check')
+          } else if ((res != null ? res.statusCode : undefined) !== 200) {
+            return cb(`status code not 200, its ${res.statusCode}`)
+          } else if (
+            _.isEqual(body != null ? body.lines : undefined, lines) &&
+            (body != null ? body._id : undefined) === doc_id.toString()
+          ) {
+            return cb()
+          } else {
+            return cb(`health check lines not equal ${body.lines} != ${lines}`)
+          }
+        })
+      },
+      cb => db.docs.remove({ _id: doc_id, project_id }, cb),
+      cb => db.docOps.remove({ doc_id }, cb)
+    ]
+    return async.series(jobs, callback)
+  }
+}

services/docstore/app/js/HttpController.js

@@ -0,0 +1,263 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    valid-typeof,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HttpController
+const DocManager = require('./DocManager')
+const logger = require('logger-sharelatex')
+const DocArchive = require('./DocArchiveManager')
+const HealthChecker = require('./HealthChecker')
+const Settings = require('settings-sharelatex')
+
+module.exports = HttpController = {
+  getDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    const include_deleted =
+      (req.query != null ? req.query.include_deleted : undefined) === 'true'
+    logger.log({ project_id, doc_id }, 'getting doc')
+    return DocManager.getFullDoc(project_id, doc_id, function(error, doc) {
+      if (error != null) {
+        return next(error)
+      }
+      logger.log({ doc_id, project_id }, 'got doc')
+      if (doc == null) {
+        return res.send(404)
+      } else if (doc.deleted && !include_deleted) {
+        return res.send(404)
+      } else {
+        return res.json(HttpController._buildDocView(doc))
+      }
+    })
+  },
+
+  getRawDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    logger.log({ project_id, doc_id }, 'getting raw doc')
+    return DocManager.getDocLines(project_id, doc_id, function(error, doc) {
+      if (error != null) {
+        return next(error)
+      }
+      if (doc == null) {
+        return res.send(404)
+      } else {
+        res.setHeader('content-type', 'text/plain')
+        return res.send(HttpController._buildRawDocView(doc))
+      }
+    })
+  },
+
+  getAllDocs(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'getting all docs')
+    return DocManager.getAllNonDeletedDocs(
+      project_id,
+      { lines: true, rev: true },
+      function(error, docs) {
+        if (docs == null) {
+          docs = []
+        }
+        if (error != null) {
+          return next(error)
+        }
+        return res.json(HttpController._buildDocsArrayView(project_id, docs))
+      }
+    )
+  },
+
+  getAllRanges(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'getting all ranges')
+    return DocManager.getAllNonDeletedDocs(
+      project_id,
+      { ranges: true },
+      function(error, docs) {
+        if (docs == null) {
+          docs = []
+        }
+        if (error != null) {
+          return next(error)
+        }
+        return res.json(HttpController._buildDocsArrayView(project_id, docs))
+      }
+    )
+  },
+
+  updateDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    const lines = req.body != null ? req.body.lines : undefined
+    const version = req.body != null ? req.body.version : undefined
+    const ranges = req.body != null ? req.body.ranges : undefined
+
+    if (lines == null || !(lines instanceof Array)) {
+      logger.error({ project_id, doc_id }, 'no doc lines provided')
+      res.send(400) // Bad Request
+      return
+    }
+
+    if (version == null || typeof version === !'number') {
+      logger.error({ project_id, doc_id }, 'no doc version provided')
+      res.send(400) // Bad Request
+      return
+    }
+
+    if (ranges == null) {
+      logger.error({ project_id, doc_id }, 'no doc ranges provided')
+      res.send(400) // Bad Request
+      return
+    }
+
+    const bodyLength = lines.reduce((len, line) => line.length + len, 0)
+    if (bodyLength > Settings.max_doc_length) {
+      logger.error(
+        { project_id, doc_id, bodyLength },
+        'document body too large'
+      )
+      res.status(413).send('document body too large')
+      return
+    }
+
+    logger.log({ project_id, doc_id }, 'got http request to update doc')
+    return DocManager.updateDoc(
+      project_id,
+      doc_id,
+      lines,
+      version,
+      ranges,
+      function(error, modified, rev) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.json({
+          modified,
+          rev
+        })
+      }
+    )
+  },
+
+  deleteDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    logger.log({ project_id, doc_id }, 'deleting doc')
+    return DocManager.deleteDoc(project_id, doc_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(204)
+    })
+  },
+
+  _buildDocView(doc) {
+    const doc_view = { _id: doc._id != null ? doc._id.toString() : undefined }
+    for (const attribute of ['lines', 'rev', 'version', 'ranges', 'deleted']) {
+      if (doc[attribute] != null) {
+        doc_view[attribute] = doc[attribute]
+      }
+    }
+    return doc_view
+  },
+
+  _buildRawDocView(doc) {
+    return ((doc != null ? doc.lines : undefined) || []).join('\n')
+  },
+
+  _buildDocsArrayView(project_id, docs) {
+    const docViews = []
+    for (const doc of Array.from(docs)) {
+      if (doc != null) {
+        // There can end up being null docs for some reason :( (probably a race condition)
+        docViews.push(HttpController._buildDocView(doc))
+      } else {
+        logger.error(
+          { err: new Error('null doc'), project_id },
+          'encountered null doc'
+        )
+      }
+    }
+    return docViews
+  },
+
+  archiveAllDocs(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'archiving all docs')
+    return DocArchive.archiveAllDocs(project_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(204)
+    })
+  },
+
+  unArchiveAllDocs(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'unarchiving all docs')
+    return DocArchive.unArchiveAllDocs(project_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(200)
+    })
+  },
+
+  destroyAllDocs(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'destroying all docs')
+    return DocArchive.destroyAllDocs(project_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(204)
+    })
+  },
+
+  healthCheck(req, res) {
+    return HealthChecker.check(function(err) {
+      if (err != null) {
+        logger.err({ err }, 'error performing health check')
+        return res.send(500)
+      } else {
+        return res.send(200)
+      }
+    })
+  }
+}

services/docstore/app/js/MongoManager.js

@@ -0,0 +1,175 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MongoManager
+const { db, ObjectId } = require('./mongojs')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+module.exports = MongoManager = {
+  findDoc(project_id, doc_id, filter, callback) {
+    if (callback == null) {
+      callback = function(error, doc) {}
+    }
+    return db.docs.find(
+      {
+        _id: ObjectId(doc_id.toString()),
+        project_id: ObjectId(project_id.toString())
+      },
+      filter,
+      function(error, docs) {
+        if (docs == null) {
+          docs = []
+        }
+        return callback(error, docs[0])
+      }
+    )
+  },
+
+  getProjectsDocs(project_id, options, filter, callback) {
+    if (options == null) {
+      options = { include_deleted: true }
+    }
+    const query = { project_id: ObjectId(project_id.toString()) }
+    if (!options.include_deleted) {
+      query.deleted = { $ne: true }
+    }
+    return db.docs.find(query, filter, callback)
+  },
+
+  getArchivedProjectDocs(project_id, callback) {
+    const query = {
+      project_id: ObjectId(project_id.toString()),
+      inS3: true
+    }
+    return db.docs.find(query, {}, callback)
+  },
+
+  upsertIntoDocCollection(project_id, doc_id, updates, callback) {
+    const update = {
+      $set: updates,
+      $inc: {
+        rev: 1
+      },
+      $unset: {
+        inS3: true
+      }
+    }
+    update.$set.project_id = ObjectId(project_id)
+    return db.docs.update(
+      { _id: ObjectId(doc_id) },
+      update,
+      { upsert: true },
+      callback
+    )
+  },
+
+  markDocAsDeleted(project_id, doc_id, callback) {
+    return db.docs.update(
+      {
+        _id: ObjectId(doc_id),
+        project_id: ObjectId(project_id)
+      },
+      {
+        $set: { deleted: true }
+      },
+      callback
+    )
+  },
+
+  markDocAsArchived(doc_id, rev, callback) {
+    const update = {
+      $set: {},
+      $unset: {}
+    }
+    update.$set.inS3 = true
+    update.$unset.lines = true
+    update.$unset.ranges = true
+    const query = {
+      _id: doc_id,
+      rev
+    }
+    return db.docs.update(query, update, err => callback(err))
+  },
+
+  getDocVersion(doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, version) {}
+    }
+    return db.docOps.find(
+      {
+        doc_id: ObjectId(doc_id)
+      },
+      {
+        version: 1
+      },
+      function(error, docs) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (docs.length < 1 || docs[0].version == null) {
+          return callback(null, 0)
+        } else {
+          return callback(null, docs[0].version)
+        }
+      }
+    )
+  },
+
+  setDocVersion(doc_id, version, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return db.docOps.update(
+      {
+        doc_id: ObjectId(doc_id)
+      },
+      {
+        $set: { version }
+      },
+      {
+        upsert: true
+      },
+      callback
+    )
+  },
+
+  destroyDoc(doc_id, callback) {
+    return db.docs.remove(
+      {
+        _id: ObjectId(doc_id)
+      },
+      function(err) {
+        if (err != null) {
+          return callback(err)
+        }
+        return db.docOps.remove(
+          {
+            doc_id: ObjectId(doc_id)
+          },
+          callback
+        )
+      }
+    )
+  }
+}
+;[
+  'findDoc',
+  'getProjectsDocs',
+  'getArchivedProjectDocs',
+  'upsertIntoDocCollection',
+  'markDocAsArchived',
+  'getDocVersion',
+  'setDocVersion'
+].map(method =>
+  metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger)
+)

services/docstore/app/js/RangeManager.js

@@ -0,0 +1,69 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RangeManager
+const _ = require('underscore')
+const { ObjectId } = require('./mongojs')
+
+module.exports = RangeManager = {
+  shouldUpdateRanges(doc_ranges, incoming_ranges) {
+    if (incoming_ranges == null) {
+      throw new Error('expected incoming_ranges')
+    }
+
+    // If the ranges are empty, we don't store them in the DB, so set
+    // doc_ranges to an empty object as default, since this is was the
+    // incoming_ranges will be for an empty range set.
+    if (doc_ranges == null) {
+      doc_ranges = {}
+    }
+
+    return !_.isEqual(doc_ranges, incoming_ranges)
+  },
+
+  jsonRangesToMongo(ranges) {
+    if (ranges == null) {
+      return null
+    }
+
+    const updateMetadata = function(metadata) {
+      if ((metadata != null ? metadata.ts : undefined) != null) {
+        metadata.ts = new Date(metadata.ts)
+      }
+      if ((metadata != null ? metadata.user_id : undefined) != null) {
+        return (metadata.user_id = RangeManager._safeObjectId(metadata.user_id))
+      }
+    }
+
+    for (const change of Array.from(ranges.changes || [])) {
+      change.id = RangeManager._safeObjectId(change.id)
+      updateMetadata(change.metadata)
+    }
+    for (const comment of Array.from(ranges.comments || [])) {
+      comment.id = RangeManager._safeObjectId(comment.id)
+      if ((comment.op != null ? comment.op.t : undefined) != null) {
+        comment.op.t = RangeManager._safeObjectId(comment.op.t)
+      }
+      updateMetadata(comment.metadata)
+    }
+    return ranges
+  },
+
+  _safeObjectId(data) {
+    try {
+      return ObjectId(data)
+    } catch (error) {
+      return data
+    }
+  }
+}

services/docstore/app/js/mongojs.js

@@ -0,0 +1,9 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+const Settings = require('settings-sharelatex')
+const mongojs = require('mongojs')
+const db = mongojs(Settings.mongo.url, ['docs', 'docOps'])
+module.exports = {
+  db,
+  ObjectId: mongojs.ObjectId
+}

services/docstore/buildscript.txt

@@ -1,10 +1,10 @@
 docstore
---public-repo=True
---language=coffeescript
---env-add=
 --node-version=10.19.0
---acceptance-creds=aws
+--script-version=1.3.5
 --dependencies=mongo
+--language=es
 --docker-repos=gcr.io/overleaf-ops
+--acceptance-creds=aws
 --env-pass-through=
---script-version=1.3.5
+--env-add=
+--public-repo=True

services/docstore/config/settings.defaults.coffee

@@ -1,31 +0,0 @@
-http = require('http')
-http.globalAgent.maxSockets = 300
-
-Settings =
-	internal:
-		docstore:
-			port: 3016
-			host: process.env['LISTEN_ADDRESS'] or "localhost"
-
-	mongo:{}
-
-	docstore:
-		healthCheck:
-			project_id: process.env['HEALTH_CHECK_PROJECT_ID']
-
-	max_doc_length: 2 * 1024 * 1024 # 2mb
-
-if process.env['MONGO_CONNECTION_STRING']?
-	Settings.mongo.url = process.env['MONGO_CONNECTION_STRING']
-else if process.env['MONGO_HOST']?
-	Settings.mongo.url = "mongodb://#{process.env['MONGO_HOST']}/sharelatex"
-else
-	Settings.mongo.url = "mongodb://127.0.0.1/sharelatex"
-
-if process.env['AWS_ACCESS_KEY_ID']? and process.env['AWS_SECRET_ACCESS_KEY']? and process.env['AWS_BUCKET']?
-	Settings.docstore.s3 =
-		key: process.env['AWS_ACCESS_KEY_ID']
-		secret: process.env['AWS_SECRET_ACCESS_KEY']
-		bucket: process.env['AWS_BUCKET']
-
-module.exports = Settings

services/docstore/config/settings.defaults.js

@@ -0,0 +1,48 @@
+/*
+ * decaffeinate suggestions:
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const http = require('http')
+http.globalAgent.maxSockets = 300
+
+const Settings = {
+  internal: {
+    docstore: {
+      port: 3016,
+      host: process.env.LISTEN_ADDRESS || 'localhost'
+    }
+  },
+
+  mongo: {},
+
+  docstore: {
+    healthCheck: {
+      project_id: process.env.HEALTH_CHECK_PROJECT_ID
+    }
+  },
+
+  max_doc_length: 2 * 1024 * 1024 // 2mb
+}
+
+if (process.env.MONGO_CONNECTION_STRING != null) {
+  Settings.mongo.url = process.env.MONGO_CONNECTION_STRING
+} else if (process.env.MONGO_HOST != null) {
+  Settings.mongo.url = `mongodb://${process.env.MONGO_HOST}/sharelatex`
+} else {
+  Settings.mongo.url = 'mongodb://127.0.0.1/sharelatex'
+}
+
+if (
+  process.env.AWS_ACCESS_KEY_ID != null &&
+  process.env.AWS_SECRET_ACCESS_KEY != null &&
+  process.env.AWS_BUCKET != null
+) {
+  Settings.docstore.s3 = {
+    key: process.env.AWS_ACCESS_KEY_ID,
+    secret: process.env.AWS_SECRET_ACCESS_KEY,
+    bucket: process.env.AWS_BUCKET
+  }
+}
+
+module.exports = Settings

services/docstore/nodemon.json

@@ -10,10 +10,9 @@
   },
 
   "watch": [
-    "app/coffee/",
-    "app.coffee",
+    "app/js/",
+    "app.js",
     "config/"
   ],
-  "ext": "coffee"
-
+  "ext": "js"
 }

services/docstore/package.json

@@ -8,23 +8,20 @@
     "url": "https://github.com/sharelatex/docstore-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
-    "test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
-    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
-    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
-    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "start": "node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
+    "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "nodemon": "nodemon --config nodemon.json",
-    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+    "lint": "node_modules/.bin/eslint .",
+    "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
+    "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write"
   },
   "dependencies": {
     "async": "~0.8.0",
     "body-parser": "~1.0.2",
-    "coffee-script": "^1.12.7",
-    "express": "~4.1.1",
+    "express": "~4.5.0",
     "logger-sharelatex": "^1.7.0",
     "metrics-sharelatex": "^2.2.0",
     "mongojs": "2.4.0",
@@ -32,10 +29,28 @@
     "underscore": "~1.6.0"
   },
   "devDependencies": {
+    "babel-eslint": "^10.0.3",
     "bunyan": "~0.22.3",
     "chai": "~1.9.1",
+    "eslint": "^6.6.0",
+    "eslint-config-prettier": "^6.10.0",
+    "eslint-config-standard": "^14.1.0",
+    "eslint-config-standard-jsx": "^8.1.0",
+    "eslint-config-standard-react": "^9.2.0",
+    "eslint-plugin-chai-expect": "^2.1.0",
+    "eslint-plugin-chai-friendly": "^0.5.0",
+    "eslint-plugin-import": "^2.20.1",
+    "eslint-plugin-jsx-a11y": "^6.2.3",
+    "eslint-plugin-mocha": "^6.2.2",
+    "eslint-plugin-node": "^11.0.0",
+    "eslint-plugin-prettier": "^3.1.2",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-react": "^7.18.3",
+    "eslint-plugin-standard": "^4.0.1",
     "mocha": "^4.0.1",
-    "request": "~2.34.0",
+    "prettier": "^1.19.1",
+    "prettier-eslint-cli": "^5.0.0",
+    "request": "~2.47.0",
     "sandboxed-module": "~0.3.0",
     "sinon": "~3.2.1"
   },

services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee

@@ -1,85 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-{db, ObjectId} = require "../../../app/js/mongojs"
-expect = chai.expect
-DocstoreApp = require "./helpers/DocstoreApp"
-
-DocstoreClient = require "./helpers/DocstoreClient"
-
-describe "Deleting a doc", ->
-	beforeEach (done) ->
-		@project_id = ObjectId()
-		@doc_id = ObjectId()
-		@lines = ["original", "lines"]
-		@version = 42
-		@ranges = []
-		DocstoreApp.ensureRunning =>
-			DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) =>
-				throw error if error?
-				done()
-
-	describe "when the doc exists", ->
-		beforeEach (done) ->
-			DocstoreClient.deleteDoc @project_id, @doc_id, (error, @res, doc) =>
-				done()
-
-		afterEach (done) ->
-			db.docs.remove({_id: @doc_id}, done)
-
-		it "should insert a deleted doc into the docs collection", (done) ->
-			db.docs.find _id: @doc_id, (error, docs) =>
-				docs[0]._id.should.deep.equal @doc_id
-				docs[0].lines.should.deep.equal @lines
-				docs[0].deleted.should.equal true
-				done()
-
-	describe "when the doc does not exist", ->
-		it "should return a 404", (done) ->
-			missing_doc_id = ObjectId()
-			DocstoreClient.deleteDoc @project_id, missing_doc_id, (error, res, doc) ->
-				res.statusCode.should.equal 404
-				done()
-
-describe "Destroying a project's documents", ->
-	describe "when the doc exists", ->
-		beforeEach (done) ->
-			db.docOps.insert {doc_id: ObjectId(@doc_id), version: 1}, (err) ->
-				return done(err) if err?
-				DocstoreClient.destroyAllDoc @project_id, done
-
-		it "should remove the doc from the docs collection", (done) ->
-			db.docs.find _id: @doc_id, (err, docs) ->
-				expect(err).not.to.exist
-				expect(docs).to.deep.equal []
-				done()
-
-		it "should remove the docOps from the docOps collection", (done) ->
-			db.docOps.find doc_id: @doc_id, (err, docOps) ->
-				expect(err).not.to.exist
-				expect(docOps).to.deep.equal []
-				done()
-
-	describe "when the doc is archived", ->
-		beforeEach (done) ->
-			DocstoreClient.archiveAllDoc @project_id, (err) ->
-				return done(err) if err?
-				DocstoreClient.destroyAllDoc @project_id, done
-
-		it "should remove the doc from the docs collection", (done) ->
-			db.docs.find _id: @doc_id, (err, docs) ->
-				expect(err).not.to.exist
-				expect(docs).to.deep.equal []
-				done()
-
-		it "should remove the docOps from the docOps collection", (done) ->
-			db.docOps.find doc_id: @doc_id, (err, docOps) ->
-				expect(err).not.to.exist
-				expect(docOps).to.deep.equal []
-				done()
-
-		it "should remove the doc contents from s3", (done) ->
-			DocstoreClient.getS3Doc @project_id, @doc_id, (error, res, s3_doc) =>
-				throw error if error?
-				expect(res.statusCode).to.equal 404
-				done()

services/docstore/test/acceptance/coffee/GettingAllDocsTests.coffee

@@ -1,63 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-{ObjectId} = require "mongojs"
-async = require "async"
-DocstoreApp = require "./helpers/DocstoreApp"
-
-DocstoreClient = require "./helpers/DocstoreClient"
-
-describe "Getting all docs", ->
-	beforeEach (done) ->
-		@project_id = ObjectId()
-		@docs = [{
-			_id: ObjectId()
-			lines: ["one", "two", "three"]
-			ranges: {"mock": "one"}
-			rev: 2
-		}, {
-			_id: ObjectId()
-			lines: ["aaa", "bbb", "ccc"]
-			ranges: {"mock": "two"}
-			rev: 4
-		}, {
-			_id: ObjectId()
-			lines: ["111", "222", "333"]
-			ranges: {"mock": "three"}
-			rev: 6
-		}]
-		@deleted_doc = {
-			_id: ObjectId()
-			lines: ["deleted"]
-			ranges: {"mock": "four"}
-			rev: 8
-		}
-		version = 42
-		jobs = for doc in @docs
-			do (doc) =>
-				(callback) => 
-					DocstoreClient.createDoc @project_id, doc._id, doc.lines, version, doc.ranges, callback
-		jobs.push (cb) =>
-			DocstoreClient.createDoc @project_id, @deleted_doc._id, @deleted_doc.lines, version, @deleted_doc.ranges, (err)=>
-					DocstoreClient.deleteDoc @project_id, @deleted_doc._id, cb
-		jobs.unshift (cb)->
-			DocstoreApp.ensureRunning cb
-		async.series jobs, done 
-
-	it "getAllDocs should return all the (non-deleted) docs", (done) ->
-		DocstoreClient.getAllDocs @project_id, (error, res, docs) =>
-			throw error if error?
-			docs.length.should.equal @docs.length
-			for doc, i in docs
-				doc.lines.should.deep.equal @docs[i].lines
-			done()
-
-	it "getAllRanges should return all the (non-deleted) doc ranges", (done) ->
-		DocstoreClient.getAllRanges @project_id, (error, res, docs) =>
-			throw error if error?
-			docs.length.should.equal @docs.length
-			for doc, i in docs
-				doc.ranges.should.deep.equal @docs[i].ranges
-			done()
-
-

services/docstore/test/acceptance/coffee/GettingDocsTests.coffee

@@ -1,63 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-{ObjectId} = require "mongojs"
-DocstoreApp = require "./helpers/DocstoreApp"
-
-DocstoreClient = require "./helpers/DocstoreClient"
-
-describe "Getting a doc", ->
-	beforeEach (done) ->
-		@project_id = ObjectId()
-		@doc_id = ObjectId()
-		@lines = ["original", "lines"]
-		@version = 42
-		@ranges = {
-			changes: [{
-				id: ObjectId().toString()
-				op: { i: "foo", p: 3 }
-				meta:
-					user_id: ObjectId().toString()
-					ts: new Date().toString()
-			}]
-		}
-		DocstoreApp.ensureRunning =>
-			DocstoreClient.createDoc @project_id, @doc_id, @lines, @version, @ranges, (error) =>
-				throw error if error?
-				done()
-
-	describe "when the doc exists", ->
-		it "should get the doc lines and version", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @lines
-				doc.version.should.equal @version
-				doc.ranges.should.deep.equal @ranges
-				done()
-
-	describe "when the doc does not exist", ->
-		it "should return a 404", (done) ->
-			missing_doc_id = ObjectId()
-			DocstoreClient.getDoc @project_id, missing_doc_id, {}, (error, res, doc) ->
-				res.statusCode.should.equal 404
-				done()
-
-	describe "when the doc is a deleted doc", ->
-		beforeEach (done) ->
-			@deleted_doc_id = ObjectId()
-			DocstoreClient.createDoc @project_id, @deleted_doc_id, @lines, @version, @ranges, (error) =>
-				throw error if error?
-				DocstoreClient.deleteDoc @project_id, @deleted_doc_id, done
-
-		it "should return the doc", (done) ->
-			DocstoreClient.getDoc @project_id, @deleted_doc_id, {include_deleted:true},(error, res, doc) =>
-				doc.lines.should.deep.equal @lines
-				doc.version.should.equal @version
-				doc.ranges.should.deep.equal @ranges
-				doc.deleted.should.equal true
-				done()
-
-		it "should return a 404 when the query string is not set", (done)->
-			DocstoreClient.getDoc @project_id, @deleted_doc_id, {},(error, res, doc) =>
-				res.statusCode.should.equal 404
-				done()
-	

services/docstore/test/acceptance/coffee/UpdatingDocsTests.coffee

@@ -1,225 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-{ObjectId} = require "mongojs"
-DocstoreApp = require "./helpers/DocstoreApp"
-
-DocstoreClient = require "./helpers/DocstoreClient"
-
-describe "Applying updates to a doc", ->
-	beforeEach (done) ->
-		@project_id = ObjectId()
-		@doc_id = ObjectId()
-		@originalLines = ["original", "lines"]
-		@newLines = ["new", "lines"]
-		@originalRanges = {
-			changes: [{
-				id: ObjectId().toString()
-				op: { i: "foo", p: 3 }
-				meta:
-					user_id: ObjectId().toString()
-					ts: new Date().toString()
-			}]
-		}
-		@newRanges = {
-			changes: [{
-				id: ObjectId().toString()
-				op: { i: "bar", p: 6 }
-				meta:
-					user_id: ObjectId().toString()
-					ts: new Date().toString()
-			}]
-		}
-		@version = 42
-		DocstoreApp.ensureRunning =>
-			DocstoreClient.createDoc @project_id, @doc_id, @originalLines, @version, @originalRanges, (error) =>
-				throw error if error?
-				done()
-
-	describe "when nothing has been updated", ->
-		beforeEach (done) ->
-			DocstoreClient.updateDoc @project_id, @doc_id, @originalLines, @version, @originalRanges, (error, res, @body) =>
-				done()
-
-		it "should return modified = false", ->
-			@body.modified.should.equal false
-
-		it "should not update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				doc.version.should.equal @version
-				doc.ranges.should.deep.equal @originalRanges
-				done()
-
-	describe "when the lines have changed", ->
-		beforeEach (done) ->
-			DocstoreClient.updateDoc @project_id, @doc_id, @newLines, @version, @originalRanges, (error, res, @body) =>
-				done()
-
-		it "should return modified = true", ->
-			@body.modified.should.equal true
-
-		it "should return the rev", ->
-			@body.rev.should.equal 2
-
-		it "should update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @newLines
-				doc.version.should.equal @version
-				doc.ranges.should.deep.equal @originalRanges
-				done()
-
-	describe "when the version has changed", ->
-		beforeEach (done) ->
-			DocstoreClient.updateDoc @project_id, @doc_id, @originalLines, @version + 1, @originalRanges, (error, res, @body) =>
-				done()
-
-		it "should return modified = true", ->
-			@body.modified.should.equal true
-
-		it "should return the rev", ->
-			@body.rev.should.equal 1
-
-		it "should update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				doc.version.should.equal @version + 1
-				doc.ranges.should.deep.equal @originalRanges
-				done()
-
-	describe "when the ranges have changed", ->
-		beforeEach (done) ->
-			DocstoreClient.updateDoc @project_id, @doc_id, @originalLines, @version, @newRanges, (error, res, @body) =>
-				done()
-
-		it "should return modified = true", ->
-			@body.modified.should.equal true
-
-		it "should return the rev", ->
-			@body.rev.should.equal 2
-
-		it "should update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				doc.version.should.equal @version
-				doc.ranges.should.deep.equal @newRanges
-				done()
-
-	describe "when the doc does not exist", ->
-		beforeEach (done) ->
-			@missing_doc_id = ObjectId()
-			DocstoreClient.updateDoc @project_id, @missing_doc_id, @originalLines, 0, @originalRanges, (error, @res, @body) =>
-				done()
-
-		it "should create the doc", ->
-			@body.rev.should.equal 1
-
-		it "should be retreivable", (done)->
-			DocstoreClient.getDoc @project_id, @missing_doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				doc.version.should.equal 0
-				doc.ranges.should.deep.equal @originalRanges
-				done()
-
-	describe "when malformed doc lines are provided", ->
-		describe "when the lines are not an array", ->
-			beforeEach (done) ->
-				DocstoreClient.updateDoc @project_id, @doc_id, { foo: "bar" }, @version, @originalRanges, (error, @res, @body) =>
-					done()
-
-			it "should return 400", ->
-				@res.statusCode.should.equal 400
-
-			it "should not update the doc in the API", (done) ->
-				DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-					doc.lines.should.deep.equal @originalLines
-					done()
-
-		describe "when the lines are not present", ->
-			beforeEach (done) ->
-				DocstoreClient.updateDoc @project_id, @doc_id, null, @version, @originalRanges, (error, @res, @body) =>
-					done()
-
-			it "should return 400", ->
-				@res.statusCode.should.equal 400
-
-			it "should not update the doc in the API", (done) ->
-				DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-					doc.lines.should.deep.equal @originalLines
-					done()
-	
-	describe "when no version is provided", ->
-		beforeEach (done) ->
-			DocstoreClient.updateDoc @project_id, @doc_id, @originalLines, null, @originalRanges, (error, @res, @body) =>
-				done()
-
-		it "should return 400", ->
-			@res.statusCode.should.equal 400
-
-		it "should not update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				doc.version.should.equal @version
-				done()
-
-	describe "when the content is large", ->
-		beforeEach (done) ->
-			line = new Array(1025).join("x") # 1kb
-			@largeLines = Array.apply(null, Array(1024)).map(() -> line) # 1mb
-			DocstoreClient.updateDoc @project_id, @doc_id, @largeLines, @version, @originalRanges, (error, res, @body) =>
-				done()
-
-		it "should return modified = true", ->
-			@body.modified.should.equal true
-
-		it "should update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @largeLines
-				done()
-
-	describe "when there is a large json payload", ->
-		beforeEach (done) ->
-			line = new Array(1025).join("x") # 1kb
-			@largeLines = Array.apply(null, Array(1024)).map(() -> line) # 1kb
-			@originalRanges.padding = Array.apply(null, Array(2049)).map(() -> line) # 2mb + 1kb
-			DocstoreClient.updateDoc @project_id, @doc_id, @largeLines, @version, @originalRanges, (error, @res, @body) =>
-				done()
-
-		it "should return modified = true", ->
-			@body.modified.should.equal true
-
-		it "should update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @largeLines
-				done()
-
-	describe "when the document body is too large", ->
-		beforeEach (done) ->
-			line = new Array(1025).join("x") # 1kb
-			@largeLines = Array.apply(null, Array(2049)).map(() -> line) # 2mb + 1kb
-			DocstoreClient.updateDoc @project_id, @doc_id, @largeLines, @version, @originalRanges, (error, @res, @body) =>
-				done()
-
-		it "should return 413", ->
-			@res.statusCode.should.equal 413
-
-		it "should report body too large", ->
-			@res.body.should.equal 'document body too large'
-
-		it "should not update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				done()
-
-	describe "when the json payload is too large", ->
-		beforeEach (done) ->
-			line = new Array(1025).join("x") # 1kb
-			@largeLines = Array.apply(null, Array(1024)).map(() -> line) # 1kb
-			@originalRanges.padding = Array.apply(null, Array(4096)).map(() -> line) # 4mb
-			DocstoreClient.updateDoc @project_id, @doc_id, @largeLines, @version, @originalRanges, (error, @res, @body) =>
-				done()
-
-		it "should not update the doc in the API", (done) ->
-			DocstoreClient.getDoc @project_id, @doc_id, {}, (error, res, doc) =>
-				doc.lines.should.deep.equal @originalLines
-				done()

services/docstore/test/acceptance/coffee/helpers/DocstoreApp.coffee

@@ -1,21 +0,0 @@
-app = require('../../../../app')
-require("logger-sharelatex").logger.level("error")
-settings = require("settings-sharelatex")
-
-module.exports =
-	running: false
-	initing: false
-	callbacks: []
-	ensureRunning: (callback = (error) ->) ->
-		if @running
-			return callback()
-		else if @initing
-			@callbacks.push callback
-		else
-			@initing = true
-			@callbacks.push callback
-			app.listen settings.internal.docstore.port, "localhost", (error) => 
-				throw error if error?
-				@running = true
-				for callback in @callbacks
-					callback()

services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee

@@ -1,57 +0,0 @@
-request = require("request").defaults(jar: false)
-{db, ObjectId} = require("../../../../app/js/mongojs")
-settings = require("settings-sharelatex")
-DocArchiveManager = require("../../../../app/js/DocArchiveManager.js")
-
-module.exports = DocstoreClient =
-
-	createDoc: (project_id, doc_id, lines, version, ranges, callback = (error) ->) ->
-		DocstoreClient.updateDoc project_id, doc_id, lines, version, ranges, callback
-
-	getDoc: (project_id, doc_id, qs, callback = (error, res, body) ->) ->
-		request.get {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}"
-			json: true
-			qs:qs
-		}, callback
-
-	getAllDocs: (project_id, callback = (error, res, body) ->) ->
-		request.get {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc"
-			json: true
-		}, callback
-
-	getAllRanges: (project_id, callback = (error, res, body) ->) ->
-		request.get {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/ranges"
-			json: true
-		}, callback
-
-	updateDoc: (project_id, doc_id, lines, version, ranges, callback = (error, res, body) ->) ->
-		request.post {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}"
-			json:
-				lines: lines
-				version: version
-				ranges: ranges
-		}, callback
-
-	deleteDoc: (project_id, doc_id, callback = (error, res, body) ->) ->
-		request.del {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}"
-		}, callback	
-		
-	archiveAllDoc: (project_id, callback = (error, res, body) ->) ->
-		request.post {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive"
-		}, callback
-
-	destroyAllDoc: (project_id, callback = (error, res, body) ->) ->
-		request.post {
-			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/destroy"
-		}, callback
-
-	getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
-		options = DocArchiveManager.buildS3Options(project_id+"/"+doc_id)
-		options.json = true
-		request.get options, callback

services/docstore/test/acceptance/js/ArchiveDocsTests.js

@@ -0,0 +1,1108 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { db, ObjectId, ISODate } = require('../../../app/js/mongojs')
+const async = require('async')
+const Settings = require('settings-sharelatex')
+const DocArchiveManager = require('../../../app/js/DocArchiveManager.js')
+const request = require('request')
+const DocstoreApp = require('./helpers/DocstoreApp')
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Archiving', function() {
+  before(function(done) {
+    return DocstoreApp.ensureRunning(done)
+  })
+
+  describe('multiple docs in a project', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.docs = [
+        {
+          _id: ObjectId(),
+          lines: ['one', 'two', 'three'],
+          ranges: {},
+          version: 2
+        },
+        {
+          _id: ObjectId(),
+          lines: ['aaa', 'bbb', 'ccc'],
+          ranges: {},
+          version: 4
+        }
+      ]
+      const jobs = Array.from(this.docs).map(doc =>
+        (doc => {
+          return callback => {
+            return DocstoreClient.createDoc(
+              this.project_id,
+              doc._id,
+              doc.lines,
+              doc.version,
+              doc.ranges,
+              callback
+            )
+          }
+        })(doc)
+      )
+
+      return async.series(jobs, error => {
+        if (error != null) {
+          throw error
+        }
+        return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+          this.res = res
+          return done()
+        })
+      })
+    })
+
+    it('should archive all the docs', function(done) {
+      this.res.statusCode.should.equal(204)
+      return done()
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      const jobs = Array.from(this.docs).map(doc =>
+        (doc => {
+          return callback => {
+            return db.docs.findOne({ _id: doc._id }, (error, doc) => {
+              should.not.exist(doc.lines)
+              should.not.exist(doc.ranges)
+              doc.inS3.should.equal(true)
+              return callback()
+            })
+          }
+        })(doc)
+      )
+      return async.series(jobs, done)
+    })
+
+    it('should set the docs in s3 correctly', function(done) {
+      const jobs = Array.from(this.docs).map(doc =>
+        (doc => {
+          return callback => {
+            return DocstoreClient.getS3Doc(
+              this.project_id,
+              doc._id,
+              (error, res, s3_doc) => {
+                s3_doc.lines.should.deep.equal(doc.lines)
+                s3_doc.ranges.should.deep.equal(doc.ranges)
+                return callback()
+              }
+            )
+          }
+        })(doc)
+      )
+      return async.series(jobs, done)
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      it('should return the docs', function(done) {
+        for (let i = 0; i < this.fetched_docs.length; i++) {
+          const doc = this.fetched_docs[i]
+          doc.lines.should.deep.equal(this.docs[i].lines)
+        }
+        return done()
+      })
+
+      return it('should restore the docs to mongo', function(done) {
+        const jobs = Array.from(this.docs).map((doc, i) =>
+          ((doc, i) => {
+            return callback => {
+              return db.docs.findOne({ _id: doc._id }, (error, doc) => {
+                doc.lines.should.deep.equal(this.docs[i].lines)
+                doc.ranges.should.deep.equal(this.docs[i].ranges)
+                should.not.exist(doc.inS3)
+                return callback()
+              })
+            }
+          })(doc, i)
+        )
+        return async.series(jobs, done)
+      })
+    })
+  })
+
+  describe('a deleted doc', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.doc = {
+        _id: ObjectId(),
+        lines: ['one', 'two', 'three'],
+        ranges: {},
+        version: 2
+      }
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc._id,
+        this.doc.lines,
+        this.doc.version,
+        this.doc.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.deleteDoc(
+            this.project_id,
+            this.doc._id,
+            error => {
+              if (error != null) {
+                throw error
+              }
+              return DocstoreClient.archiveAllDoc(
+                this.project_id,
+                (error, res) => {
+                  this.res = res
+                  if (error != null) {
+                    throw error
+                  }
+                  return done()
+                }
+              )
+            }
+          )
+        }
+      )
+    })
+
+    it('should successully archive the docs', function(done) {
+      this.res.statusCode.should.equal(204)
+      return done()
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        should.not.exist(doc.lines)
+        should.not.exist(doc.ranges)
+        doc.inS3.should.equal(true)
+        doc.deleted.should.equal(true)
+        return done()
+      })
+    })
+
+    it('should set the doc in s3 correctly', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc._id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          s3_doc.lines.should.deep.equal(this.doc.lines)
+          s3_doc.ranges.should.deep.equal(this.doc.ranges)
+          return done()
+        }
+      )
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      it('should not included the deleted', function(done) {
+        this.fetched_docs.length.should.equal(0)
+        return done()
+      })
+
+      return it('should restore the doc to mongo', function(done) {
+        return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+          if (error != null) {
+            throw error
+          }
+          doc.lines.should.deep.equal(this.doc.lines)
+          doc.ranges.should.deep.equal(this.doc.ranges)
+          should.not.exist(doc.inS3)
+          doc.deleted.should.equal(true)
+          return done()
+        })
+      })
+    })
+  })
+
+  describe('a doc with large lines', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.timeout(1000 * 30)
+      const quarterMegInBytes = 250000
+      const big_line = require('crypto')
+        .randomBytes(quarterMegInBytes)
+        .toString('hex')
+      this.doc = {
+        _id: ObjectId(),
+        lines: [big_line, big_line, big_line, big_line],
+        ranges: {},
+        version: 2
+      }
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc._id,
+        this.doc.lines,
+        this.doc.version,
+        this.doc.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+            this.res = res
+            if (error != null) {
+              throw error
+            }
+            return done()
+          })
+        }
+      )
+    })
+
+    it('should successully archive the docs', function(done) {
+      this.res.statusCode.should.equal(204)
+      return done()
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        should.not.exist(doc.lines)
+        should.not.exist(doc.ranges)
+        doc.inS3.should.equal(true)
+        return done()
+      })
+    })
+
+    it('should set the doc in s3 correctly', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc._id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          s3_doc.lines.should.deep.equal(this.doc.lines)
+          s3_doc.ranges.should.deep.equal(this.doc.ranges)
+          return done()
+        }
+      )
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      return it('should restore the doc to mongo', function(done) {
+        return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+          if (error != null) {
+            throw error
+          }
+          doc.lines.should.deep.equal(this.doc.lines)
+          doc.ranges.should.deep.equal(this.doc.ranges)
+          should.not.exist(doc.inS3)
+          return done()
+        })
+      })
+    })
+  })
+
+  describe('a doc with naughty strings', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.doc = {
+        _id: ObjectId(),
+        lines: [
+          '',
+          'undefined',
+          'undef',
+          'null',
+          'NULL',
+          '(null)',
+          'nil',
+          'NIL',
+          'true',
+          'false',
+          'True',
+          'False',
+          'None',
+          '\\',
+          '\\\\',
+          '0',
+          '1',
+          '1.00',
+          '$1.00',
+          '1/2',
+          '1E2',
+          '1E02',
+          '1E+02',
+          '-1',
+          '-1.00',
+          '-$1.00',
+          '-1/2',
+          '-1E2',
+          '-1E02',
+          '-1E+02',
+          '1/0',
+          '0/0',
+          '-2147483648/-1',
+          '-9223372036854775808/-1',
+          '0.00',
+          '0..0',
+          '.',
+          '0.0.0',
+          '0,00',
+          '0,,0',
+          ',',
+          '0,0,0',
+          '0.0/0',
+          '1.0/0.0',
+          '0.0/0.0',
+          '1,0/0,0',
+          '0,0/0,0',
+          '--1',
+          '-',
+          '-.',
+          '-,',
+          '999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999',
+          'NaN',
+          'Infinity',
+          '-Infinity',
+          '0x0',
+          '0xffffffff',
+          '0xffffffffffffffff',
+          '0xabad1dea',
+          '123456789012345678901234567890123456789',
+          '1,000.00',
+          '1 000.00',
+          "1'000.00",
+          '1,000,000.00',
+          '1 000 000.00',
+          "1'000'000.00",
+          '1.000,00',
+          '1 000,00',
+          "1'000,00",
+          '1.000.000,00',
+          '1 000i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟',
+          '̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕',
+          'Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮',
+          "˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥",
+          '00˙Ɩ$-',
+          'Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ',
+          '𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠',
+          '𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌',
+          '𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈',
+          '𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰',
+          '𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘',
+          '𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐',
+          '⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢',
+          '<script>alert(123)</script>',
+          '&lt;script&gt;alert(&#39;123&#39;);&lt;/script&gt;',
+          '<img src=x onerror=alert(123) />',
+          '<svg><script>123<1>alert(123)</script> ',
+          '"><script>alert(123)</script>',
+          "'><script>alert(123)</script>",
+          '><script>alert(123)</script>',
+          '</script><script>alert(123)</script>',
+          '< / script >< script >alert(123)< / script >',
+          ' onfocus=JaVaSCript:alert(123) autofocus ',
+          '" onfocus=JaVaSCript:alert(123) autofocus ',
+          "' onfocus=JaVaSCript:alert(123) autofocus ",
+          '＜script＞alert(123)＜/script＞',
+          '<sc<script>ript>alert(123)</sc</script>ript>',
+          '--><script>alert(123)</script>',
+          '";alert(123);t="',
+          "';alert(123);t='",
+          'JavaSCript:alert(123)',
+          ';alert(123);',
+          'src=JaVaSCript:prompt(132)',
+          '"><script>alert(123);</script x="',
+          "'><script>alert(123);</script x='",
+          '><script>alert(123);</script x=',
+          '" autofocus onkeyup="javascript:alert(123)',
+          "' autofocus onkeyup='javascript:alert(123)",
+          '<script\\x20type="text/javascript">javascript:alert(1);</script>',
+          '<script\\x3Etype="text/javascript">javascript:alert(1);</script>',
+          '<script\\x0Dtype="text/javascript">javascript:alert(1);</script>',
+          '<script\\x09type="text/javascript">javascript:alert(1);</script>',
+          '<script\\x0Ctype="text/javascript">javascript:alert(1);</script>',
+          '<script\\x2Ftype="text/javascript">javascript:alert(1);</script>',
+          '<script\\x0Atype="text/javascript">javascript:alert(1);</script>',
+          '\'`"><\\x3Cscript>javascript:alert(1)</script>        ',
+          '\'`"><\\x00script>javascript:alert(1)</script>',
+          'ABC<div style="x\\x3Aexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:expression\\x5C(javascript:alert(1)">DEF',
+          'ABC<div style="x:expression\\x00(javascript:alert(1)">DEF',
+          'ABC<div style="x:exp\\x00ression(javascript:alert(1)">DEF',
+          'ABC<div style="x:exp\\x5Cression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x0Aexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x09expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE3\\x80\\x80expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x84expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xC2\\xA0expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x80expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x8Aexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x0Dexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x0Cexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x87expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xEF\\xBB\\xBFexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x20expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x88expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x00expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x8Bexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x86expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x85expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x82expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\x0Bexpression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x81expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x83expression(javascript:alert(1)">DEF',
+          'ABC<div style="x:\\xE2\\x80\\x89expression(javascript:alert(1)">DEF',
+          '<a href="\\x0Bjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x0Fjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xC2\\xA0javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x05javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE1\\xA0\\x8Ejavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x18javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x11javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x88javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x89javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x80javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x17javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x03javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x0Ejavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Ajavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x00javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x10javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x82javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x20javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x13javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x09javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x8Ajavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x14javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x19javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\xAFjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Fjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x81javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Djavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x87javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x07javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE1\\x9A\\x80javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x83javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x04javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x01javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x08javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x84javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x86javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE3\\x80\\x80javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x12javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x0Djavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x0Ajavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x0Cjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x15javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\xA8javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x16javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x02javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Bjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x06javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\xA9javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x80\\x85javascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Ejavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\xE2\\x81\\x9Fjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="\\x1Cjavascript:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="javascript\\x00:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="javascript\\x3A:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="javascript\\x09:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="javascript\\x0D:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '<a href="javascript\\x0A:javascript:alert(1)" id="fuzzelement1">test</a>',
+          '`"\'><img src=xxx:x \\x0Aonerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x22onerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x0Bonerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x0Donerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x2Fonerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x09onerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x0Conerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x00onerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x27onerror=javascript:alert(1)>',
+          '`"\'><img src=xxx:x \\x20onerror=javascript:alert(1)>',
+          '"`\'><script>\\x3Bjavascript:alert(1)</script>',
+          '"`\'><script>\\x0Djavascript:alert(1)</script>',
+          '"`\'><script>\\xEF\\xBB\\xBFjavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x81javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x84javascript:alert(1)</script>',
+          '"`\'><script>\\xE3\\x80\\x80javascript:alert(1)</script>',
+          '"`\'><script>\\x09javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x89javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x85javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x88javascript:alert(1)</script>',
+          '"`\'><script>\\x00javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\xA8javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x8Ajavascript:alert(1)</script>',
+          '"`\'><script>\\xE1\\x9A\\x80javascript:alert(1)</script>',
+          '"`\'><script>\\x0Cjavascript:alert(1)</script>',
+          '"`\'><script>\\x2Bjavascript:alert(1)</script>',
+          '"`\'><script>\\xF0\\x90\\x96\\x9Ajavascript:alert(1)</script>',
+          '"`\'><script>-javascript:alert(1)</script>',
+          '"`\'><script>\\x0Ajavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\xAFjavascript:alert(1)</script>',
+          '"`\'><script>\\x7Ejavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x87javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x81\\x9Fjavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\xA9javascript:alert(1)</script>',
+          '"`\'><script>\\xC2\\x85javascript:alert(1)</script>',
+          '"`\'><script>\\xEF\\xBF\\xAEjavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x83javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x8Bjavascript:alert(1)</script>',
+          '"`\'><script>\\xEF\\xBF\\xBEjavascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x80javascript:alert(1)</script>',
+          '"`\'><script>\\x21javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x82javascript:alert(1)</script>',
+          '"`\'><script>\\xE2\\x80\\x86javascript:alert(1)</script>',
+          '"`\'><script>\\xE1\\xA0\\x8Ejavascript:alert(1)</script>',
+          '"`\'><script>\\x0Bjavascript:alert(1)</script>',
+          '"`\'><script>\\x20javascript:alert(1)</script>',
+          '"`\'><script>\\xC2\\xA0javascript:alert(1)</script>',
+          '<img \\x00src=x onerror="alert(1)">',
+          '<img \\x47src=x onerror="javascript:alert(1)">',
+          '<img \\x11src=x onerror="javascript:alert(1)">',
+          '<img \\x12src=x onerror="javascript:alert(1)">',
+          '<img\\x47src=x onerror="javascript:alert(1)">',
+          '<img\\x10src=x onerror="javascript:alert(1)">',
+          '<img\\x13src=x onerror="javascript:alert(1)">',
+          '<img\\x32src=x onerror="javascript:alert(1)">',
+          '<img\\x47src=x onerror="javascript:alert(1)">',
+          '<img\\x11src=x onerror="javascript:alert(1)">',
+          '<img \\x47src=x onerror="javascript:alert(1)">',
+          '<img \\x34src=x onerror="javascript:alert(1)">',
+          '<img \\x39src=x onerror="javascript:alert(1)">',
+          '<img \\x00src=x onerror="javascript:alert(1)">',
+          '<img src\\x09=x onerror="javascript:alert(1)">',
+          '<img src\\x10=x onerror="javascript:alert(1)">',
+          '<img src\\x13=x onerror="javascript:alert(1)">',
+          '<img src\\x32=x onerror="javascript:alert(1)">',
+          '<img src\\x12=x onerror="javascript:alert(1)">',
+          '<img src\\x11=x onerror="javascript:alert(1)">',
+          '<img src\\x00=x onerror="javascript:alert(1)">',
+          '<img src\\x47=x onerror="javascript:alert(1)">',
+          '<img src=x\\x09onerror="javascript:alert(1)">',
+          '<img src=x\\x10onerror="javascript:alert(1)">',
+          '<img src=x\\x11onerror="javascript:alert(1)">',
+          '<img src=x\\x12onerror="javascript:alert(1)">',
+          '<img src=x\\x13onerror="javascript:alert(1)">',
+          '<img[a][b][c]src[d]=x[e]onerror=[f]"alert(1)">',
+          '<img src=x onerror=\\x09"javascript:alert(1)">',
+          '<img src=x onerror=\\x10"javascript:alert(1)">',
+          '<img src=x onerror=\\x11"javascript:alert(1)">',
+          '<img src=x onerror=\\x12"javascript:alert(1)">',
+          '<img src=x onerror=\\x32"javascript:alert(1)">',
+          '<img src=x onerror=\\x00"javascript:alert(1)">',
+          '<a href=java&#1&#2&#3&#4&#5&#6&#7&#8&#11&#12script:javascript:alert(1)>XXX</a>',
+          '<img src="x` `<script>javascript:alert(1)</script>"` `>',
+          '<img src onerror /" \'"= alt=javascript:alert(1)//">',
+          '<title onpropertychange=javascript:alert(1)></title><title title=>',
+          '<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(1)></a>">',
+          '<!--[if]><script>javascript:alert(1)</script -->',
+          '<!--[if<img src=x onerror=javascript:alert(1)//]> -->',
+          '<script src="/\\%(jscript)s"></script>',
+          '<script src="\\\\%(jscript)s"></script>',
+          '<IMG """><SCRIPT>alert("XSS")</SCRIPT>">',
+          '<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))>',
+          '<IMG SRC=# onmouseover="alert(\'xxs\')">',
+          '<IMG SRC= onmouseover="alert(\'xxs\')">',
+          '<IMG onmouseover="alert(\'xxs\')">',
+          '<IMG SRC=&#106;&#97;&#118;&#97;&#115;&#99;&#114;&#105;&#112;&#116;&#58;&#97;&#108;&#101;&#114;&#116;&#40;&#39;&#88;&#83;&#83;&#39;&#41;>',
+          '<IMG SRC=&#0000106&#0000097&#0000118&#0000097&#0000115&#0000099&#0000114&#0000105&#0000112&#0000116&#0000058&#0000097&#0000108&#0000101&#0000114&#0000116&#0000040&#0000039&#0000088&#0000083&#0000083&#0000039&#0000041>',
+          '<IMG SRC=&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>',
+          '<IMG SRC="jav   ascript:alert(\'XSS\');">',
+          '<IMG SRC="jav&#x09;ascript:alert(\'XSS\');">',
+          '<IMG SRC="jav&#x0A;ascript:alert(\'XSS\');">',
+          '<IMG SRC="jav&#x0D;ascript:alert(\'XSS\');">',
+          'perl -e \'print "<IMG SRC=java\\0script:alert(\\"XSS\\")>";\' > out',
+          '<IMG SRC=" &#14;  javascript:alert(\'XSS\');">',
+          '<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT>',
+          '<BODY onload!#$%&()*~+-_.,:;?@[/|\\]^`=alert("XSS")>',
+          '<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT>',
+          '<<SCRIPT>alert("XSS");//<</SCRIPT>',
+          '<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >',
+          '<SCRIPT SRC=//ha.ckers.org/.j>',
+          '<IMG SRC="javascript:alert(\'XSS\')"',
+          '<iframe src=http://ha.ckers.org/scriptlet.html <',
+          "\\\";alert('XSS');//",
+          '<plaintext>',
+          '1;DROP TABLE users',
+          "1'; DROP TABLE users-- 1",
+          "' OR 1=1 -- 1",
+          "' OR '1'='1",
+          '-',
+          '--',
+          '--version',
+          '--help',
+          '$USER',
+          '/dev/null; touch /tmp/blns.fail ; echo',
+          '`touch /tmp/blns.fail`',
+          '$(touch /tmp/blns.fail)',
+          '@{[system "touch /tmp/blns.fail"]}',
+          'eval("puts \'hello world\'")',
+          'System("ls -al /")',
+          '`ls -al /`',
+          'Kernel.exec("ls -al /")',
+          'Kernel.exit(1)',
+          "%x('ls -al /')",
+          '<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo>',
+          '$HOME',
+          "$ENV{'HOME'}",
+          '%d',
+          '%s',
+          '%*.*s',
+          '../../../../../../../../../../../etc/passwd%00',
+          '../../../../../../../../../../../etc/hosts',
+          '() { 0; }; touch /tmp/blns.shellshock1.fail;',
+          '() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }',
+          'CON',
+          'PRN',
+          'AUX',
+          'CLOCK$',
+          'NUL',
+          'A:',
+          'ZZ:',
+          'COM1',
+          'LPT1',
+          'LPT2',
+          'LPT3',
+          'COM2',
+          'COM3',
+          'COM4',
+          'Scunthorpe General Hospital',
+          'Penistone Community Church',
+          'Lightwater Country Park',
+          'Jimmy Clitheroe',
+          'Horniman Museum',
+          'shitake mushrooms',
+          'RomansInSussex.co.uk',
+          'http://www.cum.qc.ca/',
+          'Craig Cockburn, Software Specialist',
+          'Linda Callahan',
+          'Dr. Herman I. Libshitz',
+          'magna cum laude',
+          'Super Bowl XXX',
+          'medieval erection of parapets',
+          'evaluate',
+          'mocha',
+          'expression',
+          'Arsenal canal',
+          'classic',
+          'Tyson Gay',
+          "If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.",
+          'Roses are \u001b[0;31mred\u001b[0m, violets are \u001b[0;34mblue. Hope you enjoy terminal hue',
+          'But now...\u001b[20Cfor my greatest trick...\u001b[8m',
+          'The quic\b\b\b\b\b\bk brown fo\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007x... [Beeeep]',
+          'Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗'
+        ],
+        ranges: {},
+        version: 2
+      }
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc._id,
+        this.doc.lines,
+        this.doc.version,
+        this.doc.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+            this.res = res
+            if (error != null) {
+              throw error
+            }
+            return done()
+          })
+        }
+      )
+    })
+
+    it('should successully archive the docs', function(done) {
+      this.res.statusCode.should.equal(204)
+      return done()
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        should.not.exist(doc.lines)
+        should.not.exist(doc.ranges)
+        doc.inS3.should.equal(true)
+        return done()
+      })
+    })
+
+    it('should set the doc in s3 correctly', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc._id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          s3_doc.lines.should.deep.equal(this.doc.lines)
+          s3_doc.ranges.should.deep.equal(this.doc.ranges)
+          return done()
+        }
+      )
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      return it('should restore the doc to mongo', function(done) {
+        return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+          if (error != null) {
+            throw error
+          }
+          doc.lines.should.deep.equal(this.doc.lines)
+          doc.ranges.should.deep.equal(this.doc.ranges)
+          should.not.exist(doc.inS3)
+          return done()
+        })
+      })
+    })
+  })
+
+  describe('a doc with ranges', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.doc = {
+        _id: ObjectId(),
+        lines: ['one', 'two', 'three'],
+        ranges: {
+          changes: [
+            {
+              id: ObjectId(),
+              op: { i: 'foo', p: 24 },
+              metadata: {
+                user_id: ObjectId(),
+                ts: new Date('2017-01-27T16:10:44.194Z')
+              }
+            },
+            {
+              id: ObjectId(),
+              op: { d: 'bar', p: 50 },
+              metadata: {
+                user_id: ObjectId(),
+                ts: new Date('2017-01-27T18:10:44.194Z')
+              }
+            }
+          ],
+          comments: [
+            {
+              id: ObjectId(),
+              op: { c: 'comment', p: 284, t: ObjectId() },
+              metadata: {
+                user_id: ObjectId(),
+                ts: new Date('2017-01-26T14:22:04.869Z')
+              }
+            }
+          ]
+        },
+        version: 2
+      }
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc._id,
+        this.doc.lines,
+        this.doc.version,
+        this.doc.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+            this.res = res
+            if (error != null) {
+              throw error
+            }
+            return done()
+          })
+        }
+      )
+    })
+
+    it('should successully archive the docs', function(done) {
+      this.res.statusCode.should.equal(204)
+      return done()
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        should.not.exist(doc.lines)
+        should.not.exist(doc.ranges)
+        doc.inS3.should.equal(true)
+        return done()
+      })
+    })
+
+    it('should set the doc in s3 correctly', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc._id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          s3_doc.lines.should.deep.equal(this.doc.lines)
+          const ranges = JSON.parse(JSON.stringify(this.doc.ranges)) // ObjectId -> String
+          s3_doc.ranges.should.deep.equal(ranges)
+          return done()
+        }
+      )
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      return it('should restore the doc to mongo', function(done) {
+        return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+          if (error != null) {
+            throw error
+          }
+          doc.lines.should.deep.equal(this.doc.lines)
+          doc.ranges.should.deep.equal(this.doc.ranges)
+          should.not.exist(doc.inS3)
+          return done()
+        })
+      })
+    })
+  })
+
+  describe('a doc that is archived twice', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.doc = {
+        _id: ObjectId(),
+        lines: ['abc', 'def', 'ghi'],
+        ranges: {},
+        version: 2
+      }
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc._id,
+        this.doc.lines,
+        this.doc.version,
+        this.doc.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+            this.res = res
+            if (error != null) {
+              throw error
+            }
+            this.res.statusCode.should.equal(204)
+            return DocstoreClient.archiveAllDoc(
+              this.project_id,
+              (error, res1) => {
+                this.res = res1
+                if (error != null) {
+                  throw error
+                }
+                this.res.statusCode.should.equal(204)
+                return done()
+              }
+            )
+          })
+        }
+      )
+    })
+
+    it('should set inS3 and unset lines and ranges in each doc', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        should.not.exist(doc.lines)
+        should.not.exist(doc.ranges)
+        doc.inS3.should.equal(true)
+        return done()
+      })
+    })
+
+    it('should set the doc in s3 correctly', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc._id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          s3_doc.lines.should.deep.equal(this.doc.lines)
+          s3_doc.ranges.should.deep.equal(this.doc.ranges)
+          return done()
+        }
+      )
+    })
+
+    return describe('after unarchiving from a request for the project', function() {
+      before(function(done) {
+        return DocstoreClient.getAllDocs(
+          this.project_id,
+          (error, res, fetched_docs) => {
+            this.fetched_docs = fetched_docs
+            if (error != null) {
+              throw error
+            }
+            return done()
+          }
+        )
+      })
+
+      return it('should restore the doc to mongo', function(done) {
+        return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+          if (error != null) {
+            throw error
+          }
+          doc.lines.should.deep.equal(this.doc.lines)
+          doc.ranges.should.deep.equal(this.doc.ranges)
+          should.not.exist(doc.inS3)
+          return done()
+        })
+      })
+    })
+  })
+
+  return describe('a doc with the old schema (just an array of lines)', function() {
+    before(function(done) {
+      this.project_id = ObjectId()
+      this.doc = {
+        _id: ObjectId(),
+        lines: ['abc', 'def', 'ghi'],
+        ranges: {},
+        version: 2
+      }
+      const options = DocArchiveManager.buildS3Options(
+        `${this.project_id}/${this.doc._id}`
+      )
+      options.json = this.doc.lines
+      return request.put(options, (error, res, body) => {
+        if (error != null) {
+          throw error
+        }
+        res.statusCode.should.equal(200)
+        return db.docs.insert(
+          {
+            project_id: this.project_id,
+            _id: this.doc._id,
+            rev: this.doc.version,
+            inS3: true
+          },
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return DocstoreClient.getAllDocs(
+              this.project_id,
+              (error, res, fetched_docs) => {
+                this.fetched_docs = fetched_docs
+                if (error != null) {
+                  throw error
+                }
+                return done()
+              }
+            )
+          }
+        )
+      })
+    })
+
+    it('should restore the doc to mongo', function(done) {
+      return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+        if (error != null) {
+          throw error
+        }
+        doc.lines.should.deep.equal(this.doc.lines)
+        should.not.exist(doc.inS3)
+        return done()
+      })
+    })
+
+    return it('should return the doc', function(done) {
+      this.fetched_docs[0].lines.should.deep.equal(this.doc.lines)
+      return done()
+    })
+  })
+})

services/docstore/test/acceptance/js/DeletingDocsTests.js

@@ -0,0 +1,159 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { db, ObjectId } = require('../../../app/js/mongojs')
+const { expect } = chai
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Deleting a doc', function() {
+  beforeEach(function(done) {
+    this.project_id = ObjectId()
+    this.doc_id = ObjectId()
+    this.lines = ['original', 'lines']
+    this.version = 42
+    this.ranges = []
+    return DocstoreApp.ensureRunning(() => {
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc_id,
+        this.lines,
+        this.version,
+        this.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the doc exists', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.deleteDoc(
+        this.project_id,
+        this.doc_id,
+        (error, res, doc) => {
+          this.res = res
+          return done()
+        }
+      )
+    })
+
+    afterEach(function(done) {
+      return db.docs.remove({ _id: this.doc_id }, done)
+    })
+
+    return it('should insert a deleted doc into the docs collection', function(done) {
+      return db.docs.find({ _id: this.doc_id }, (error, docs) => {
+        docs[0]._id.should.deep.equal(this.doc_id)
+        docs[0].lines.should.deep.equal(this.lines)
+        docs[0].deleted.should.equal(true)
+        return done()
+      })
+    })
+  })
+
+  return describe('when the doc does not exist', function() {
+    return it('should return a 404', function(done) {
+      const missing_doc_id = ObjectId()
+      return DocstoreClient.deleteDoc(
+        this.project_id,
+        missing_doc_id,
+        (error, res, doc) => {
+          res.statusCode.should.equal(404)
+          return done()
+        }
+      )
+    })
+  })
+})
+
+describe("Destroying a project's documents", function() {
+  describe('when the doc exists', function() {
+    beforeEach(function(done) {
+      return db.docOps.insert(
+        { doc_id: ObjectId(this.doc_id), version: 1 },
+        function(err) {
+          if (err != null) {
+            return done(err)
+          }
+          return DocstoreClient.destroyAllDoc(this.project_id, done)
+        }
+      )
+    })
+
+    it('should remove the doc from the docs collection', function(done) {
+      return db.docs.find({ _id: this.doc_id }, (err, docs) => {
+        expect(err).not.to.exist
+        expect(docs).to.deep.equal([])
+        return done()
+      })
+    })
+
+    return it('should remove the docOps from the docOps collection', function(done) {
+      return db.docOps.find({ doc_id: this.doc_id }, (err, docOps) => {
+        expect(err).not.to.exist
+        expect(docOps).to.deep.equal([])
+        return done()
+      })
+    })
+  })
+
+  return describe('when the doc is archived', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.archiveAllDoc(this.project_id, function(err) {
+        if (err != null) {
+          return done(err)
+        }
+        return DocstoreClient.destroyAllDoc(this.project_id, done)
+      })
+    })
+
+    it('should remove the doc from the docs collection', function(done) {
+      return db.docs.find({ _id: this.doc_id }, (err, docs) => {
+        expect(err).not.to.exist
+        expect(docs).to.deep.equal([])
+        return done()
+      })
+    })
+
+    it('should remove the docOps from the docOps collection', function(done) {
+      return db.docOps.find({ doc_id: this.doc_id }, (err, docOps) => {
+        expect(err).not.to.exist
+        expect(docOps).to.deep.equal([])
+        return done()
+      })
+    })
+
+    return it('should remove the doc contents from s3', function(done) {
+      return DocstoreClient.getS3Doc(
+        this.project_id,
+        this.doc_id,
+        (error, res, s3_doc) => {
+          if (error != null) {
+            throw error
+          }
+          expect(res.statusCode).to.equal(404)
+          return done()
+        }
+      )
+    })
+  })
+})

services/docstore/test/acceptance/js/GettingAllDocsTests.js

@@ -0,0 +1,114 @@
+/* eslint-disable
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { ObjectId } = require('mongojs')
+const async = require('async')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Getting all docs', function() {
+  beforeEach(function(done) {
+    this.project_id = ObjectId()
+    this.docs = [
+      {
+        _id: ObjectId(),
+        lines: ['one', 'two', 'three'],
+        ranges: { mock: 'one' },
+        rev: 2
+      },
+      {
+        _id: ObjectId(),
+        lines: ['aaa', 'bbb', 'ccc'],
+        ranges: { mock: 'two' },
+        rev: 4
+      },
+      {
+        _id: ObjectId(),
+        lines: ['111', '222', '333'],
+        ranges: { mock: 'three' },
+        rev: 6
+      }
+    ]
+    this.deleted_doc = {
+      _id: ObjectId(),
+      lines: ['deleted'],
+      ranges: { mock: 'four' },
+      rev: 8
+    }
+    const version = 42
+    const jobs = Array.from(this.docs).map(doc =>
+      (doc => {
+        return callback => {
+          return DocstoreClient.createDoc(
+            this.project_id,
+            doc._id,
+            doc.lines,
+            version,
+            doc.ranges,
+            callback
+          )
+        }
+      })(doc)
+    )
+    jobs.push(cb => {
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.deleted_doc._id,
+        this.deleted_doc.lines,
+        version,
+        this.deleted_doc.ranges,
+        err => {
+          return DocstoreClient.deleteDoc(
+            this.project_id,
+            this.deleted_doc._id,
+            cb
+          )
+        }
+      )
+    })
+    jobs.unshift(cb => DocstoreApp.ensureRunning(cb))
+    return async.series(jobs, done)
+  })
+
+  it('getAllDocs should return all the (non-deleted) docs', function(done) {
+    return DocstoreClient.getAllDocs(this.project_id, (error, res, docs) => {
+      if (error != null) {
+        throw error
+      }
+      docs.length.should.equal(this.docs.length)
+      for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i]
+        doc.lines.should.deep.equal(this.docs[i].lines)
+      }
+      return done()
+    })
+  })
+
+  return it('getAllRanges should return all the (non-deleted) doc ranges', function(done) {
+    return DocstoreClient.getAllRanges(this.project_id, (error, res, docs) => {
+      if (error != null) {
+        throw error
+      }
+      docs.length.should.equal(this.docs.length)
+      for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i]
+        doc.ranges.should.deep.equal(this.docs[i].ranges)
+      }
+      return done()
+    })
+  })
+})

services/docstore/test/acceptance/js/GettingDocsTests.js

@@ -0,0 +1,137 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { ObjectId } = require('mongojs')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Getting a doc', function() {
+  beforeEach(function(done) {
+    this.project_id = ObjectId()
+    this.doc_id = ObjectId()
+    this.lines = ['original', 'lines']
+    this.version = 42
+    this.ranges = {
+      changes: [
+        {
+          id: ObjectId().toString(),
+          op: { i: 'foo', p: 3 },
+          meta: {
+            user_id: ObjectId().toString(),
+            ts: new Date().toString()
+          }
+        }
+      ]
+    }
+    return DocstoreApp.ensureRunning(() => {
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc_id,
+        this.lines,
+        this.version,
+        this.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the doc exists', function() {
+    return it('should get the doc lines and version', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.lines)
+          doc.version.should.equal(this.version)
+          doc.ranges.should.deep.equal(this.ranges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the doc does not exist', function() {
+    return it('should return a 404', function(done) {
+      const missing_doc_id = ObjectId()
+      return DocstoreClient.getDoc(
+        this.project_id,
+        missing_doc_id,
+        {},
+        (error, res, doc) => {
+          res.statusCode.should.equal(404)
+          return done()
+        }
+      )
+    })
+  })
+
+  return describe('when the doc is a deleted doc', function() {
+    beforeEach(function(done) {
+      this.deleted_doc_id = ObjectId()
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.deleted_doc_id,
+        this.lines,
+        this.version,
+        this.ranges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return DocstoreClient.deleteDoc(
+            this.project_id,
+            this.deleted_doc_id,
+            done
+          )
+        }
+      )
+    })
+
+    it('should return the doc', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.deleted_doc_id,
+        { include_deleted: true },
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.lines)
+          doc.version.should.equal(this.version)
+          doc.ranges.should.deep.equal(this.ranges)
+          doc.deleted.should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    return it('should return a 404 when the query string is not set', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.deleted_doc_id,
+        {},
+        (error, res, doc) => {
+          res.statusCode.should.equal(404)
+          return done()
+        }
+      )
+    })
+  })
+})

services/docstore/test/acceptance/js/UpdatingDocsTests.js

@@ -0,0 +1,499 @@
+/* eslint-disable
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { ObjectId } = require('mongojs')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Applying updates to a doc', function() {
+  beforeEach(function(done) {
+    this.project_id = ObjectId()
+    this.doc_id = ObjectId()
+    this.originalLines = ['original', 'lines']
+    this.newLines = ['new', 'lines']
+    this.originalRanges = {
+      changes: [
+        {
+          id: ObjectId().toString(),
+          op: { i: 'foo', p: 3 },
+          meta: {
+            user_id: ObjectId().toString(),
+            ts: new Date().toString()
+          }
+        }
+      ]
+    }
+    this.newRanges = {
+      changes: [
+        {
+          id: ObjectId().toString(),
+          op: { i: 'bar', p: 6 },
+          meta: {
+            user_id: ObjectId().toString(),
+            ts: new Date().toString()
+          }
+        }
+      ]
+    }
+    this.version = 42
+    return DocstoreApp.ensureRunning(() => {
+      return DocstoreClient.createDoc(
+        this.project_id,
+        this.doc_id,
+        this.originalLines,
+        this.version,
+        this.originalRanges,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when nothing has been updated', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.originalLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = false', function() {
+      return this.body.modified.should.equal(false)
+    })
+
+    return it('should not update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          doc.version.should.equal(this.version)
+          doc.ranges.should.deep.equal(this.originalRanges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the lines have changed', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.newLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = true', function() {
+      return this.body.modified.should.equal(true)
+    })
+
+    it('should return the rev', function() {
+      return this.body.rev.should.equal(2)
+    })
+
+    return it('should update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.newLines)
+          doc.version.should.equal(this.version)
+          doc.ranges.should.deep.equal(this.originalRanges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the version has changed', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.originalLines,
+        this.version + 1,
+        this.originalRanges,
+        (error, res, body) => {
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = true', function() {
+      return this.body.modified.should.equal(true)
+    })
+
+    it('should return the rev', function() {
+      return this.body.rev.should.equal(1)
+    })
+
+    return it('should update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          doc.version.should.equal(this.version + 1)
+          doc.ranges.should.deep.equal(this.originalRanges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the ranges have changed', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.originalLines,
+        this.version,
+        this.newRanges,
+        (error, res, body) => {
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = true', function() {
+      return this.body.modified.should.equal(true)
+    })
+
+    it('should return the rev', function() {
+      return this.body.rev.should.equal(2)
+    })
+
+    return it('should update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          doc.version.should.equal(this.version)
+          doc.ranges.should.deep.equal(this.newRanges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the doc does not exist', function() {
+    beforeEach(function(done) {
+      this.missing_doc_id = ObjectId()
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.missing_doc_id,
+        this.originalLines,
+        0,
+        this.originalRanges,
+        (error, res, body) => {
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should create the doc', function() {
+      return this.body.rev.should.equal(1)
+    })
+
+    return it('should be retreivable', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.missing_doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          doc.version.should.equal(0)
+          doc.ranges.should.deep.equal(this.originalRanges)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when malformed doc lines are provided', function() {
+    describe('when the lines are not an array', function() {
+      beforeEach(function(done) {
+        return DocstoreClient.updateDoc(
+          this.project_id,
+          this.doc_id,
+          { foo: 'bar' },
+          this.version,
+          this.originalRanges,
+          (error, res, body) => {
+            this.res = res
+            this.body = body
+            return done()
+          }
+        )
+      })
+
+      it('should return 400', function() {
+        return this.res.statusCode.should.equal(400)
+      })
+
+      return it('should not update the doc in the API', function(done) {
+        return DocstoreClient.getDoc(
+          this.project_id,
+          this.doc_id,
+          {},
+          (error, res, doc) => {
+            doc.lines.should.deep.equal(this.originalLines)
+            return done()
+          }
+        )
+      })
+    })
+
+    return describe('when the lines are not present', function() {
+      beforeEach(function(done) {
+        return DocstoreClient.updateDoc(
+          this.project_id,
+          this.doc_id,
+          null,
+          this.version,
+          this.originalRanges,
+          (error, res, body) => {
+            this.res = res
+            this.body = body
+            return done()
+          }
+        )
+      })
+
+      it('should return 400', function() {
+        return this.res.statusCode.should.equal(400)
+      })
+
+      return it('should not update the doc in the API', function(done) {
+        return DocstoreClient.getDoc(
+          this.project_id,
+          this.doc_id,
+          {},
+          (error, res, doc) => {
+            doc.lines.should.deep.equal(this.originalLines)
+            return done()
+          }
+        )
+      })
+    })
+  })
+
+  describe('when no version is provided', function() {
+    beforeEach(function(done) {
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.originalLines,
+        null,
+        this.originalRanges,
+        (error, res, body) => {
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return 400', function() {
+      return this.res.statusCode.should.equal(400)
+    })
+
+    return it('should not update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          doc.version.should.equal(this.version)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the content is large', function() {
+    beforeEach(function(done) {
+      const line = new Array(1025).join('x') // 1kb
+      this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1mb
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.largeLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = true', function() {
+      return this.body.modified.should.equal(true)
+    })
+
+    return it('should update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.largeLines)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when there is a large json payload', function() {
+    beforeEach(function(done) {
+      const line = new Array(1025).join('x') // 1kb
+      this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1kb
+      this.originalRanges.padding = Array.apply(null, Array(2049)).map(
+        () => line
+      ) // 2mb + 1kb
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.largeLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return modified = true', function() {
+      return this.body.modified.should.equal(true)
+    })
+
+    return it('should update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.largeLines)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when the document body is too large', function() {
+    beforeEach(function(done) {
+      const line = new Array(1025).join('x') // 1kb
+      this.largeLines = Array.apply(null, Array(2049)).map(() => line) // 2mb + 1kb
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.largeLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    it('should return 413', function() {
+      return this.res.statusCode.should.equal(413)
+    })
+
+    it('should report body too large', function() {
+      return this.res.body.should.equal('document body too large')
+    })
+
+    return it('should not update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          return done()
+        }
+      )
+    })
+  })
+
+  return describe('when the json payload is too large', function() {
+    beforeEach(function(done) {
+      const line = new Array(1025).join('x') // 1kb
+      this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1kb
+      this.originalRanges.padding = Array.apply(null, Array(4096)).map(
+        () => line
+      ) // 4mb
+      return DocstoreClient.updateDoc(
+        this.project_id,
+        this.doc_id,
+        this.largeLines,
+        this.version,
+        this.originalRanges,
+        (error, res, body) => {
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    return it('should not update the doc in the API', function(done) {
+      return DocstoreClient.getDoc(
+        this.project_id,
+        this.doc_id,
+        {},
+        (error, res, doc) => {
+          doc.lines.should.deep.equal(this.originalLines)
+          return done()
+        }
+      )
+    })
+  })
+})

services/docstore/test/acceptance/js/helpers/DocstoreApp.js

@@ -0,0 +1,48 @@
+/* eslint-disable
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const app = require('../../../../app')
+require('logger-sharelatex').logger.level('error')
+const settings = require('settings-sharelatex')
+
+module.exports = {
+  running: false,
+  initing: false,
+  callbacks: [],
+  ensureRunning(callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (this.running) {
+      return callback()
+    } else if (this.initing) {
+      return this.callbacks.push(callback)
+    } else {
+      this.initing = true
+      this.callbacks.push(callback)
+      return app.listen(settings.internal.docstore.port, 'localhost', error => {
+        if (error != null) {
+          throw error
+        }
+        this.running = true
+        return (() => {
+          const result = []
+          for (callback of Array.from(this.callbacks)) {
+            result.push(callback())
+          }
+          return result
+        })()
+      })
+    }
+  }
+}

services/docstore/test/acceptance/js/helpers/DocstoreClient.js

@@ -0,0 +1,136 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocstoreClient
+const request = require('request').defaults({ jar: false })
+const { db, ObjectId } = require('../../../../app/js/mongojs')
+const settings = require('settings-sharelatex')
+const DocArchiveManager = require('../../../../app/js/DocArchiveManager.js')
+
+module.exports = DocstoreClient = {
+  createDoc(project_id, doc_id, lines, version, ranges, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return DocstoreClient.updateDoc(
+      project_id,
+      doc_id,
+      lines,
+      version,
+      ranges,
+      callback
+    )
+  },
+
+  getDoc(project_id, doc_id, qs, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/doc/${doc_id}`,
+        json: true,
+        qs
+      },
+      callback
+    )
+  },
+
+  getAllDocs(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/doc`,
+        json: true
+      },
+      callback
+    )
+  },
+
+  getAllRanges(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/ranges`,
+        json: true
+      },
+      callback
+    )
+  },
+
+  updateDoc(project_id, doc_id, lines, version, ranges, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/doc/${doc_id}`,
+        json: {
+          lines,
+          version,
+          ranges
+        }
+      },
+      callback
+    )
+  },
+
+  deleteDoc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.del(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/doc/${doc_id}`
+      },
+      callback
+    )
+  },
+
+  archiveAllDoc(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/archive`
+      },
+      callback
+    )
+  },
+
+  destroyAllDoc(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/destroy`
+      },
+      callback
+    )
+  },
+
+  getS3Doc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    const options = DocArchiveManager.buildS3Options(project_id + '/' + doc_id)
+    options.json = true
+    return request.get(options, callback)
+  }
+}

services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee

@@ -1,356 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/DocArchiveManager.js"
-SandboxedModule = require('sandboxed-module')
-ObjectId = require("mongojs").ObjectId
-Errors = require "../../../app/js/Errors"
-crypto = require("crypto")
-
-describe "DocArchiveManager", ->
-
-	beforeEach ->
-
-		@settings =
-			docstore:
-				s3:
-					secret: "secret"
-					key: "this_key"
-					bucket:"doc-archive-unit-test"
-
-		@request = 
-			put: {}
-			get: {}
-			del: {}
-
-		@archivedDocs = [{
-			_id: ObjectId()
-			inS3:true
-			rev: 2
-		}, {
-			_id: ObjectId()
-			inS3:true
-			rev: 4
-		}, {
-			_id: ObjectId()
-			inS3:true
-			rev: 6
-		}]
-
-		@mongoDocs = [{
-			_id: ObjectId()
-			lines: ["one", "two", "three"]
-			rev: 2
-		}, {
-			_id: ObjectId()
-			lines: ["aaa", "bbb", "ccc"]
-			rev: 4
-		}, {
-			_id: ObjectId()
-			inS3: true
-			rev: 6
-		}, {
-			_id: ObjectId()
-			inS3: true
-			rev: 6
-		}, {
-			_id: ObjectId()
-			lines: ["111", "222", "333"]
-			rev: 6
-		}]
-
-		@unarchivedDocs = [{
-			_id: ObjectId()
-			lines: ["wombat", "potato", "banana"]
-			rev: 2
-		}, {
-			_id: ObjectId()
-			lines: ["llama", "turnip", "apple"]
-			rev: 4
-		}, {
-			_id: ObjectId()
-			lines: ["elephant", "swede", "nectarine"]
-			rev: 6
-		}]
-
-		@mixedDocs = @archivedDocs.concat(@unarchivedDocs)
-
-		@MongoManager =
-			markDocAsArchived: sinon.stub().callsArgWith(2, null)
-			upsertIntoDocCollection: sinon.stub().callsArgWith(3, null)
-			getProjectsDocs: sinon.stub().callsArgWith(3, null, @mongoDocs)
-			getArchivedProjectDocs: sinon.stub().callsArgWith(2, null, @mongoDocs)
-
-		@requires = 
-			"settings-sharelatex": @settings
-			"./MongoManager": @MongoManager
-			"request": @request
-			"./RangeManager": @RangeManager = {}
-			"logger-sharelatex":
-				log:->
-				err:->
-		@globals =
-			JSON: JSON
-
-		@error = "my errror"
-		@project_id = ObjectId().toString()
-		@stubbedError = new Errors.NotFoundError("Error in S3 request")
-		@DocArchiveManager = SandboxedModule.require modulePath, requires: @requires, globals: @globals
-
-	describe "archiveDoc", ->
-
-		it "should use correct options", (done)->
-			@request.put = sinon.stub().callsArgWith(1,  null, {statusCode:200,headers:{etag:""}})
-			@DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=>
-				opts = @request.put.args[0][0]
-				assert.deepEqual(opts.aws, {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket})
-				opts.body.should.equal JSON.stringify(
-					lines: @mongoDocs[0].lines
-					ranges: @mongoDocs[0].ranges
-					schema_v: 1
-				)
-				opts.timeout.should.equal (30*1000)
-				opts.uri.should.equal "https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{@mongoDocs[0]._id}"
-				done()
-
-		it "should return no md5 error", (done)->
-			data = JSON.stringify(
-				lines: @mongoDocs[0].lines
-				ranges: @mongoDocs[0].ranges
-				schema_v: 1
-			)
-			@md5 = crypto.createHash("md5").update(data).digest("hex")
-			@request.put = sinon.stub().callsArgWith(1,  null, {statusCode:200,headers:{etag:@md5}})
-			@DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=>
-				should.not.exist err
-				done()
-
-		it "should return the error", (done)->
-			@request.put = sinon.stub().callsArgWith(1, @stubbedError, {statusCode:400,headers:{etag:""}})
-			@DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=>
-				should.exist err
-				done()
-
-	describe "unarchiveDoc", ->
-
-		it "should use correct options", (done)->
-			@request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @mongoDocs[0].lines)
-			@request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {})
-			@DocArchiveManager.unarchiveDoc @project_id, @mongoDocs[0]._id, (err)=>
-				opts = @request.get.args[0][0]
-				assert.deepEqual(opts.aws, {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket})
-				opts.json.should.equal true
-				opts.timeout.should.equal (30*1000)
-				opts.uri.should.equal "https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{@mongoDocs[0]._id}"
-				done()
-
-		it "should return the error", (done)->
-			@request.get = sinon.stub().callsArgWith(1, @stubbedError, {}, {})
-			@DocArchiveManager.unarchiveDoc @project_id, @mongoDocs[0], (err)=>
-				should.exist err
-				done()
-
-		it "should error if the doc lines are a string not an array", (done)->
-			@request.get = sinon.stub().callsArgWith(1, null, statusCode:200, "this is a string")
-			@request.del = sinon.stub()
-			@DocArchiveManager.unarchiveDoc @project_id, @mongoDocs[0], (err)=>
-				should.exist err
-				@request.del.called.should.equal false
-				done()
-
-	describe "archiveAllDocs", ->
-
-		it "should archive all project docs which are not in s3", (done)->
-			@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @mongoDocs)
-			@DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)
-
-			@DocArchiveManager.archiveAllDocs @project_id, (err)=>
-				@DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[0]).should.equal true
-				@DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[1]).should.equal true
-				@DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[4]).should.equal true
-
-				@DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[2]).should.equal false
-				@DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[3]).should.equal false
-				
-				should.not.exist err
-				done()
-
-		it "should return error if have no docs", (done)->
-			@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, null)
-
-			@DocArchiveManager.archiveAllDocs @project_id, (err)=>
-				should.exist err
-				done()
-
-		it "should return the error", (done)->
-			@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, @error, null)
-
-			@DocArchiveManager.archiveAllDocs @project_id, (err)=>
-				err.should.equal @error
-				done()
-
-		describe "when most have been already put in s3", ->
-
-			beforeEach ->
-				numberOfDocs = 10 * 1000
-				@mongoDocs = []
-				while --numberOfDocs != 0
-					@mongoDocs.push({inS3:true, _id: ObjectId()})
-
-				@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @mongoDocs)
-				@DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)	
-
-			it "should not throw and error", (done)->
-				@DocArchiveManager.archiveAllDocs @project_id, (err)=>
-					should.not.exist err
-					done()
-
-
-	describe "unArchiveAllDocs", ->
-
-		it "should unarchive all inS3 docs", (done)->
-			@MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, @archivedDocs)
-			@DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null)
-			@DocArchiveManager.unArchiveAllDocs @project_id, (err)=>
-				for doc in @archivedDocs
-					@DocArchiveManager.unarchiveDoc.calledWith(@project_id, doc._id).should.equal true
-				should.not.exist err
-				done()
-
-		it "should return error if have no docs", (done)->
-			@MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, null)
-			@DocArchiveManager.unArchiveAllDocs @project_id, (err)=>
-				should.exist err
-				done()
-
-		it "should return the error", (done)->
-			@MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, @error, null)
-			@DocArchiveManager.unArchiveAllDocs @project_id, (err)=>
-				err.should.equal @error
-				done()
-
-	describe "destroyAllDocs", ->
-		beforeEach ->
-			@request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {})
-			@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @mixedDocs)
-			@MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
-			@MongoManager.destroyDoc = sinon.stub().yields()
-			for doc in @mixedDocs
-				@MongoManager.findDoc.withArgs(@project_id, doc._id).callsArgWith(3, null, doc)
-
-		it "should destroy all the docs", (done)->
-			@DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
-			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
-				for doc in @mixedDocs
-					@DocArchiveManager.destroyDoc.calledWith(@project_id, doc._id).should.equal true
-				should.not.exist err
-				done()
-
-		it "should only the s3 docs from s3", (done)->
-			docOpts = (doc) =>
-				JSON.parse(JSON.stringify({
-					aws: {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket},
-					json: true,
-					timeout: 30 * 1000
-					uri:"https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{doc._id}"
-				}))
-
-			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
-				expect(err).not.to.exist
-
-				for doc in @archivedDocs
-					sinon.assert.calledWith(@request.del, docOpts(doc))
-				for doc in @unarchivedDocs
-					expect(@request.del.calledWith(docOpts(doc))).to.equal false  # no notCalledWith
-
-				done()
-
-		it "should remove the docs from mongo", (done)->
-			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
-				expect(err).not.to.exist
-
-			for doc in @mixedDocs
-				sinon.assert.calledWith(@MongoManager.destroyDoc, doc._id)
-
-			done()
-	
-	describe "_s3DocToMongoDoc", ->
-		describe "with the old schema", ->
-			it "should return the docs lines", (done) ->
-				@DocArchiveManager._s3DocToMongoDoc ["doc", "lines"], (error, doc) ->
-					expect(doc).to.deep.equal {
-						lines: ["doc", "lines"]
-					}
-					done()
-		
-		describe "with the new schema", ->
-			it "should return the doc lines and ranges", (done) ->
-				@RangeManager.jsonRangesToMongo = sinon.stub().returns {"mongo": "ranges"}
-				@DocArchiveManager._s3DocToMongoDoc {
-					lines: ["doc", "lines"]
-					ranges: {"json": "ranges"}
-					schema_v: 1
-				}, (error, doc) ->
-					expect(doc).to.deep.equal {
-						lines: ["doc", "lines"]
-						ranges: {"mongo": "ranges"}
-					}
-					done()
-					
-			it "should return just the doc lines when there are no ranges", (done) ->
-				@DocArchiveManager._s3DocToMongoDoc {
-					lines: ["doc", "lines"]
-					schema_v: 1
-				}, (error, doc) ->
-					expect(doc).to.deep.equal {
-						lines: ["doc", "lines"]
-					}
-					done()
-		
-		describe "with an unrecognised schema", ->
-			it "should return an error", (done) ->
-				@DocArchiveManager._s3DocToMongoDoc {
-					schema_v: 2
-				}, (error, doc) ->
-					expect(error).to.exist
-					done()
-	
-	describe "_mongoDocToS3Doc", ->
-		describe "with a valid doc", ->
-			it "should return the json version", (done) ->
-				@DocArchiveManager._mongoDocToS3Doc doc = {
-					lines: ["doc", "lines"]
-					ranges: { "mock": "ranges" }
-				}, (err, s3_doc) ->
-					expect(s3_doc).to.equal JSON.stringify({
-						lines: ["doc", "lines"]
-						ranges: { "mock": "ranges" }
-						schema_v: 1
-					})
-					done()
-			
-		describe "with null bytes in the result", ->
-			beforeEach ->
-				@_stringify = JSON.stringify
-				JSON.stringify = sinon.stub().returns '{"bad": "\u0000"}'
-			
-			afterEach ->
-				JSON.stringify = @_stringify
-				
-			it "should return an error", (done) ->
-				@DocArchiveManager._mongoDocToS3Doc {
-					lines: ["doc", "lines"]
-					ranges: { "mock": "ranges" }
-				}, (err, s3_doc) ->
-					expect(err).to.exist
-					done()
-		
-		describe "without doc lines", ->
-			it "should return an error", (done) ->
-				@DocArchiveManager._mongoDocToS3Doc {}, (err, s3_doc) ->
-					expect(err).to.exist
-					done()
-			
-			

services/docstore/test/unit/coffee/DocManagerTests.coffee

@@ -1,407 +0,0 @@
-SandboxedModule = require('sandboxed-module')
-sinon = require('sinon')
-chai = require('chai')
-assert = require("chai").assert
-chai.should()
-expect = chai.expect
-modulePath = require('path').join __dirname, '../../../app/js/DocManager'
-ObjectId = require("mongojs").ObjectId
-Errors = require "../../../app/js/Errors"
-
-describe "DocManager", ->
-	beforeEach ->
-		@DocManager = SandboxedModule.require modulePath, requires:
-			"./MongoManager": @MongoManager = {}
-			"./DocArchiveManager": @DocArchiveManager = {}
-			"./RangeManager": @RangeManager = {
-				jsonRangesToMongo: (r) -> r
-				shouldUpdateRanges: sinon.stub().returns false
-			}
-			"logger-sharelatex": @logger = 
-				log: sinon.stub()
-				warn:->
-				err:->
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-		@another_project_id = ObjectId().toString()
-		@callback = sinon.stub()
-		@stubbedError = new Error("blew up")
-
-
-	describe "checkDocExists", ->
-		beforeEach ->
-			@DocManager._getDoc = sinon.stub()
-
-		it "should call get doc with a quick filter", (done)->
-			@DocManager._getDoc.callsArgWith(3, null, {_id:@doc_id})
-			@DocManager.checkDocExists @project_id, @doc_id, (err, exist)=>
-				exist.should.equal true
-				@DocManager._getDoc.calledWith(@project_id, @doc_id, {_id:1, inS3:true}).should.equal true
-				done()
-
-		it "should return false when doc is not there", (done)->
-			@DocManager._getDoc.callsArgWith(3, null)
-			@DocManager.checkDocExists @project_id, @doc_id, (err, exist)=>
-				exist.should.equal false
-				done()
-
-		it "should return error when get doc errors", (done)->
-			@DocManager._getDoc.callsArgWith(3, "error")
-			@DocManager.checkDocExists @project_id, @doc_id, (err, exist)=>
-				err.should.equal "error"
-				done()
-
-	describe "getFullDoc", ->
-		beforeEach ->
-			@DocManager._getDoc = sinon.stub()
-			@doc = 
-				_id: @doc_id
-				lines:["2134"]
-
-		it "should call get doc with a quick filter", (done)->
-			@DocManager._getDoc.callsArgWith(3, null, @doc)
-			@DocManager.getFullDoc @project_id, @doc_id, (err, doc)=>
-				doc.should.equal @doc
-				@DocManager._getDoc.calledWith(@project_id, @doc_id, {lines: true, rev: true, deleted: true, version: true, ranges: true, inS3:true}).should.equal true
-				done()
-	
-		it "should return error when get doc errors", (done)->
-			@DocManager._getDoc.callsArgWith(3, "error")
-			@DocManager.getFullDoc @project_id, @doc_id, (err, exist)=>
-				err.should.equal "error"
-				done()
-
-	describe "getRawDoc", ->
-
-		beforeEach ->
-			@DocManager._getDoc = sinon.stub()
-			@doc = 
-				lines:["2134"]
-
-		it "should call get doc with a quick filter", (done)->
-			@DocManager._getDoc.callsArgWith(3, null, @doc)
-			@DocManager.getDocLines @project_id, @doc_id, (err, doc)=>
-				doc.should.equal @doc
-				@DocManager._getDoc.calledWith(@project_id, @doc_id, {lines: true, inS3:true}).should.equal true
-				done()
-	
-		it "should return error when get doc errors", (done)->
-			@DocManager._getDoc.callsArgWith(3, "error")
-			@DocManager.getDocLines @project_id, @doc_id, (err, exist)=>
-				err.should.equal "error"
-				done()
-
-
-
-	describe "getDoc", ->
-		beforeEach ->
-			@project = { name: "mock-project" }
-			@doc = { _id: @doc_id, project_id: @project_id, lines: ["mock-lines"] }
-			@version = 42
-			@MongoManager.findDoc = sinon.stub()
-			@MongoManager.getDocVersion = sinon.stub().yields(null, @version)
-
-		describe "when using a filter", ->
-			beforeEach ->
-				@MongoManager.findDoc.yields(null, @doc)
-
-			it "should error if inS3 is not set to true", (done)->
-				@DocManager._getDoc @project_id, @doc_id, {inS3: false}, (err)->
-					expect(err).to.exist
-					done()
-
-			it "should always get inS3 even when no filter is passed", (done)->
-				@DocManager._getDoc @project_id, @doc_id, undefined, (err)=>
-					@MongoManager.findDoc.called.should.equal false
-					expect(err).to.exist
-					done()
-
-			it "should not error if inS3 is set to true", (done)->
-				@DocManager._getDoc @project_id, @doc_id, {inS3: true}, (err)->
-					expect(err).to.not.exist
-					done()
-
-		describe "when the doc is in the doc collection", ->
-			beforeEach ->
-				@MongoManager.findDoc.yields(null, @doc)
-				@DocManager._getDoc @project_id, @doc_id, {version: true, inS3:true}, @callback
-
-			it "should get the doc from the doc collection", ->
-				@MongoManager.findDoc
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should get the doc version from the docOps collection", ->
-				@MongoManager.getDocVersion
-					.calledWith(@doc_id)
-					.should.equal true
-			
-			it "should return the callback with the doc with the version", ->
-				@callback.called.should.equal true
-				doc = @callback.args[0][1]
-				doc.lines.should.equal @doc.lines
-				doc.version.should.equal @version
-
-		describe "without the version filter", ->
-			beforeEach ->
-				@MongoManager.findDoc.yields(null, @doc)
-				@DocManager._getDoc @project_id, @doc_id, {version: false, inS3:true}, @callback
-
-			it "should not get the doc version from the docOps collection", ->
-				@MongoManager.getDocVersion.called.should.equal false
-		
-		describe "when MongoManager.findDoc errors", ->
-			beforeEach ->
-				@MongoManager.findDoc.yields(@stubbedError)
-				@DocManager._getDoc @project_id, @doc_id, {version: true, inS3:true}, @callback
-
-			it "should return the error", ->
-				@callback.calledWith(@stubbedError).should.equal true
-
-		describe "when the doc is archived", ->
-			beforeEach -> 
-				@doc = { _id: @doc_id, project_id: @project_id, lines: ["mock-lines"], inS3: true }
-				@MongoManager.findDoc.yields(null, @doc)
-				@DocArchiveManager.unarchiveDoc = (project_id, doc_id, callback) =>
-					@doc.inS3 = false
-					callback()
-				sinon.spy @DocArchiveManager, "unarchiveDoc"
-				@DocManager._getDoc @project_id, @doc_id, {version: true, inS3:true}, @callback
-
-			it "should call the DocArchive to unarchive the doc", ->
-				@DocArchiveManager.unarchiveDoc.calledWith(@project_id, @doc_id).should.equal true
-			
-			it "should look up the doc twice", ->
-				@MongoManager.findDoc.calledTwice.should.equal true
-
-			it "should return the doc", ->
-				@callback.calledWith(null, @doc).should.equal true
-
-		describe "when the doc does not exist in the docs collection", ->
-			beforeEach -> 
-				@MongoManager.findDoc = sinon.stub().yields(null, null)
-				@DocManager._getDoc @project_id, @doc_id, {version: true, inS3:true}, @callback
-
-			it "should return a NotFoundError", ->
-				@callback
-					.calledWith(sinon.match.has('message', "No such doc: #{@doc_id} in project #{@project_id}"))
-					.should.equal true
-
-	describe "getAllNonDeletedDocs", ->
-		describe "when the project exists", ->
-			beforeEach -> 
-				@docs = [{ _id: @doc_id, project_id: @project_id, lines: ["mock-lines"] }]
-				@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @docs)
-				@DocArchiveManager.unArchiveAllDocs = sinon.stub().callsArgWith(1, null, @docs)
-				@filter = { lines: true }
-				@DocManager.getAllNonDeletedDocs @project_id, @filter, @callback
-
-			it "should get the project from the database", ->
-				@MongoManager.getProjectsDocs
-					.calledWith(@project_id, {include_deleted: false}, @filter)
-					.should.equal true
-
-			it "should return the docs", ->
-				@callback.calledWith(null, @docs).should.equal true
-
-		describe "when there are no docs for the project", ->
-			beforeEach -> 
-				@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, null)
-				@DocArchiveManager.unArchiveAllDocs = sinon.stub().callsArgWith(1, null)
-				@DocManager.getAllNonDeletedDocs @project_id, @filter, @callback
-
-			it "should return a NotFoundError", ->
-				@callback
-					.calledWith(sinon.match.has('message', "No docs for project #{@project_id}"))
-					.should.equal true
-
-	describe "deleteDoc", ->
-		describe "when the doc exists", ->
-			beforeEach ->
-				@lines = ["mock", "doc", "lines"]
-				@rev = 77
-				@DocManager.checkDocExists = sinon.stub().callsArgWith(2, null, true)
-				@MongoManager.markDocAsDeleted = sinon.stub().callsArg(2)
-				@DocManager.deleteDoc @project_id, @doc_id, @callback
-
-			it "should get the doc", ->
-				@DocManager.checkDocExists
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should mark doc as deleted", ->
-				@MongoManager.markDocAsDeleted
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should return the callback", ->
-				@callback.called.should.equal true
-
-		describe "when the doc does not exist", ->
-			beforeEach -> 
-				@DocManager.checkDocExists = sinon.stub().callsArgWith(2, null, false)
-				@DocManager.deleteDoc @project_id, @doc_id, @callback
-
-			it "should return a NotFoundError", ->
-				@callback
-					.calledWith(sinon.match.has('message', "No such project/doc to delete: #{@project_id}/#{@doc_id}"))
-					.should.equal true
-
-	describe "updateDoc", ->
-		beforeEach ->
-			@oldDocLines = ["old", "doc", "lines"]
-			@newDocLines = ["new", "doc", "lines"]
-			@originalRanges = {
-				changes: [{
-					id: ObjectId().toString()
-					op: { i: "foo", p: 3 }
-					meta:
-						user_id: ObjectId().toString()
-						ts: new Date().toString()
-				}]
-			}
-			@newRanges = {
-				changes: [{
-					id: ObjectId().toString()
-					op: { i: "bar", p: 6 }
-					meta:
-						user_id: ObjectId().toString()
-						ts: new Date().toString()
-				}]
-			}
-			@version = 42
-			@doc = { _id: @doc_id, project_id: @project_id, lines: @oldDocLines, rev: @rev = 5, version: @version, ranges: @originalRanges }
-
-			@MongoManager.upsertIntoDocCollection = sinon.stub().callsArg(3)
-			@MongoManager.setDocVersion = sinon.stub().yields()
-			@DocManager._getDoc = sinon.stub()
-
-		describe "when only the doc lines have changed", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, @doc)
-				@DocManager.updateDoc @project_id, @doc_id, @newDocLines, @version, @originalRanges, @callback
-
-			it "should get the existing doc", ->
-				@DocManager._getDoc
-					.calledWith(@project_id, @doc_id, {version: true, rev: true, lines: true, version: true, ranges: true, inS3:true})
-					.should.equal true
-
-			it "should upsert the document to the doc collection", ->
-				@MongoManager.upsertIntoDocCollection
-					.calledWith(@project_id, @doc_id, {lines: @newDocLines})
-					.should.equal true
-			
-			it "should not update the version", ->
-				@MongoManager.setDocVersion.called.should.equal false
-
-			it "should return the callback with the new rev", ->
-				@callback.calledWith(null, true, @rev + 1).should.equal true
-
-		describe "when the doc ranges have changed", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, @doc)
-				@RangeManager.shouldUpdateRanges.returns true
-				@DocManager.updateDoc @project_id, @doc_id, @oldDocLines, @version, @newRanges, @callback
-
-			it "should upsert the ranges", ->
-				@MongoManager.upsertIntoDocCollection
-					.calledWith(@project_id, @doc_id, {ranges: @newRanges})
-					.should.equal true
-			
-			it "should not update the version", ->
-				@MongoManager.setDocVersion.called.should.equal false
-
-			it "should return the callback with the new rev", ->
-				@callback.calledWith(null, true, @rev + 1).should.equal true
-
-		describe "when only the version has changed", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, @doc)
-				@DocManager.updateDoc @project_id, @doc_id, @oldDocLines, @version + 1, @originalRanges, @callback
-
-			it "should not change the lines or ranges", ->
-				@MongoManager.upsertIntoDocCollection.called.should.equal false
-			
-			it "should update the version", ->
-				@MongoManager.setDocVersion
-					.calledWith(@doc_id, @version + 1)
-					.should.equal true
-
-			it "should return the callback with the old rev", ->
-				@callback.calledWith(null, true, @rev).should.equal true
-
-		describe "when the doc has not changed at all", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, @doc)
-				@DocManager.updateDoc @project_id, @doc_id, @oldDocLines, @version, @originalRanges, @callback
-
-			it "should not update the ranges or lines", ->
-				@MongoManager.upsertIntoDocCollection.called.should.equal false
-			
-			it "should not update the version", ->
-				@MongoManager.setDocVersion.called.should.equal false
-
-			it "should return the callback with the old rev and modified == false", ->
-				@callback.calledWith(null, false, @rev).should.equal true
-
-		describe "when the version is null", ->
-			beforeEach ->
-				@DocManager.updateDoc @project_id, @doc_id, @newDocLines, null, @originalRanges, @callback
-
-			it "should return an error", ->
-				@callback.calledWith(sinon.match.has('message', "no lines, version or ranges provided")).should.equal true
-
-		describe "when the lines are null", ->
-			beforeEach ->
-				@DocManager.updateDoc @project_id, @doc_id, null, @version, @originalRanges, @callback
-
-			it "should return an error", ->
-				@callback.calledWith(sinon.match.has('message', "no lines, version or ranges provided")).should.equal true
-
-		describe "when the ranges are null", ->
-			beforeEach ->
-				@DocManager.updateDoc @project_id, @doc_id, @newDocLines, @version, null, @callback
-
-			it "should return an error", ->
-				@callback.calledWith(sinon.match.has('message', "no lines, version or ranges provided")).should.equal true
-
-		describe "when there is a generic error getting the doc", ->
-			beforeEach ->
-				@error =  new Error("doc could not be found")
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, @error, null, null)
-				@DocManager.updateDoc @project_id, @doc_id, @newDocLines, @version, @originalRanges, @callback
-			
-			it "should not upsert the document to the doc collection", ->
-				@MongoManager.upsertIntoDocCollection.called.should.equal false
-
-			it "should return the callback with the error", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the doc lines have not changed", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, @doc)
-				@DocManager.updateDoc @project_id, @doc_id, @oldDocLines.slice(), @version, @originalRanges, @callback
-
-			it "should not update the doc", ->
-				@MongoManager.upsertIntoDocCollection.called.should.equal false
-
-			it "should return the callback with the existing rev", ->
-				@callback.calledWith(null, false, @rev).should.equal true
-
-		describe "when the doc does not exist", ->
-			beforeEach ->
-				@DocManager._getDoc = sinon.stub().callsArgWith(3, null, null, null)
-				@DocManager.updateDoc @project_id, @doc_id, @newDocLines, @version, @originalRanges, @callback
-			
-			it "should upsert the document to the doc collection", ->
-				@MongoManager.upsertIntoDocCollection
-					.calledWith(@project_id, @doc_id, {lines: @newDocLines, ranges: @originalRanges})
-					.should.equal true
-				
-			it "should set the version", ->
-				@MongoManager.setDocVersion
-					.calledWith(@doc_id, @version)
-					.should.equal true
-
-			it "should return the callback with the new rev", ->
-				@callback.calledWith(null, true, 1).should.equal true

services/docstore/test/unit/coffee/HttpControllerTests.coffee

@@ -1,357 +0,0 @@
-SandboxedModule = require('sandboxed-module')
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-chai.should()
-expect = chai.expect
-modulePath = require('path').join __dirname, '../../../app/js/HttpController'
-ObjectId = require("mongojs").ObjectId
-
-describe "HttpController", ->
-	beforeEach ->
-		@HttpController = SandboxedModule.require modulePath, requires:
-			"./DocManager": @DocManager = {}
-			"./DocArchiveManager": @DocArchiveManager = {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
-			"./HealthChecker": {}
-		@res = { send: sinon.stub(), json: sinon.stub(), setHeader:sinon.stub() }
-		@res.status = sinon.stub().returns(@res)
-		@req = { query:{}}
-		@next = sinon.stub()
-		@project_id = "mock-project-id"
-		@doc_id = "mock-doc-id"
-		@doc = {
-			_id: @doc_id
-			lines: ["mock", "lines", " here", "", "", " spaces "]
-			version: 42
-			rev: 5
-		}
-		@deletedDoc = {
-			deleted:true
-			_id: @doc_id
-			lines: ["mock", "lines", " here", "", "", " spaces "]
-			version: 42
-			rev: 5
-		}
-
-	describe "getDoc", ->
-
-		describe "without deleted docs", ->
-			beforeEach ->
-				@req.params =
-					project_id: @project_id
-					doc_id: @doc_id
-				@DocManager.getFullDoc = sinon.stub().callsArgWith(2, null, @doc)
-				@HttpController.getDoc @req, @res, @next
-
-			it "should get the document with the version (including deleted)", ->
-				@DocManager.getFullDoc
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should return the doc as JSON", ->
-				@res.json
-					.calledWith({
-						_id: @doc_id
-						lines: @doc.lines
-						rev: @doc.rev
-						version: @doc.version
-					})
-					.should.equal true
-
-		describe "which is deleted", ->
-			beforeEach ->
-				@req.params =
-					project_id: @project_id
-					doc_id: @doc_id
-				@DocManager.getFullDoc = sinon.stub().callsArgWith(2, null, @deletedDoc)
-
-			it "should get the doc from the doc manager", ->
-				@HttpController.getDoc @req, @res, @next
-				@DocManager.getFullDoc.calledWith(@project_id, @doc_id).should.equal true
-
-			it "should return 404 if the query string delete is not set ", ->
-				@HttpController.getDoc @req, @res, @next
-				@res.send.calledWith(404).should.equal true
-
-			it "should return the doc as JSON if include_deleted is set to true", ->
-				@req.query.include_deleted = "true"
-				@HttpController.getDoc @req, @res, @next
-				@res.json
-					.calledWith({
-						_id: @doc_id
-						lines: @doc.lines
-						rev: @doc.rev
-						deleted: true
-						version: @doc.version
-					})
-					.should.equal true
-
-	describe "getRawDoc", ->
-		beforeEach ->
-			@req.params =
-				project_id: @project_id
-				doc_id: @doc_id
-			@DocManager.getDocLines = sinon.stub().callsArgWith(2, null, @doc)
-			@HttpController.getRawDoc @req, @res, @next
-
-		it "should get the document without the version", ->
-			@DocManager.getDocLines
-				.calledWith(@project_id, @doc_id)
-				.should.equal true
-
-		it "should set the content type header", ->
-			@res.setHeader.calledWith('content-type', 'text/plain').should.equal true
-
-		it "should send the raw version of the doc", ->
-			assert.deepEqual @res.send.args[0][0], "#{@doc.lines[0]}\n#{@doc.lines[1]}\n#{@doc.lines[2]}\n#{@doc.lines[3]}\n#{@doc.lines[4]}\n#{@doc.lines[5]}"
-
-	describe "getAllDocs", ->
-		describe "normally", ->
-			beforeEach ->
-				@req.params =
-					project_id: @project_id
-				@docs = [{
-					_id: ObjectId()
-					lines: ["mock", "lines", "one"]
-					rev: 2
-				}, {
-					_id: ObjectId()
-					lines: ["mock", "lines", "two"]
-					rev: 4
-				}]
-				@DocManager.getAllNonDeletedDocs = sinon.stub().callsArgWith(2, null, @docs)
-				@HttpController.getAllDocs @req, @res, @next
-
-			it "should get all the (non-deleted) docs", ->
-				@DocManager.getAllNonDeletedDocs
-					.calledWith(@project_id, {lines: true, rev: true})
-					.should.equal true
-
-			it "should return the doc as JSON", ->
-				@res.json
-					.calledWith([{
-						_id:   @docs[0]._id.toString()
-						lines: @docs[0].lines
-						rev:   @docs[0].rev
-					}, {
-						_id:   @docs[1]._id.toString()
-						lines: @docs[1].lines
-						rev:   @docs[1].rev
-					}])
-					.should.equal true
-
-		describe "with a null doc", ->
-			beforeEach ->
-				@req.params =
-					project_id: @project_id
-				@docs = [{
-					_id: ObjectId()
-					lines: ["mock", "lines", "one"]
-					rev: 2
-				},
-				null,
-				{
-					_id: ObjectId()
-					lines: ["mock", "lines", "two"]
-					rev: 4
-				}]
-				@DocManager.getAllNonDeletedDocs = sinon.stub().callsArgWith(2, null, @docs)
-				@HttpController.getAllDocs @req, @res, @next
-
-			it "should return the non null docs as JSON", ->
-				@res.json
-					.calledWith([{
-						_id:   @docs[0]._id.toString()
-						lines: @docs[0].lines
-						rev:   @docs[0].rev
-					}, {
-						_id:   @docs[2]._id.toString()
-						lines: @docs[2].lines
-						rev:   @docs[2].rev
-					}])
-					.should.equal true
-
-			it "should log out an error", ->
-				@logger.error
-					.calledWith(
-						err: sinon.match.has('message', "null doc")
-						project_id: @project_id
-						"encountered null doc"
-					)
-					.should.equal true
-
-	describe "getAllRanges", ->
-		describe "normally", ->
-			beforeEach ->
-				@req.params =
-					project_id: @project_id
-				@docs = [{
-					_id: ObjectId()
-					ranges: {"mock_ranges": "one"}
-				}, {
-					_id: ObjectId()
-					ranges: {"mock_ranges": "two"}
-				}]
-				@DocManager.getAllNonDeletedDocs = sinon.stub().callsArgWith(2, null, @docs)
-				@HttpController.getAllRanges @req, @res, @next
-
-			it "should get all the (non-deleted) doc ranges", ->
-				@DocManager.getAllNonDeletedDocs
-					.calledWith(@project_id, {ranges: true})
-					.should.equal true
-
-			it "should return the doc as JSON", ->
-				@res.json
-					.calledWith([{
-						_id:    @docs[0]._id.toString()
-						ranges: @docs[0].ranges
-					}, {
-						_id:    @docs[1]._id.toString()
-						ranges: @docs[1].ranges
-					}])
-					.should.equal true
-
-	describe "updateDoc", ->
-		beforeEach ->
-			@req.params =
-				project_id: @project_id
-				doc_id: @doc_id
-
-		describe "when the doc lines exist and were updated", ->
-			beforeEach ->
-				@req.body =
-					lines: @lines = ["hello", "world"]
-					version: @version = 42
-					ranges: @ranges = { changes: "mock" }
-				@DocManager.updateDoc = sinon.stub().yields(null, true, @rev = 5)
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should update the document", ->
-				@DocManager.updateDoc
-					.calledWith(@project_id, @doc_id, @lines, @version, @ranges)
-					.should.equal true
-
-			it "should return a modified status", ->
-				@res.json
-					.calledWith(modified: true, rev: @rev)
-					.should.equal true
-
-		describe "when the doc lines exist and were not updated", ->
-			beforeEach ->
-				@req.body =
-					lines: @lines = ["hello", "world"]
-					version: @version = 42
-					ranges: {}
-				@DocManager.updateDoc = sinon.stub().yields(null, false, @rev = 5)
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should return a modified status", ->
-				@res.json
-					.calledWith(modified: false, rev: @rev)
-					.should.equal true
-
-		describe "when the doc lines are not provided", ->
-			beforeEach ->
-				@req.body = { version: 42, ranges: {} }
-				@DocManager.updateDoc = sinon.stub().yields(null, false)
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should not update the document", ->
-				@DocManager.updateDoc.called.should.equal false
-
-			it "should return a 400 (bad request) response", ->
-				@res.send
-					.calledWith(400)
-					.should.equal true
-
-		describe "when the doc version are not provided", ->
-			beforeEach ->
-				@req.body = { version: 42, lines: ["hello world"] }
-				@DocManager.updateDoc = sinon.stub().yields(null, false)
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should not update the document", ->
-				@DocManager.updateDoc.called.should.equal false
-
-			it "should return a 400 (bad request) response", ->
-				@res.send
-					.calledWith(400)
-					.should.equal true
-
-		describe "when the doc ranges is not provided", ->
-			beforeEach ->
-				@req.body = { lines : [ "foo" ], version: 42 }
-				@DocManager.updateDoc = sinon.stub().yields(null, false)
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should not update the document", ->
-				@DocManager.updateDoc.called.should.equal false
-
-			it "should return a 400 (bad request) response", ->
-				@res.send
-					.calledWith(400)
-					.should.equal true
-
-		describe "when the doc body is too large", ->
-			beforeEach ->
-				@req.body =
-					lines: @lines = Array(2049).fill('a'.repeat(1024))
-					version: @version = 42
-					ranges: @ranges = { changes: "mock" }
-				@HttpController.updateDoc @req, @res, @next
-
-			it "should return a 413 (too large) response", ->
-				sinon.assert.calledWith(@res.status, 413)
-
-			it "should report that the document body is too large", ->
-				sinon.assert.calledWith(@res.send, "document body too large")
-
-	describe "deleteDoc", ->
-		beforeEach ->
-			@req.params =
-				project_id: @project_id
-				doc_id: @doc_id
-			@DocManager.deleteDoc = sinon.stub().callsArg(2)
-			@HttpController.deleteDoc @req, @res, @next
-
-		it "should delete the document", ->
-			@DocManager.deleteDoc
-				.calledWith(@project_id, @doc_id)
-				.should.equal true
-
-		it "should return a 204 (No Content)", ->
-			@res.send
-				.calledWith(204)
-				.should.equal true
-
-	describe "archiveAllDocs", ->
-		beforeEach ->
-			@req.params =
-				project_id: @project_id
-			@DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1)
-			@HttpController.archiveAllDocs @req, @res, @next
-
-		it "should archive the project", ->
-			@DocArchiveManager.archiveAllDocs
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should return a 204 (No Content)", ->
-			@res.send
-				.calledWith(204)
-				.should.equal true
-
-	describe "destroyAllDocs", ->
-		beforeEach ->
-			@req.params =
-				project_id: @project_id
-			@DocArchiveManager.destroyAllDocs = sinon.stub().callsArg(1)
-			@HttpController.destroyAllDocs @req, @res, @next
-
-		it "should destroy the docs", ->
-			sinon.assert.calledWith(@DocArchiveManager.destroyAllDocs, @project_id)
-
-		it "should return 204", ->
-			sinon.assert.calledWith(@res.send, 204)
-

services/docstore/test/unit/coffee/MongoManagerTests.coffee

@@ -1,168 +0,0 @@
-SandboxedModule = require('sandboxed-module')
-sinon = require('sinon')
-require('chai').should()
-modulePath = require('path').join __dirname, '../../../app/js/MongoManager'
-ObjectId = require("mongojs").ObjectId
-assert = require("chai").assert
-
-describe "MongoManager", ->
-	beforeEach ->
-		@MongoManager = SandboxedModule.require modulePath, requires:
-			"./mongojs":
-				db: @db = { docs: {}, docOps: {} }
-				ObjectId: ObjectId
-			'metrics-sharelatex': {timeAsyncMethod: sinon.stub()}
-			'logger-sharelatex': {log: ()->}
-		@project_id = ObjectId().toString()
-		@doc_id = ObjectId().toString()
-		@callback = sinon.stub()
-		@stubbedErr = new Error("hello world")
-
-	describe "findDoc", ->
-		beforeEach ->
-			@doc = { name: "mock-doc"}
-			@db.docs.find = sinon.stub().callsArgWith(2, null, [@doc])
-			@filter = { lines: true }
-			@MongoManager.findDoc @project_id, @doc_id, @filter, @callback
-
-		it "should find the doc", ->
-			@db.docs.find
-				.calledWith({
-					_id: ObjectId(@doc_id)
-					project_id: ObjectId(@project_id)
-				}, @filter)
-				.should.equal true
-
-		it "should call the callback with the doc", ->
-			@callback.calledWith(null, @doc).should.equal true
-
-	describe "getProjectsDocs", ->
-		beforeEach ->
-			@filter = {lines: true}
-			@doc1 = { name: "mock-doc1" }
-			@doc2 = { name: "mock-doc2" }
-			@doc3 = { name: "mock-doc3" }
-			@doc4 = { name: "mock-doc4" }
-			@db.docs.find = sinon.stub().callsArgWith(2, null, [@doc, @doc3, @doc4])
-		
-		describe "with included_deleted = false", ->
-			beforeEach -> 
-				@MongoManager.getProjectsDocs @project_id, include_deleted: false, @filter, @callback
-
-			it "should find the non-deleted docs via the project_id", ->
-				@db.docs.find
-					.calledWith({
-						project_id: ObjectId(@project_id)
-						deleted: { $ne: true }
-					}, @filter)
-					.should.equal true
-
-			it "should call the callback with the docs", ->
-				@callback.calledWith(null, [@doc, @doc3, @doc4]).should.equal true
-				
-		describe "with included_deleted = true", ->
-			beforeEach ->
-				@MongoManager.getProjectsDocs @project_id, include_deleted: true, @filter, @callback
-
-			it "should find all via the project_id", ->
-				@db.docs.find
-					.calledWith({
-						project_id: ObjectId(@project_id)
-					}, @filter)
-					.should.equal true
-
-			it "should call the callback with the docs", ->
-				@callback.calledWith(null, [@doc, @doc3, @doc4]).should.equal true
-
-	describe "upsertIntoDocCollection", ->
-		beforeEach ->
-			@db.docs.update = sinon.stub().callsArgWith(3, @stubbedErr)
-			@oldRev = 77
-
-		it "should upsert the document", (done)->	
-			@MongoManager.upsertIntoDocCollection @project_id, @doc_id, {@lines}, (err)=>
-				args = @db.docs.update.args[0]
-				assert.deepEqual args[0], {_id: ObjectId(@doc_id)}
-				assert.equal args[1]["$set"]["lines"], @lines
-				assert.equal args[1]["$inc"]["rev"], 1
-				assert.deepEqual args[1]["$set"]["project_id"], ObjectId(@project_id)
-				done()
-
-		it "should return the error", (done)->
-			@MongoManager.upsertIntoDocCollection @project_id, @doc_id, {@lines}, (err)=>
-				err.should.equal @stubbedErr
-				done()
-
-	describe "markDocAsDeleted", ->
-		beforeEach ->
-			@db.docs.update = sinon.stub().callsArgWith(2, @stubbedErr)
-			@oldRev = 77
-
-		it "should process the update", (done) ->
-			@MongoManager.markDocAsDeleted @project_id, @doc_id, (err)=>
-				args = @db.docs.update.args[0]
-				assert.deepEqual args[0], {_id: ObjectId(@doc_id), project_id: ObjectId(@project_id)}
-				assert.equal args[1]["$set"]["deleted"], true
-				done()
-
-		it "should return the error", (done)->
-			@MongoManager.markDocAsDeleted @project_id, @doc_id, (err)=>
-				err.should.equal @stubbedErr
-				done()
-
-	describe "destroyDoc", ->
-		beforeEach (done) ->
-			@db.docs.remove = sinon.stub().yields()
-			@db.docOps.remove = sinon.stub().yields()
-			@MongoManager.destroyDoc '123456789012', done
-
-		it "should destroy the doc", ->
-			sinon.assert.calledWith(@db.docs.remove, {_id: ObjectId('123456789012')})
-
-		it "should destroy the docOps", ->
-			sinon.assert.calledWith(@db.docOps.remove, {doc_id: ObjectId('123456789012')})
-
-	describe "getDocVersion", ->
-		describe "when the doc exists", ->
-			beforeEach ->
-				@doc =
-					version: @version = 42
-				@db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc])
-				@MongoManager.getDocVersion @doc_id, @callback
-
-			it "should look for the doc in the database", ->
-				@db.docOps.find
-					.calledWith({ doc_id: ObjectId(@doc_id) }, {version: 1})
-					.should.equal true
-
-			it "should call the callback with the version", ->
-				@callback.calledWith(null, @version).should.equal true
-
-		describe "when the doc doesn't exist", ->
-			beforeEach ->
-				@db.docOps.find = sinon.stub().callsArgWith(2, null, [])
-				@MongoManager.getDocVersion @doc_id, @callback
-
-			it "should call the callback with 0", ->
-				@callback.calledWith(null, 0).should.equal true
-
-	describe "setDocVersion", ->
-		beforeEach ->
-			@version = 42
-			@db.docOps.update = sinon.stub().callsArg(3)
-			@MongoManager.setDocVersion @doc_id, @version, @callback
-
-		it "should update the doc version", ->
-			@db.docOps.update
-				.calledWith({
-					doc_id: ObjectId(@doc_id)
-				}, {
-					$set:
-						version: @version
-				}, {
-					upsert: true 
-				})
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true

services/docstore/test/unit/coffee/RangeManagerTests.coffee

@@ -1,155 +0,0 @@
-SandboxedModule = require('sandboxed-module')
-sinon = require('sinon')
-require('chai').should()
-expect = require('chai').expect
-modulePath = require('path').join __dirname, '../../../app/js/RangeManager'
-ObjectId = require("mongojs").ObjectId
-assert = require("chai").assert
-_ = require "underscore"
-
-describe "RangeManager", ->
-	beforeEach ->
-		@RangeManager = SandboxedModule.require modulePath, requires:
-			"./mongojs":
-				ObjectId: ObjectId
-
-	describe "jsonRangesToMongo", ->
-		it "should convert ObjectIds and dates to proper objects", ->
-			change_id = ObjectId().toString()
-			comment_id = ObjectId().toString()
-			user_id = ObjectId().toString()
-			thread_id = ObjectId().toString()
-			ts = new Date().toJSON()
-			@RangeManager.jsonRangesToMongo({
-				changes: [{
-					id: change_id
-					op: { i: "foo", p: 3 }
-					metadata:
-						user_id: user_id
-						ts: ts
-				}]
-				comments: [{
-					id: comment_id
-					op: { c: "foo", p: 3, t: thread_id }
-				}]
-			}).should.deep.equal {
-				changes: [{
-					id: ObjectId(change_id)
-					op: { i: "foo", p: 3 }
-					metadata:
-						user_id: ObjectId(user_id)
-						ts: new Date(ts)
-				}]
-				comments: [{
-					id: ObjectId(comment_id)
-					op: { c: "foo", p: 3, t: ObjectId(thread_id) }
-				}]
-			}
-		
-		it "should leave malformed ObjectIds as they are", ->
-			change_id = "foo"
-			comment_id = "bar"
-			user_id = "baz"
-			@RangeManager.jsonRangesToMongo({
-				changes: [{
-					id: change_id
-					metadata:
-						user_id: user_id
-				}]
-				comments: [{
-					id: comment_id
-				}]
-			}).should.deep.equal {
-				changes: [{
-					id: change_id
-					metadata:
-						user_id: user_id
-				}]
-				comments: [{
-					id: comment_id
-				}]
-			}
-		
-		it "should be consistent when transformed through json -> mongo -> json", ->
-			change_id = ObjectId().toString()
-			comment_id = ObjectId().toString()
-			user_id = ObjectId().toString()
-			thread_id = ObjectId().toString()
-			ts = new Date().toJSON()
-			ranges1 = {
-				changes: [{
-					id: change_id
-					op: { i: "foo", p: 3 }
-					metadata:
-						user_id: user_id
-						ts: ts
-				}]
-				comments: [{
-					id: comment_id
-					op: { c: "foo", p: 3, t: thread_id }
-				}]
-			}
-			ranges1_copy = JSON.parse(JSON.stringify(ranges1)) # jsonRangesToMongo modifies in place
-			ranges2 = JSON.parse(JSON.stringify(@RangeManager.jsonRangesToMongo(ranges1_copy)))
-			ranges1.should.deep.equal ranges2
-			
-	describe "shouldUpdateRanges", ->
-		beforeEach () ->
-			@ranges = {
-				changes: [{
-					id: ObjectId()
-					op: { i: "foo", p: 3 }
-					metadata:
-						user_id: ObjectId()
-						ts: new Date()
-				}]
-				comments: [{
-					id: ObjectId()
-					op: { c: "foo", p: 3, t: ObjectId() }
-				}]
-			}
-			@ranges_copy = @RangeManager.jsonRangesToMongo(JSON.parse(JSON.stringify(@ranges)))
-
-		describe "with a blank new range", ->
-			it "should throw an error", ->
-				expect(() =>
-					@RangeManager.shouldUpdateRanges(@ranges, null)
-				).to.throw(Error)
-		
-		describe "with a blank old range", ->
-			it "should treat it like {}", ->
-				@RangeManager.shouldUpdateRanges(null, {}).should.equal false
-				@RangeManager.shouldUpdateRanges(null, @ranges).should.equal true
-		
-		describe "with no changes", ->
-			it "should return false", ->
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal false
-		
-		describe "with changes", ->
-			it "should return true when the change id changes", ->
-				@ranges_copy.changes[0].id = ObjectId()
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the change user id changes", ->
-				@ranges_copy.changes[0].metadata.user_id = ObjectId()
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the change ts changes", ->
-				@ranges_copy.changes[0].metadata.ts = new Date(Date.now() + 1000)
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the change op changes", ->
-				@ranges_copy.changes[0].op.i = "bar"
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the comment id changes", ->
-				@ranges_copy.comments[0].id = ObjectId()
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the comment offset changes", ->
-				@ranges_copy.comments[0].op.p = 17
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true
-
-			it "should return true when the comment content changes", ->
-				@ranges_copy.comments[0].op.c = "bar"
-				@RangeManager.shouldUpdateRanges(@ranges, @ranges_copy).should.equal true

services/docstore/test/unit/js/DocArchiveManagerTests.js

@@ -0,0 +1,577 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { assert } = require('chai')
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../app/js/DocArchiveManager.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongojs')
+const Errors = require('../../../app/js/Errors')
+const crypto = require('crypto')
+
+describe('DocArchiveManager', function() {
+  beforeEach(function() {
+    this.settings = {
+      docstore: {
+        s3: {
+          secret: 'secret',
+          key: 'this_key',
+          bucket: 'doc-archive-unit-test'
+        }
+      }
+    }
+
+    this.request = {
+      put: {},
+      get: {},
+      del: {}
+    }
+
+    this.archivedDocs = [
+      {
+        _id: ObjectId(),
+        inS3: true,
+        rev: 2
+      },
+      {
+        _id: ObjectId(),
+        inS3: true,
+        rev: 4
+      },
+      {
+        _id: ObjectId(),
+        inS3: true,
+        rev: 6
+      }
+    ]
+
+    this.mongoDocs = [
+      {
+        _id: ObjectId(),
+        lines: ['one', 'two', 'three'],
+        rev: 2
+      },
+      {
+        _id: ObjectId(),
+        lines: ['aaa', 'bbb', 'ccc'],
+        rev: 4
+      },
+      {
+        _id: ObjectId(),
+        inS3: true,
+        rev: 6
+      },
+      {
+        _id: ObjectId(),
+        inS3: true,
+        rev: 6
+      },
+      {
+        _id: ObjectId(),
+        lines: ['111', '222', '333'],
+        rev: 6
+      }
+    ]
+
+    this.unarchivedDocs = [
+      {
+        _id: ObjectId(),
+        lines: ['wombat', 'potato', 'banana'],
+        rev: 2
+      },
+      {
+        _id: ObjectId(),
+        lines: ['llama', 'turnip', 'apple'],
+        rev: 4
+      },
+      {
+        _id: ObjectId(),
+        lines: ['elephant', 'swede', 'nectarine'],
+        rev: 6
+      }
+    ]
+
+    this.mixedDocs = this.archivedDocs.concat(this.unarchivedDocs)
+
+    this.MongoManager = {
+      markDocAsArchived: sinon.stub().callsArgWith(2, null),
+      upsertIntoDocCollection: sinon.stub().callsArgWith(3, null),
+      getProjectsDocs: sinon.stub().callsArgWith(3, null, this.mongoDocs),
+      getArchivedProjectDocs: sinon.stub().callsArgWith(2, null, this.mongoDocs)
+    }
+
+    this.requires = {
+      'settings-sharelatex': this.settings,
+      './MongoManager': this.MongoManager,
+      request: this.request,
+      './RangeManager': (this.RangeManager = {}),
+      'logger-sharelatex': {
+        log() {},
+        err() {}
+      }
+    }
+    this.globals = { JSON }
+
+    this.error = 'my errror'
+    this.project_id = ObjectId().toString()
+    this.stubbedError = new Errors.NotFoundError('Error in S3 request')
+    return (this.DocArchiveManager = SandboxedModule.require(modulePath, {
+      requires: this.requires,
+      globals: this.globals
+    }))
+  })
+
+  describe('archiveDoc', function() {
+    it('should use correct options', function(done) {
+      this.request.put = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 200, headers: { etag: '' } })
+      return this.DocArchiveManager.archiveDoc(
+        this.project_id,
+        this.mongoDocs[0],
+        err => {
+          const opts = this.request.put.args[0][0]
+          assert.deepEqual(opts.aws, {
+            key: this.settings.docstore.s3.key,
+            secret: this.settings.docstore.s3.secret,
+            bucket: this.settings.docstore.s3.bucket
+          })
+          opts.body.should.equal(
+            JSON.stringify({
+              lines: this.mongoDocs[0].lines,
+              ranges: this.mongoDocs[0].ranges,
+              schema_v: 1
+            })
+          )
+          opts.timeout.should.equal(30 * 1000)
+          opts.uri.should.equal(
+            `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
+          )
+          return done()
+        }
+      )
+    })
+
+    it('should return no md5 error', function(done) {
+      const data = JSON.stringify({
+        lines: this.mongoDocs[0].lines,
+        ranges: this.mongoDocs[0].ranges,
+        schema_v: 1
+      })
+      this.md5 = crypto
+        .createHash('md5')
+        .update(data)
+        .digest('hex')
+      this.request.put = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 200, headers: { etag: this.md5 } })
+      return this.DocArchiveManager.archiveDoc(
+        this.project_id,
+        this.mongoDocs[0],
+        err => {
+          should.not.exist(err)
+          return done()
+        }
+      )
+    })
+
+    return it('should return the error', function(done) {
+      this.request.put = sinon.stub().callsArgWith(1, this.stubbedError, {
+        statusCode: 400,
+        headers: { etag: '' }
+      })
+      return this.DocArchiveManager.archiveDoc(
+        this.project_id,
+        this.mongoDocs[0],
+        err => {
+          should.exist(err)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('unarchiveDoc', function() {
+    it('should use correct options', function(done) {
+      this.request.get = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 200 }, this.mongoDocs[0].lines)
+      this.request.del = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 204 }, {})
+      return this.DocArchiveManager.unarchiveDoc(
+        this.project_id,
+        this.mongoDocs[0]._id,
+        err => {
+          const opts = this.request.get.args[0][0]
+          assert.deepEqual(opts.aws, {
+            key: this.settings.docstore.s3.key,
+            secret: this.settings.docstore.s3.secret,
+            bucket: this.settings.docstore.s3.bucket
+          })
+          opts.json.should.equal(true)
+          opts.timeout.should.equal(30 * 1000)
+          opts.uri.should.equal(
+            `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
+          )
+          return done()
+        }
+      )
+    })
+
+    it('should return the error', function(done) {
+      this.request.get = sinon.stub().callsArgWith(1, this.stubbedError, {}, {})
+      return this.DocArchiveManager.unarchiveDoc(
+        this.project_id,
+        this.mongoDocs[0],
+        err => {
+          should.exist(err)
+          return done()
+        }
+      )
+    })
+
+    return it('should error if the doc lines are a string not an array', function(done) {
+      this.request.get = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 200 }, 'this is a string')
+      this.request.del = sinon.stub()
+      return this.DocArchiveManager.unarchiveDoc(
+        this.project_id,
+        this.mongoDocs[0],
+        err => {
+          should.exist(err)
+          this.request.del.called.should.equal(false)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('archiveAllDocs', function() {
+    it('should archive all project docs which are not in s3', function(done) {
+      this.MongoManager.getProjectsDocs = sinon
+        .stub()
+        .callsArgWith(3, null, this.mongoDocs)
+      this.DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)
+
+      return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
+        this.DocArchiveManager.archiveDoc
+          .calledWith(this.project_id, this.mongoDocs[0])
+          .should.equal(true)
+        this.DocArchiveManager.archiveDoc
+          .calledWith(this.project_id, this.mongoDocs[1])
+          .should.equal(true)
+        this.DocArchiveManager.archiveDoc
+          .calledWith(this.project_id, this.mongoDocs[4])
+          .should.equal(true)
+
+        this.DocArchiveManager.archiveDoc
+          .calledWith(this.project_id, this.mongoDocs[2])
+          .should.equal(false)
+        this.DocArchiveManager.archiveDoc
+          .calledWith(this.project_id, this.mongoDocs[3])
+          .should.equal(false)
+
+        should.not.exist(err)
+        return done()
+      })
+    })
+
+    it('should return error if have no docs', function(done) {
+      this.MongoManager.getProjectsDocs = sinon
+        .stub()
+        .callsArgWith(3, null, null)
+
+      return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
+        should.exist(err)
+        return done()
+      })
+    })
+
+    it('should return the error', function(done) {
+      this.MongoManager.getProjectsDocs = sinon
+        .stub()
+        .callsArgWith(3, this.error, null)
+
+      return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
+        err.should.equal(this.error)
+        return done()
+      })
+    })
+
+    return describe('when most have been already put in s3', function() {
+      beforeEach(function() {
+        let numberOfDocs = 10 * 1000
+        this.mongoDocs = []
+        while (--numberOfDocs !== 0) {
+          this.mongoDocs.push({ inS3: true, _id: ObjectId() })
+        }
+
+        this.MongoManager.getProjectsDocs = sinon
+          .stub()
+          .callsArgWith(3, null, this.mongoDocs)
+        return (this.DocArchiveManager.archiveDoc = sinon
+          .stub()
+          .callsArgWith(2, null))
+      })
+
+      return it('should not throw and error', function(done) {
+        return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
+          should.not.exist(err)
+          return done()
+        })
+      })
+    })
+  })
+
+  describe('unArchiveAllDocs', function() {
+    it('should unarchive all inS3 docs', function(done) {
+      this.MongoManager.getArchivedProjectDocs = sinon
+        .stub()
+        .callsArgWith(1, null, this.archivedDocs)
+      this.DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null)
+      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
+        for (const doc of Array.from(this.archivedDocs)) {
+          this.DocArchiveManager.unarchiveDoc
+            .calledWith(this.project_id, doc._id)
+            .should.equal(true)
+        }
+        should.not.exist(err)
+        return done()
+      })
+    })
+
+    it('should return error if have no docs', function(done) {
+      this.MongoManager.getArchivedProjectDocs = sinon
+        .stub()
+        .callsArgWith(1, null, null)
+      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
+        should.exist(err)
+        return done()
+      })
+    })
+
+    return it('should return the error', function(done) {
+      this.MongoManager.getArchivedProjectDocs = sinon
+        .stub()
+        .callsArgWith(1, this.error, null)
+      return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
+        err.should.equal(this.error)
+        return done()
+      })
+    })
+  })
+
+  describe('destroyAllDocs', function() {
+    beforeEach(function() {
+      this.request.del = sinon
+        .stub()
+        .callsArgWith(1, null, { statusCode: 204 }, {})
+      this.MongoManager.getProjectsDocs = sinon
+        .stub()
+        .callsArgWith(3, null, this.mixedDocs)
+      this.MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
+      this.MongoManager.destroyDoc = sinon.stub().yields()
+      return Array.from(this.mixedDocs).map(doc =>
+        this.MongoManager.findDoc
+          .withArgs(this.project_id, doc._id)
+          .callsArgWith(3, null, doc)
+      )
+    })
+
+    it('should destroy all the docs', function(done) {
+      this.DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
+      return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
+        for (const doc of Array.from(this.mixedDocs)) {
+          this.DocArchiveManager.destroyDoc
+            .calledWith(this.project_id, doc._id)
+            .should.equal(true)
+        }
+        should.not.exist(err)
+        return done()
+      })
+    })
+
+    it('should only the s3 docs from s3', function(done) {
+      const docOpts = doc => {
+        return JSON.parse(
+          JSON.stringify({
+            aws: {
+              key: this.settings.docstore.s3.key,
+              secret: this.settings.docstore.s3.secret,
+              bucket: this.settings.docstore.s3.bucket
+            },
+            json: true,
+            timeout: 30 * 1000,
+            uri: `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${doc._id}`
+          })
+        )
+      }
+
+      return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
+        let doc
+        expect(err).not.to.exist
+
+        for (doc of Array.from(this.archivedDocs)) {
+          sinon.assert.calledWith(this.request.del, docOpts(doc))
+        }
+        for (doc of Array.from(this.unarchivedDocs)) {
+          expect(this.request.del.calledWith(docOpts(doc))).to.equal(false)
+        } // no notCalledWith
+
+        return done()
+      })
+    })
+
+    return it('should remove the docs from mongo', function(done) {
+      this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
+        return expect(err).not.to.exist
+      })
+
+      for (const doc of Array.from(this.mixedDocs)) {
+        sinon.assert.calledWith(this.MongoManager.destroyDoc, doc._id)
+      }
+
+      return done()
+    })
+  })
+
+  describe('_s3DocToMongoDoc', function() {
+    describe('with the old schema', function() {
+      return it('should return the docs lines', function(done) {
+        return this.DocArchiveManager._s3DocToMongoDoc(
+          ['doc', 'lines'],
+          (error, doc) => {
+            expect(doc).to.deep.equal({
+              lines: ['doc', 'lines']
+            })
+            return done()
+          }
+        )
+      })
+    })
+
+    describe('with the new schema', function() {
+      it('should return the doc lines and ranges', function(done) {
+        this.RangeManager.jsonRangesToMongo = sinon
+          .stub()
+          .returns({ mongo: 'ranges' })
+        return this.DocArchiveManager._s3DocToMongoDoc(
+          {
+            lines: ['doc', 'lines'],
+            ranges: { json: 'ranges' },
+            schema_v: 1
+          },
+          (error, doc) => {
+            expect(doc).to.deep.equal({
+              lines: ['doc', 'lines'],
+              ranges: { mongo: 'ranges' }
+            })
+            return done()
+          }
+        )
+      })
+
+      return it('should return just the doc lines when there are no ranges', function(done) {
+        return this.DocArchiveManager._s3DocToMongoDoc(
+          {
+            lines: ['doc', 'lines'],
+            schema_v: 1
+          },
+          (error, doc) => {
+            expect(doc).to.deep.equal({
+              lines: ['doc', 'lines']
+            })
+            return done()
+          }
+        )
+      })
+    })
+
+    return describe('with an unrecognised schema', function() {
+      return it('should return an error', function(done) {
+        return this.DocArchiveManager._s3DocToMongoDoc(
+          {
+            schema_v: 2
+          },
+          (error, doc) => {
+            expect(error).to.exist
+            return done()
+          }
+        )
+      })
+    })
+  })
+
+  return describe('_mongoDocToS3Doc', function() {
+    describe('with a valid doc', function() {
+      return it('should return the json version', function(done) {
+        let doc
+        return this.DocArchiveManager._mongoDocToS3Doc(
+          (doc = {
+            lines: ['doc', 'lines'],
+            ranges: { mock: 'ranges' }
+          }),
+          (err, s3_doc) => {
+            expect(s3_doc).to.equal(
+              JSON.stringify({
+                lines: ['doc', 'lines'],
+                ranges: { mock: 'ranges' },
+                schema_v: 1
+              })
+            )
+            return done()
+          }
+        )
+      })
+    })
+
+    describe('with null bytes in the result', function() {
+      beforeEach(function() {
+        this._stringify = JSON.stringify
+        return (JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}'))
+      })
+
+      afterEach(function() {
+        return (JSON.stringify = this._stringify)
+      })
+
+      return it('should return an error', function(done) {
+        return this.DocArchiveManager._mongoDocToS3Doc(
+          {
+            lines: ['doc', 'lines'],
+            ranges: { mock: 'ranges' }
+          },
+          (err, s3_doc) => {
+            expect(err).to.exist
+            return done()
+          }
+        )
+      })
+    })
+
+    return describe('without doc lines', function() {
+      return it('should return an error', function(done) {
+        return this.DocArchiveManager._mongoDocToS3Doc({}, (err, s3_doc) => {
+          expect(err).to.exist
+          return done()
+        })
+      })
+    })
+  })
+})

services/docstore/test/unit/js/DocManagerTests.js

@@ -0,0 +1,807 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-dupe-keys,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+const chai = require('chai')
+const { assert } = require('chai')
+chai.should()
+const { expect } = chai
+const modulePath = require('path').join(__dirname, '../../../app/js/DocManager')
+const { ObjectId } = require('mongojs')
+const Errors = require('../../../app/js/Errors')
+
+describe('DocManager', function() {
+  beforeEach(function() {
+    this.DocManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './MongoManager': (this.MongoManager = {}),
+        './DocArchiveManager': (this.DocArchiveManager = {}),
+        './RangeManager': (this.RangeManager = {
+          jsonRangesToMongo(r) {
+            return r
+          },
+          shouldUpdateRanges: sinon.stub().returns(false)
+        }),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          warn() {},
+          err() {}
+        })
+      }
+    })
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+    this.another_project_id = ObjectId().toString()
+    this.callback = sinon.stub()
+    return (this.stubbedError = new Error('blew up'))
+  })
+
+  describe('checkDocExists', function() {
+    beforeEach(function() {
+      return (this.DocManager._getDoc = sinon.stub())
+    })
+
+    it('should call get doc with a quick filter', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, null, { _id: this.doc_id })
+      return this.DocManager.checkDocExists(
+        this.project_id,
+        this.doc_id,
+        (err, exist) => {
+          exist.should.equal(true)
+          this.DocManager._getDoc
+            .calledWith(this.project_id, this.doc_id, { _id: 1, inS3: true })
+            .should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    it('should return false when doc is not there', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, null)
+      return this.DocManager.checkDocExists(
+        this.project_id,
+        this.doc_id,
+        (err, exist) => {
+          exist.should.equal(false)
+          return done()
+        }
+      )
+    })
+
+    return it('should return error when get doc errors', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, 'error')
+      return this.DocManager.checkDocExists(
+        this.project_id,
+        this.doc_id,
+        (err, exist) => {
+          err.should.equal('error')
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('getFullDoc', function() {
+    beforeEach(function() {
+      this.DocManager._getDoc = sinon.stub()
+      return (this.doc = {
+        _id: this.doc_id,
+        lines: ['2134']
+      })
+    })
+
+    it('should call get doc with a quick filter', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, null, this.doc)
+      return this.DocManager.getFullDoc(
+        this.project_id,
+        this.doc_id,
+        (err, doc) => {
+          doc.should.equal(this.doc)
+          this.DocManager._getDoc
+            .calledWith(this.project_id, this.doc_id, {
+              lines: true,
+              rev: true,
+              deleted: true,
+              version: true,
+              ranges: true,
+              inS3: true
+            })
+            .should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    return it('should return error when get doc errors', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, 'error')
+      return this.DocManager.getFullDoc(
+        this.project_id,
+        this.doc_id,
+        (err, exist) => {
+          err.should.equal('error')
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('getRawDoc', function() {
+    beforeEach(function() {
+      this.DocManager._getDoc = sinon.stub()
+      return (this.doc = { lines: ['2134'] })
+    })
+
+    it('should call get doc with a quick filter', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, null, this.doc)
+      return this.DocManager.getDocLines(
+        this.project_id,
+        this.doc_id,
+        (err, doc) => {
+          doc.should.equal(this.doc)
+          this.DocManager._getDoc
+            .calledWith(this.project_id, this.doc_id, {
+              lines: true,
+              inS3: true
+            })
+            .should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    return it('should return error when get doc errors', function(done) {
+      this.DocManager._getDoc.callsArgWith(3, 'error')
+      return this.DocManager.getDocLines(
+        this.project_id,
+        this.doc_id,
+        (err, exist) => {
+          err.should.equal('error')
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('getDoc', function() {
+    beforeEach(function() {
+      this.project = { name: 'mock-project' }
+      this.doc = {
+        _id: this.doc_id,
+        project_id: this.project_id,
+        lines: ['mock-lines']
+      }
+      this.version = 42
+      this.MongoManager.findDoc = sinon.stub()
+      return (this.MongoManager.getDocVersion = sinon
+        .stub()
+        .yields(null, this.version))
+    })
+
+    describe('when using a filter', function() {
+      beforeEach(function() {
+        return this.MongoManager.findDoc.yields(null, this.doc)
+      })
+
+      it('should error if inS3 is not set to true', function(done) {
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { inS3: false },
+          err => {
+            expect(err).to.exist
+            return done()
+          }
+        )
+      })
+
+      it('should always get inS3 even when no filter is passed', function(done) {
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          undefined,
+          err => {
+            this.MongoManager.findDoc.called.should.equal(false)
+            expect(err).to.exist
+            return done()
+          }
+        )
+      })
+
+      return it('should not error if inS3 is set to true', function(done) {
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { inS3: true },
+          err => {
+            expect(err).to.not.exist
+            return done()
+          }
+        )
+      })
+    })
+
+    describe('when the doc is in the doc collection', function() {
+      beforeEach(function() {
+        this.MongoManager.findDoc.yields(null, this.doc)
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { version: true, inS3: true },
+          this.callback
+        )
+      })
+
+      it('should get the doc from the doc collection', function() {
+        return this.MongoManager.findDoc
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should get the doc version from the docOps collection', function() {
+        return this.MongoManager.getDocVersion
+          .calledWith(this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should return the callback with the doc with the version', function() {
+        this.callback.called.should.equal(true)
+        const doc = this.callback.args[0][1]
+        doc.lines.should.equal(this.doc.lines)
+        return doc.version.should.equal(this.version)
+      })
+    })
+
+    describe('without the version filter', function() {
+      beforeEach(function() {
+        this.MongoManager.findDoc.yields(null, this.doc)
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { version: false, inS3: true },
+          this.callback
+        )
+      })
+
+      return it('should not get the doc version from the docOps collection', function() {
+        return this.MongoManager.getDocVersion.called.should.equal(false)
+      })
+    })
+
+    describe('when MongoManager.findDoc errors', function() {
+      beforeEach(function() {
+        this.MongoManager.findDoc.yields(this.stubbedError)
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { version: true, inS3: true },
+          this.callback
+        )
+      })
+
+      return it('should return the error', function() {
+        return this.callback.calledWith(this.stubbedError).should.equal(true)
+      })
+    })
+
+    describe('when the doc is archived', function() {
+      beforeEach(function() {
+        this.doc = {
+          _id: this.doc_id,
+          project_id: this.project_id,
+          lines: ['mock-lines'],
+          inS3: true
+        }
+        this.MongoManager.findDoc.yields(null, this.doc)
+        this.DocArchiveManager.unarchiveDoc = (
+          project_id,
+          doc_id,
+          callback
+        ) => {
+          this.doc.inS3 = false
+          return callback()
+        }
+        sinon.spy(this.DocArchiveManager, 'unarchiveDoc')
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { version: true, inS3: true },
+          this.callback
+        )
+      })
+
+      it('should call the DocArchive to unarchive the doc', function() {
+        return this.DocArchiveManager.unarchiveDoc
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should look up the doc twice', function() {
+        return this.MongoManager.findDoc.calledTwice.should.equal(true)
+      })
+
+      return it('should return the doc', function() {
+        return this.callback.calledWith(null, this.doc).should.equal(true)
+      })
+    })
+
+    return describe('when the doc does not exist in the docs collection', function() {
+      beforeEach(function() {
+        this.MongoManager.findDoc = sinon.stub().yields(null, null)
+        return this.DocManager._getDoc(
+          this.project_id,
+          this.doc_id,
+          { version: true, inS3: true },
+          this.callback
+        )
+      })
+
+      return it('should return a NotFoundError', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has(
+              'message',
+              `No such doc: ${this.doc_id} in project ${this.project_id}`
+            )
+          )
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('getAllNonDeletedDocs', function() {
+    describe('when the project exists', function() {
+      beforeEach(function() {
+        this.docs = [
+          {
+            _id: this.doc_id,
+            project_id: this.project_id,
+            lines: ['mock-lines']
+          }
+        ]
+        this.MongoManager.getProjectsDocs = sinon
+          .stub()
+          .callsArgWith(3, null, this.docs)
+        this.DocArchiveManager.unArchiveAllDocs = sinon
+          .stub()
+          .callsArgWith(1, null, this.docs)
+        this.filter = { lines: true }
+        return this.DocManager.getAllNonDeletedDocs(
+          this.project_id,
+          this.filter,
+          this.callback
+        )
+      })
+
+      it('should get the project from the database', function() {
+        return this.MongoManager.getProjectsDocs
+          .calledWith(this.project_id, { include_deleted: false }, this.filter)
+          .should.equal(true)
+      })
+
+      return it('should return the docs', function() {
+        return this.callback.calledWith(null, this.docs).should.equal(true)
+      })
+    })
+
+    return describe('when there are no docs for the project', function() {
+      beforeEach(function() {
+        this.MongoManager.getProjectsDocs = sinon
+          .stub()
+          .callsArgWith(3, null, null)
+        this.DocArchiveManager.unArchiveAllDocs = sinon
+          .stub()
+          .callsArgWith(1, null)
+        return this.DocManager.getAllNonDeletedDocs(
+          this.project_id,
+          this.filter,
+          this.callback
+        )
+      })
+
+      return it('should return a NotFoundError', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has('message', `No docs for project ${this.project_id}`)
+          )
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('deleteDoc', function() {
+    describe('when the doc exists', function() {
+      beforeEach(function() {
+        this.lines = ['mock', 'doc', 'lines']
+        this.rev = 77
+        this.DocManager.checkDocExists = sinon
+          .stub()
+          .callsArgWith(2, null, true)
+        this.MongoManager.markDocAsDeleted = sinon.stub().callsArg(2)
+        return this.DocManager.deleteDoc(
+          this.project_id,
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      it('should get the doc', function() {
+        return this.DocManager.checkDocExists
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should mark doc as deleted', function() {
+        return this.MongoManager.markDocAsDeleted
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should return the callback', function() {
+        return this.callback.called.should.equal(true)
+      })
+    })
+
+    return describe('when the doc does not exist', function() {
+      beforeEach(function() {
+        this.DocManager.checkDocExists = sinon
+          .stub()
+          .callsArgWith(2, null, false)
+        return this.DocManager.deleteDoc(
+          this.project_id,
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      return it('should return a NotFoundError', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has(
+              'message',
+              `No such project/doc to delete: ${this.project_id}/${this.doc_id}`
+            )
+          )
+          .should.equal(true)
+      })
+    })
+  })
+
+  return describe('updateDoc', function() {
+    beforeEach(function() {
+      this.oldDocLines = ['old', 'doc', 'lines']
+      this.newDocLines = ['new', 'doc', 'lines']
+      this.originalRanges = {
+        changes: [
+          {
+            id: ObjectId().toString(),
+            op: { i: 'foo', p: 3 },
+            meta: {
+              user_id: ObjectId().toString(),
+              ts: new Date().toString()
+            }
+          }
+        ]
+      }
+      this.newRanges = {
+        changes: [
+          {
+            id: ObjectId().toString(),
+            op: { i: 'bar', p: 6 },
+            meta: {
+              user_id: ObjectId().toString(),
+              ts: new Date().toString()
+            }
+          }
+        ]
+      }
+      this.version = 42
+      this.doc = {
+        _id: this.doc_id,
+        project_id: this.project_id,
+        lines: this.oldDocLines,
+        rev: (this.rev = 5),
+        version: this.version,
+        ranges: this.originalRanges
+      }
+
+      this.MongoManager.upsertIntoDocCollection = sinon.stub().callsArg(3)
+      this.MongoManager.setDocVersion = sinon.stub().yields()
+      return (this.DocManager._getDoc = sinon.stub())
+    })
+
+    describe('when only the doc lines have changed', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.newDocLines,
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should get the existing doc', function() {
+        return this.DocManager._getDoc
+          .calledWith(this.project_id, this.doc_id, {
+            version: true,
+            rev: true,
+            lines: true,
+            version: true,
+            ranges: true,
+            inS3: true
+          })
+          .should.equal(true)
+      })
+
+      it('should upsert the document to the doc collection', function() {
+        return this.MongoManager.upsertIntoDocCollection
+          .calledWith(this.project_id, this.doc_id, { lines: this.newDocLines })
+          .should.equal(true)
+      })
+
+      it('should not update the version', function() {
+        return this.MongoManager.setDocVersion.called.should.equal(false)
+      })
+
+      return it('should return the callback with the new rev', function() {
+        return this.callback
+          .calledWith(null, true, this.rev + 1)
+          .should.equal(true)
+      })
+    })
+
+    describe('when the doc ranges have changed', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+        this.RangeManager.shouldUpdateRanges.returns(true)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.oldDocLines,
+          this.version,
+          this.newRanges,
+          this.callback
+        )
+      })
+
+      it('should upsert the ranges', function() {
+        return this.MongoManager.upsertIntoDocCollection
+          .calledWith(this.project_id, this.doc_id, { ranges: this.newRanges })
+          .should.equal(true)
+      })
+
+      it('should not update the version', function() {
+        return this.MongoManager.setDocVersion.called.should.equal(false)
+      })
+
+      return it('should return the callback with the new rev', function() {
+        return this.callback
+          .calledWith(null, true, this.rev + 1)
+          .should.equal(true)
+      })
+    })
+
+    describe('when only the version has changed', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.oldDocLines,
+          this.version + 1,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should not change the lines or ranges', function() {
+        return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+          false
+        )
+      })
+
+      it('should update the version', function() {
+        return this.MongoManager.setDocVersion
+          .calledWith(this.doc_id, this.version + 1)
+          .should.equal(true)
+      })
+
+      return it('should return the callback with the old rev', function() {
+        return this.callback.calledWith(null, true, this.rev).should.equal(true)
+      })
+    })
+
+    describe('when the doc has not changed at all', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.oldDocLines,
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should not update the ranges or lines', function() {
+        return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+          false
+        )
+      })
+
+      it('should not update the version', function() {
+        return this.MongoManager.setDocVersion.called.should.equal(false)
+      })
+
+      return it('should return the callback with the old rev and modified == false', function() {
+        return this.callback
+          .calledWith(null, false, this.rev)
+          .should.equal(true)
+      })
+    })
+
+    describe('when the version is null', function() {
+      beforeEach(function() {
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.newDocLines,
+          null,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      return it('should return an error', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has('message', 'no lines, version or ranges provided')
+          )
+          .should.equal(true)
+      })
+    })
+
+    describe('when the lines are null', function() {
+      beforeEach(function() {
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          null,
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      return it('should return an error', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has('message', 'no lines, version or ranges provided')
+          )
+          .should.equal(true)
+      })
+    })
+
+    describe('when the ranges are null', function() {
+      beforeEach(function() {
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.newDocLines,
+          this.version,
+          null,
+          this.callback
+        )
+      })
+
+      return it('should return an error', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has('message', 'no lines, version or ranges provided')
+          )
+          .should.equal(true)
+      })
+    })
+
+    describe('when there is a generic error getting the doc', function() {
+      beforeEach(function() {
+        this.error = new Error('doc could not be found')
+        this.DocManager._getDoc = sinon
+          .stub()
+          .callsArgWith(3, this.error, null, null)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.newDocLines,
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should not upsert the document to the doc collection', function() {
+        return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+          false
+        )
+      })
+
+      return it('should return the callback with the error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+
+    describe('when the doc lines have not changed', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.oldDocLines.slice(),
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should not update the doc', function() {
+        return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+          false
+        )
+      })
+
+      return it('should return the callback with the existing rev', function() {
+        return this.callback
+          .calledWith(null, false, this.rev)
+          .should.equal(true)
+      })
+    })
+
+    return describe('when the doc does not exist', function() {
+      beforeEach(function() {
+        this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, null, null)
+        return this.DocManager.updateDoc(
+          this.project_id,
+          this.doc_id,
+          this.newDocLines,
+          this.version,
+          this.originalRanges,
+          this.callback
+        )
+      })
+
+      it('should upsert the document to the doc collection', function() {
+        return this.MongoManager.upsertIntoDocCollection
+          .calledWith(this.project_id, this.doc_id, {
+            lines: this.newDocLines,
+            ranges: this.originalRanges
+          })
+          .should.equal(true)
+      })
+
+      it('should set the version', function() {
+        return this.MongoManager.setDocVersion
+          .calledWith(this.doc_id, this.version)
+          .should.equal(true)
+      })
+
+      return it('should return the callback with the new rev', function() {
+        return this.callback.calledWith(null, true, 1).should.equal(true)
+      })
+    })
+  })
+})

services/docstore/test/unit/js/HttpControllerTests.js

@@ -0,0 +1,489 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const { assert } = require('chai')
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const modulePath = require('path').join(
+  __dirname,
+  '../../../app/js/HttpController'
+)
+const { ObjectId } = require('mongojs')
+
+describe('HttpController', function() {
+  beforeEach(function() {
+    this.HttpController = SandboxedModule.require(modulePath, {
+      requires: {
+        './DocManager': (this.DocManager = {}),
+        './DocArchiveManager': (this.DocArchiveManager = {}),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          error: sinon.stub()
+        }),
+        './HealthChecker': {}
+      }
+    })
+    this.res = {
+      send: sinon.stub(),
+      json: sinon.stub(),
+      setHeader: sinon.stub()
+    }
+    this.res.status = sinon.stub().returns(this.res)
+    this.req = { query: {} }
+    this.next = sinon.stub()
+    this.project_id = 'mock-project-id'
+    this.doc_id = 'mock-doc-id'
+    this.doc = {
+      _id: this.doc_id,
+      lines: ['mock', 'lines', ' here', '', '', ' spaces '],
+      version: 42,
+      rev: 5
+    }
+    return (this.deletedDoc = {
+      deleted: true,
+      _id: this.doc_id,
+      lines: ['mock', 'lines', ' here', '', '', ' spaces '],
+      version: 42,
+      rev: 5
+    })
+  })
+
+  describe('getDoc', function() {
+    describe('without deleted docs', function() {
+      beforeEach(function() {
+        this.req.params = {
+          project_id: this.project_id,
+          doc_id: this.doc_id
+        }
+        this.DocManager.getFullDoc = sinon
+          .stub()
+          .callsArgWith(2, null, this.doc)
+        return this.HttpController.getDoc(this.req, this.res, this.next)
+      })
+
+      it('should get the document with the version (including deleted)', function() {
+        return this.DocManager.getFullDoc
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should return the doc as JSON', function() {
+        return this.res.json
+          .calledWith({
+            _id: this.doc_id,
+            lines: this.doc.lines,
+            rev: this.doc.rev,
+            version: this.doc.version
+          })
+          .should.equal(true)
+      })
+    })
+
+    return describe('which is deleted', function() {
+      beforeEach(function() {
+        this.req.params = {
+          project_id: this.project_id,
+          doc_id: this.doc_id
+        }
+        return (this.DocManager.getFullDoc = sinon
+          .stub()
+          .callsArgWith(2, null, this.deletedDoc))
+      })
+
+      it('should get the doc from the doc manager', function() {
+        this.HttpController.getDoc(this.req, this.res, this.next)
+        return this.DocManager.getFullDoc
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should return 404 if the query string delete is not set ', function() {
+        this.HttpController.getDoc(this.req, this.res, this.next)
+        return this.res.send.calledWith(404).should.equal(true)
+      })
+
+      return it('should return the doc as JSON if include_deleted is set to true', function() {
+        this.req.query.include_deleted = 'true'
+        this.HttpController.getDoc(this.req, this.res, this.next)
+        return this.res.json
+          .calledWith({
+            _id: this.doc_id,
+            lines: this.doc.lines,
+            rev: this.doc.rev,
+            deleted: true,
+            version: this.doc.version
+          })
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('getRawDoc', function() {
+    beforeEach(function() {
+      this.req.params = {
+        project_id: this.project_id,
+        doc_id: this.doc_id
+      }
+      this.DocManager.getDocLines = sinon.stub().callsArgWith(2, null, this.doc)
+      return this.HttpController.getRawDoc(this.req, this.res, this.next)
+    })
+
+    it('should get the document without the version', function() {
+      return this.DocManager.getDocLines
+        .calledWith(this.project_id, this.doc_id)
+        .should.equal(true)
+    })
+
+    it('should set the content type header', function() {
+      return this.res.setHeader
+        .calledWith('content-type', 'text/plain')
+        .should.equal(true)
+    })
+
+    return it('should send the raw version of the doc', function() {
+      return assert.deepEqual(
+        this.res.send.args[0][0],
+        `${this.doc.lines[0]}\n${this.doc.lines[1]}\n${this.doc.lines[2]}\n${this.doc.lines[3]}\n${this.doc.lines[4]}\n${this.doc.lines[5]}`
+      )
+    })
+  })
+
+  describe('getAllDocs', function() {
+    describe('normally', function() {
+      beforeEach(function() {
+        this.req.params = { project_id: this.project_id }
+        this.docs = [
+          {
+            _id: ObjectId(),
+            lines: ['mock', 'lines', 'one'],
+            rev: 2
+          },
+          {
+            _id: ObjectId(),
+            lines: ['mock', 'lines', 'two'],
+            rev: 4
+          }
+        ]
+        this.DocManager.getAllNonDeletedDocs = sinon
+          .stub()
+          .callsArgWith(2, null, this.docs)
+        return this.HttpController.getAllDocs(this.req, this.res, this.next)
+      })
+
+      it('should get all the (non-deleted) docs', function() {
+        return this.DocManager.getAllNonDeletedDocs
+          .calledWith(this.project_id, { lines: true, rev: true })
+          .should.equal(true)
+      })
+
+      return it('should return the doc as JSON', function() {
+        return this.res.json
+          .calledWith([
+            {
+              _id: this.docs[0]._id.toString(),
+              lines: this.docs[0].lines,
+              rev: this.docs[0].rev
+            },
+            {
+              _id: this.docs[1]._id.toString(),
+              lines: this.docs[1].lines,
+              rev: this.docs[1].rev
+            }
+          ])
+          .should.equal(true)
+      })
+    })
+
+    return describe('with a null doc', function() {
+      beforeEach(function() {
+        this.req.params = { project_id: this.project_id }
+        this.docs = [
+          {
+            _id: ObjectId(),
+            lines: ['mock', 'lines', 'one'],
+            rev: 2
+          },
+          null,
+          {
+            _id: ObjectId(),
+            lines: ['mock', 'lines', 'two'],
+            rev: 4
+          }
+        ]
+        this.DocManager.getAllNonDeletedDocs = sinon
+          .stub()
+          .callsArgWith(2, null, this.docs)
+        return this.HttpController.getAllDocs(this.req, this.res, this.next)
+      })
+
+      it('should return the non null docs as JSON', function() {
+        return this.res.json
+          .calledWith([
+            {
+              _id: this.docs[0]._id.toString(),
+              lines: this.docs[0].lines,
+              rev: this.docs[0].rev
+            },
+            {
+              _id: this.docs[2]._id.toString(),
+              lines: this.docs[2].lines,
+              rev: this.docs[2].rev
+            }
+          ])
+          .should.equal(true)
+      })
+
+      return it('should log out an error', function() {
+        return this.logger.error
+          .calledWith(
+            {
+              err: sinon.match.has('message', 'null doc'),
+              project_id: this.project_id
+            },
+            'encountered null doc'
+          )
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('getAllRanges', function() {
+    return describe('normally', function() {
+      beforeEach(function() {
+        this.req.params = { project_id: this.project_id }
+        this.docs = [
+          {
+            _id: ObjectId(),
+            ranges: { mock_ranges: 'one' }
+          },
+          {
+            _id: ObjectId(),
+            ranges: { mock_ranges: 'two' }
+          }
+        ]
+        this.DocManager.getAllNonDeletedDocs = sinon
+          .stub()
+          .callsArgWith(2, null, this.docs)
+        return this.HttpController.getAllRanges(this.req, this.res, this.next)
+      })
+
+      it('should get all the (non-deleted) doc ranges', function() {
+        return this.DocManager.getAllNonDeletedDocs
+          .calledWith(this.project_id, { ranges: true })
+          .should.equal(true)
+      })
+
+      return it('should return the doc as JSON', function() {
+        return this.res.json
+          .calledWith([
+            {
+              _id: this.docs[0]._id.toString(),
+              ranges: this.docs[0].ranges
+            },
+            {
+              _id: this.docs[1]._id.toString(),
+              ranges: this.docs[1].ranges
+            }
+          ])
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('updateDoc', function() {
+    beforeEach(function() {
+      return (this.req.params = {
+        project_id: this.project_id,
+        doc_id: this.doc_id
+      })
+    })
+
+    describe('when the doc lines exist and were updated', function() {
+      beforeEach(function() {
+        this.req.body = {
+          lines: (this.lines = ['hello', 'world']),
+          version: (this.version = 42),
+          ranges: (this.ranges = { changes: 'mock' })
+        }
+        this.DocManager.updateDoc = sinon
+          .stub()
+          .yields(null, true, (this.rev = 5))
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      it('should update the document', function() {
+        return this.DocManager.updateDoc
+          .calledWith(
+            this.project_id,
+            this.doc_id,
+            this.lines,
+            this.version,
+            this.ranges
+          )
+          .should.equal(true)
+      })
+
+      return it('should return a modified status', function() {
+        return this.res.json
+          .calledWith({ modified: true, rev: this.rev })
+          .should.equal(true)
+      })
+    })
+
+    describe('when the doc lines exist and were not updated', function() {
+      beforeEach(function() {
+        this.req.body = {
+          lines: (this.lines = ['hello', 'world']),
+          version: (this.version = 42),
+          ranges: {}
+        }
+        this.DocManager.updateDoc = sinon
+          .stub()
+          .yields(null, false, (this.rev = 5))
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      return it('should return a modified status', function() {
+        return this.res.json
+          .calledWith({ modified: false, rev: this.rev })
+          .should.equal(true)
+      })
+    })
+
+    describe('when the doc lines are not provided', function() {
+      beforeEach(function() {
+        this.req.body = { version: 42, ranges: {} }
+        this.DocManager.updateDoc = sinon.stub().yields(null, false)
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      it('should not update the document', function() {
+        return this.DocManager.updateDoc.called.should.equal(false)
+      })
+
+      return it('should return a 400 (bad request) response', function() {
+        return this.res.send.calledWith(400).should.equal(true)
+      })
+    })
+
+    describe('when the doc version are not provided', function() {
+      beforeEach(function() {
+        this.req.body = { version: 42, lines: ['hello world'] }
+        this.DocManager.updateDoc = sinon.stub().yields(null, false)
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      it('should not update the document', function() {
+        return this.DocManager.updateDoc.called.should.equal(false)
+      })
+
+      return it('should return a 400 (bad request) response', function() {
+        return this.res.send.calledWith(400).should.equal(true)
+      })
+    })
+
+    describe('when the doc ranges is not provided', function() {
+      beforeEach(function() {
+        this.req.body = { lines: ['foo'], version: 42 }
+        this.DocManager.updateDoc = sinon.stub().yields(null, false)
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      it('should not update the document', function() {
+        return this.DocManager.updateDoc.called.should.equal(false)
+      })
+
+      return it('should return a 400 (bad request) response', function() {
+        return this.res.send.calledWith(400).should.equal(true)
+      })
+    })
+
+    return describe('when the doc body is too large', function() {
+      beforeEach(function() {
+        this.req.body = {
+          lines: (this.lines = Array(2049).fill('a'.repeat(1024))),
+          version: (this.version = 42),
+          ranges: (this.ranges = { changes: 'mock' })
+        }
+        return this.HttpController.updateDoc(this.req, this.res, this.next)
+      })
+
+      it('should return a 413 (too large) response', function() {
+        return sinon.assert.calledWith(this.res.status, 413)
+      })
+
+      return it('should report that the document body is too large', function() {
+        return sinon.assert.calledWith(this.res.send, 'document body too large')
+      })
+    })
+  })
+
+  describe('deleteDoc', function() {
+    beforeEach(function() {
+      this.req.params = {
+        project_id: this.project_id,
+        doc_id: this.doc_id
+      }
+      this.DocManager.deleteDoc = sinon.stub().callsArg(2)
+      return this.HttpController.deleteDoc(this.req, this.res, this.next)
+    })
+
+    it('should delete the document', function() {
+      return this.DocManager.deleteDoc
+        .calledWith(this.project_id, this.doc_id)
+        .should.equal(true)
+    })
+
+    return it('should return a 204 (No Content)', function() {
+      return this.res.send.calledWith(204).should.equal(true)
+    })
+  })
+
+  describe('archiveAllDocs', function() {
+    beforeEach(function() {
+      this.req.params = { project_id: this.project_id }
+      this.DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1)
+      return this.HttpController.archiveAllDocs(this.req, this.res, this.next)
+    })
+
+    it('should archive the project', function() {
+      return this.DocArchiveManager.archiveAllDocs
+        .calledWith(this.project_id)
+        .should.equal(true)
+    })
+
+    return it('should return a 204 (No Content)', function() {
+      return this.res.send.calledWith(204).should.equal(true)
+    })
+  })
+
+  return describe('destroyAllDocs', function() {
+    beforeEach(function() {
+      this.req.params = { project_id: this.project_id }
+      this.DocArchiveManager.destroyAllDocs = sinon.stub().callsArg(1)
+      return this.HttpController.destroyAllDocs(this.req, this.res, this.next)
+    })
+
+    it('should destroy the docs', function() {
+      return sinon.assert.calledWith(
+        this.DocArchiveManager.destroyAllDocs,
+        this.project_id
+      )
+    })
+
+    return it('should return 204', function() {
+      return sinon.assert.calledWith(this.res.send, 204)
+    })
+  })
+})

services/docstore/test/unit/js/MongoManagerTests.js

@@ -0,0 +1,293 @@
+/* eslint-disable
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+require('chai').should()
+const modulePath = require('path').join(
+  __dirname,
+  '../../../app/js/MongoManager'
+)
+const { ObjectId } = require('mongojs')
+const { assert } = require('chai')
+
+describe('MongoManager', function() {
+  beforeEach(function() {
+    this.MongoManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './mongojs': {
+          db: (this.db = { docs: {}, docOps: {} }),
+          ObjectId
+        },
+        'metrics-sharelatex': { timeAsyncMethod: sinon.stub() },
+        'logger-sharelatex': { log() {} }
+      }
+    })
+    this.project_id = ObjectId().toString()
+    this.doc_id = ObjectId().toString()
+    this.callback = sinon.stub()
+    return (this.stubbedErr = new Error('hello world'))
+  })
+
+  describe('findDoc', function() {
+    beforeEach(function() {
+      this.doc = { name: 'mock-doc' }
+      this.db.docs.find = sinon.stub().callsArgWith(2, null, [this.doc])
+      this.filter = { lines: true }
+      return this.MongoManager.findDoc(
+        this.project_id,
+        this.doc_id,
+        this.filter,
+        this.callback
+      )
+    })
+
+    it('should find the doc', function() {
+      return this.db.docs.find
+        .calledWith(
+          {
+            _id: ObjectId(this.doc_id),
+            project_id: ObjectId(this.project_id)
+          },
+          this.filter
+        )
+        .should.equal(true)
+    })
+
+    return it('should call the callback with the doc', function() {
+      return this.callback.calledWith(null, this.doc).should.equal(true)
+    })
+  })
+
+  describe('getProjectsDocs', function() {
+    beforeEach(function() {
+      this.filter = { lines: true }
+      this.doc1 = { name: 'mock-doc1' }
+      this.doc2 = { name: 'mock-doc2' }
+      this.doc3 = { name: 'mock-doc3' }
+      this.doc4 = { name: 'mock-doc4' }
+      return (this.db.docs.find = sinon
+        .stub()
+        .callsArgWith(2, null, [this.doc, this.doc3, this.doc4]))
+    })
+
+    describe('with included_deleted = false', function() {
+      beforeEach(function() {
+        return this.MongoManager.getProjectsDocs(
+          this.project_id,
+          { include_deleted: false },
+          this.filter,
+          this.callback
+        )
+      })
+
+      it('should find the non-deleted docs via the project_id', function() {
+        return this.db.docs.find
+          .calledWith(
+            {
+              project_id: ObjectId(this.project_id),
+              deleted: { $ne: true }
+            },
+            this.filter
+          )
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the docs', function() {
+        return this.callback
+          .calledWith(null, [this.doc, this.doc3, this.doc4])
+          .should.equal(true)
+      })
+    })
+
+    return describe('with included_deleted = true', function() {
+      beforeEach(function() {
+        return this.MongoManager.getProjectsDocs(
+          this.project_id,
+          { include_deleted: true },
+          this.filter,
+          this.callback
+        )
+      })
+
+      it('should find all via the project_id', function() {
+        return this.db.docs.find
+          .calledWith(
+            {
+              project_id: ObjectId(this.project_id)
+            },
+            this.filter
+          )
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the docs', function() {
+        return this.callback
+          .calledWith(null, [this.doc, this.doc3, this.doc4])
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('upsertIntoDocCollection', function() {
+    beforeEach(function() {
+      this.db.docs.update = sinon.stub().callsArgWith(3, this.stubbedErr)
+      return (this.oldRev = 77)
+    })
+
+    it('should upsert the document', function(done) {
+      return this.MongoManager.upsertIntoDocCollection(
+        this.project_id,
+        this.doc_id,
+        { lines: this.lines },
+        err => {
+          const args = this.db.docs.update.args[0]
+          assert.deepEqual(args[0], { _id: ObjectId(this.doc_id) })
+          assert.equal(args[1].$set.lines, this.lines)
+          assert.equal(args[1].$inc.rev, 1)
+          assert.deepEqual(args[1].$set.project_id, ObjectId(this.project_id))
+          return done()
+        }
+      )
+    })
+
+    return it('should return the error', function(done) {
+      return this.MongoManager.upsertIntoDocCollection(
+        this.project_id,
+        this.doc_id,
+        { lines: this.lines },
+        err => {
+          err.should.equal(this.stubbedErr)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('markDocAsDeleted', function() {
+    beforeEach(function() {
+      this.db.docs.update = sinon.stub().callsArgWith(2, this.stubbedErr)
+      return (this.oldRev = 77)
+    })
+
+    it('should process the update', function(done) {
+      return this.MongoManager.markDocAsDeleted(
+        this.project_id,
+        this.doc_id,
+        err => {
+          const args = this.db.docs.update.args[0]
+          assert.deepEqual(args[0], {
+            _id: ObjectId(this.doc_id),
+            project_id: ObjectId(this.project_id)
+          })
+          assert.equal(args[1].$set.deleted, true)
+          return done()
+        }
+      )
+    })
+
+    return it('should return the error', function(done) {
+      return this.MongoManager.markDocAsDeleted(
+        this.project_id,
+        this.doc_id,
+        err => {
+          err.should.equal(this.stubbedErr)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('destroyDoc', function() {
+    beforeEach(function(done) {
+      this.db.docs.remove = sinon.stub().yields()
+      this.db.docOps.remove = sinon.stub().yields()
+      return this.MongoManager.destroyDoc('123456789012', done)
+    })
+
+    it('should destroy the doc', function() {
+      return sinon.assert.calledWith(this.db.docs.remove, {
+        _id: ObjectId('123456789012')
+      })
+    })
+
+    return it('should destroy the docOps', function() {
+      return sinon.assert.calledWith(this.db.docOps.remove, {
+        doc_id: ObjectId('123456789012')
+      })
+    })
+  })
+
+  describe('getDocVersion', function() {
+    describe('when the doc exists', function() {
+      beforeEach(function() {
+        this.doc = { version: (this.version = 42) }
+        this.db.docOps.find = sinon.stub().callsArgWith(2, null, [this.doc])
+        return this.MongoManager.getDocVersion(this.doc_id, this.callback)
+      })
+
+      it('should look for the doc in the database', function() {
+        return this.db.docOps.find
+          .calledWith({ doc_id: ObjectId(this.doc_id) }, { version: 1 })
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the version', function() {
+        return this.callback.calledWith(null, this.version).should.equal(true)
+      })
+    })
+
+    return describe("when the doc doesn't exist", function() {
+      beforeEach(function() {
+        this.db.docOps.find = sinon.stub().callsArgWith(2, null, [])
+        return this.MongoManager.getDocVersion(this.doc_id, this.callback)
+      })
+
+      return it('should call the callback with 0', function() {
+        return this.callback.calledWith(null, 0).should.equal(true)
+      })
+    })
+  })
+
+  return describe('setDocVersion', function() {
+    beforeEach(function() {
+      this.version = 42
+      this.db.docOps.update = sinon.stub().callsArg(3)
+      return this.MongoManager.setDocVersion(
+        this.doc_id,
+        this.version,
+        this.callback
+      )
+    })
+
+    it('should update the doc version', function() {
+      return this.db.docOps.update
+        .calledWith(
+          {
+            doc_id: ObjectId(this.doc_id)
+          },
+          {
+            $set: {
+              version: this.version
+            }
+          },
+          {
+            upsert: true
+          }
+        )
+        .should.equal(true)
+    })
+
+    return it('should call the callback', function() {
+      return this.callback.called.should.equal(true)
+    })
+  })
+})

services/docstore/test/unit/js/RangeManagerTests.js

@@ -0,0 +1,257 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+require('chai').should()
+const { expect } = require('chai')
+const modulePath = require('path').join(
+  __dirname,
+  '../../../app/js/RangeManager'
+)
+const { ObjectId } = require('mongojs')
+const { assert } = require('chai')
+const _ = require('underscore')
+
+describe('RangeManager', function() {
+  beforeEach(function() {
+    return (this.RangeManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './mongojs': {
+          ObjectId
+        }
+      }
+    }))
+  })
+
+  describe('jsonRangesToMongo', function() {
+    it('should convert ObjectIds and dates to proper objects', function() {
+      const change_id = ObjectId().toString()
+      const comment_id = ObjectId().toString()
+      const user_id = ObjectId().toString()
+      const thread_id = ObjectId().toString()
+      const ts = new Date().toJSON()
+      return this.RangeManager.jsonRangesToMongo({
+        changes: [
+          {
+            id: change_id,
+            op: { i: 'foo', p: 3 },
+            metadata: {
+              user_id,
+              ts
+            }
+          }
+        ],
+        comments: [
+          {
+            id: comment_id,
+            op: { c: 'foo', p: 3, t: thread_id }
+          }
+        ]
+      }).should.deep.equal({
+        changes: [
+          {
+            id: ObjectId(change_id),
+            op: { i: 'foo', p: 3 },
+            metadata: {
+              user_id: ObjectId(user_id),
+              ts: new Date(ts)
+            }
+          }
+        ],
+        comments: [
+          {
+            id: ObjectId(comment_id),
+            op: { c: 'foo', p: 3, t: ObjectId(thread_id) }
+          }
+        ]
+      })
+    })
+
+    it('should leave malformed ObjectIds as they are', function() {
+      const change_id = 'foo'
+      const comment_id = 'bar'
+      const user_id = 'baz'
+      return this.RangeManager.jsonRangesToMongo({
+        changes: [
+          {
+            id: change_id,
+            metadata: {
+              user_id
+            }
+          }
+        ],
+        comments: [
+          {
+            id: comment_id
+          }
+        ]
+      }).should.deep.equal({
+        changes: [
+          {
+            id: change_id,
+            metadata: {
+              user_id
+            }
+          }
+        ],
+        comments: [
+          {
+            id: comment_id
+          }
+        ]
+      })
+    })
+
+    return it('should be consistent when transformed through json -> mongo -> json', function() {
+      const change_id = ObjectId().toString()
+      const comment_id = ObjectId().toString()
+      const user_id = ObjectId().toString()
+      const thread_id = ObjectId().toString()
+      const ts = new Date().toJSON()
+      const ranges1 = {
+        changes: [
+          {
+            id: change_id,
+            op: { i: 'foo', p: 3 },
+            metadata: {
+              user_id,
+              ts
+            }
+          }
+        ],
+        comments: [
+          {
+            id: comment_id,
+            op: { c: 'foo', p: 3, t: thread_id }
+          }
+        ]
+      }
+      const ranges1_copy = JSON.parse(JSON.stringify(ranges1)) // jsonRangesToMongo modifies in place
+      const ranges2 = JSON.parse(
+        JSON.stringify(this.RangeManager.jsonRangesToMongo(ranges1_copy))
+      )
+      return ranges1.should.deep.equal(ranges2)
+    })
+  })
+
+  return describe('shouldUpdateRanges', function() {
+    beforeEach(function() {
+      this.ranges = {
+        changes: [
+          {
+            id: ObjectId(),
+            op: { i: 'foo', p: 3 },
+            metadata: {
+              user_id: ObjectId(),
+              ts: new Date()
+            }
+          }
+        ],
+        comments: [
+          {
+            id: ObjectId(),
+            op: { c: 'foo', p: 3, t: ObjectId() }
+          }
+        ]
+      }
+      return (this.ranges_copy = this.RangeManager.jsonRangesToMongo(
+        JSON.parse(JSON.stringify(this.ranges))
+      ))
+    })
+
+    describe('with a blank new range', function() {
+      return it('should throw an error', function() {
+        return expect(() => {
+          return this.RangeManager.shouldUpdateRanges(this.ranges, null)
+        }).to.throw(Error)
+      })
+    })
+
+    describe('with a blank old range', function() {
+      return it('should treat it like {}', function() {
+        this.RangeManager.shouldUpdateRanges(null, {}).should.equal(false)
+        return this.RangeManager.shouldUpdateRanges(
+          null,
+          this.ranges
+        ).should.equal(true)
+      })
+    })
+
+    describe('with no changes', function() {
+      return it('should return false', function() {
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(false)
+      })
+    })
+
+    return describe('with changes', function() {
+      it('should return true when the change id changes', function() {
+        this.ranges_copy.changes[0].id = ObjectId()
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      it('should return true when the change user id changes', function() {
+        this.ranges_copy.changes[0].metadata.user_id = ObjectId()
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      it('should return true when the change ts changes', function() {
+        this.ranges_copy.changes[0].metadata.ts = new Date(Date.now() + 1000)
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      it('should return true when the change op changes', function() {
+        this.ranges_copy.changes[0].op.i = 'bar'
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      it('should return true when the comment id changes', function() {
+        this.ranges_copy.comments[0].id = ObjectId()
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      it('should return true when the comment offset changes', function() {
+        this.ranges_copy.comments[0].op.p = 17
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+
+      return it('should return true when the comment content changes', function() {
+        this.ranges_copy.comments[0].op.c = 'bar'
+        return this.RangeManager.shouldUpdateRanges(
+          this.ranges,
+          this.ranges_copy
+        ).should.equal(true)
+      })
+    })
+  })
+})