Merge pull request #77 from overleaf/msm-decaffeinate

track-changes decaffeination

services/track-changes/.dockerignore

@@ -5,5 +5,3 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
-app.js
-**/js/*

services/track-changes/.eslintrc

@@ -0,0 +1,65 @@
+// this file was auto-generated, do not edit it directly.
+// instead run bin/update_build_scripts from
+// https://github.com/sharelatex/sharelatex-dev-environment
+// Version: 1.3.5
+{
+  "extends": [
+    "standard",
+    "prettier",
+    "prettier/standard"
+  ],
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
+  "plugins": [
+    "mocha",
+    "chai-expect",
+    "chai-friendly"
+  ],
+  "env": {
+    "node": true,
+    "mocha": true
+  },
+  "rules": {
+    // Swap the no-unused-expressions rule with a more chai-friendly one
+    "no-unused-expressions": 0,
+    "chai-friendly/no-unused-expressions": "error"
+  },
+  "overrides": [
+    {
+      // Test specific rules
+      "files": ["test/**/*.js"],
+      "globals": {
+        "expect": true
+      },
+      "rules": {
+        // mocha-specific rules
+        "mocha/handle-done-callback": "error",
+        "mocha/no-exclusive-tests": "error",
+        "mocha/no-global-tests": "error",
+        "mocha/no-identical-title": "error",
+        "mocha/no-nested-tests": "error",
+        "mocha/no-pending-tests": "error",
+        "mocha/no-skipped-tests": "error",
+        "mocha/no-mocha-arrows": "error",
+
+        // chai-specific rules
+        "chai-expect/missing-assertion": "error",
+        "chai-expect/terminating-properties": "error",
+
+        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
+        // we don't enforce this at the top-level - just in tests to manage `this` scope
+        // based on mocha's context mechanism
+        "mocha/prefer-arrow-callback": "error"
+      }
+    },
+    {
+      // Backend specific rules
+      "files": ["app/**/*.js", "app.js", "index.js"],
+      "rules": {
+        // don't allow console.log in backend code
+        "no-console": "error"
+      }
+    }
+  ]
+}

services/track-changes/.gitignore

@@ -1,8 +1,3 @@
 **.swp
 node_modules/
-app/js
-app.js
-**/*.map
-test/unit/js
-test/acceptance/js
 forever/

services/track-changes/.prettierrc

@@ -0,0 +1,8 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3.5
+{
+  "semi": false,
+  "singleQuote": true
+}

services/track-changes/Dockerfile

@@ -17,7 +17,6 @@ RUN npm install --quiet
 COPY . /app
 
 
-RUN npm run compile:all
 
 FROM base
 

services/track-changes/Jenkinsfile

@@ -37,6 +37,13 @@ pipeline {
       }
     }
 
+    stage('Linting') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
+      }
+    }
+
     stage('Unit Tests') {
       steps {
         sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

services/track-changes/Makefile

@@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 clean:
 	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	rm -f app.js
-	rm -rf app/js
-	rm -rf test/unit/js
-	rm -rf test/acceptance/js
 
-test: test_unit test_acceptance
+format:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+
+format_fix:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+
+lint:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+
+test: format lint test_unit test_acceptance
 
 test_unit:
 	@[ ! -d test/unit ] && echo "track-changes has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit

services/track-changes/app.coffee

@@ -1,98 +0,0 @@
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("track-changes")
-Settings = require "settings-sharelatex"
-logger = require "logger-sharelatex"
-TrackChangesLogger = logger.initialize("track-changes").logger
-
-if Settings.sentry?.dsn?
-	logger.initializeErrorReporting(Settings.sentry.dsn)
-
-# log updates as truncated strings
-truncateFn = (updates) ->
-		JSON.parse(
-			JSON.stringify updates, (key, value) ->
-				if typeof value == 'string' && (len = value.length) > 80
-					return value.substr(0,32) + "...(message of length #{len} truncated)..." + value.substr(-32)
-				else
-					return value
-		)
-
-TrackChangesLogger.addSerializers {
-	rawUpdate: truncateFn
-	rawUpdates: truncateFn
-	newUpdates: truncateFn
-	lastUpdate: truncateFn
-}
-
-Path = require "path"
-
-Metrics.memory.monitor(logger)
-
-child_process = require "child_process"
-
-HttpController = require "./app/js/HttpController"
-express = require "express"
-app = express()
-
-app.use Metrics.http.monitor(logger)
-
-Metrics.injectMetricsRoute(app)
-
-app.post "/project/:project_id/doc/:doc_id/flush", HttpController.flushDoc
-
-app.get "/project/:project_id/doc/:doc_id/diff", HttpController.getDiff
-
-app.get "/project/:project_id/doc/:doc_id/check", HttpController.checkDoc
-
-app.get "/project/:project_id/updates", HttpController.getUpdates
-
-app.post "/project/:project_id/flush", HttpController.flushProject
-
-app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpController.restore
-
-app.post  '/project/:project_id/doc/:doc_id/push', HttpController.pushDocHistory
-app.post  '/project/:project_id/doc/:doc_id/pull', HttpController.pullDocHistory
-
-app.post '/flush/all', HttpController.flushAll
-app.post '/check/dangling', HttpController.checkDanglingUpdates
-
-packWorker = null # use a single packing worker
-
-app.post "/pack", (req, res, next) ->
-	if packWorker?
-		res.send "pack already running"
-	else
-		logger.log "running pack"
-		packWorker = child_process.fork(__dirname + '/app/js/PackWorker.js',
-			[req.query.limit || 1000, req.query.delay || 1000, req.query.timeout || 30*60*1000])
-		packWorker.on 'exit', (code, signal) ->
-			logger.log {code, signal}, "history auto pack exited"
-			packWorker = null
-		res.send "pack started"
-
-app.get "/status", (req, res, next) ->
-	res.send "track-changes is alive"
-
-app.get "/oops", (req, res, next) ->
-	throw new Error("dummy test error")
-
-app.get "/check_lock", HttpController.checkLock
-
-app.get "/health_check",  HttpController.healthCheck
-
-app.use (error, req, res, next) ->
-	logger.error err: error, req: req, "an internal error occured"
-	res.send 500
-
-port = Settings.internal?.trackchanges?.port or 3015
-host = Settings.internal?.trackchanges?.host or "localhost"
-
-if !module.parent # Called directly
-	app.listen port, host, (error) ->
-		if error?
-			logger.error err: error, "could not start track-changes server"
-		else
-			logger.info "trackchanges starting up, listening on #{host}:#{port}"
-
-module.exports = app
-

services/track-changes/app.js

@@ -0,0 +1,144 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const Metrics = require('metrics-sharelatex')
+Metrics.initialize('track-changes')
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const TrackChangesLogger = logger.initialize('track-changes').logger
+
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
+  logger.initializeErrorReporting(Settings.sentry.dsn)
+}
+
+// log updates as truncated strings
+const truncateFn = updates =>
+  JSON.parse(
+    JSON.stringify(updates, function(key, value) {
+      let len
+      if (typeof value === 'string' && (len = value.length) > 80) {
+        return (
+          value.substr(0, 32) +
+          `...(message of length ${len} truncated)...` +
+          value.substr(-32)
+        )
+      } else {
+        return value
+      }
+    })
+  )
+TrackChangesLogger.addSerializers({
+  rawUpdate: truncateFn,
+  rawUpdates: truncateFn,
+  newUpdates: truncateFn,
+  lastUpdate: truncateFn
+})
+
+const Path = require('path')
+
+Metrics.memory.monitor(logger)
+
+const child_process = require('child_process')
+
+const HttpController = require('./app/js/HttpController')
+const express = require('express')
+const app = express()
+
+app.use(Metrics.http.monitor(logger))
+
+Metrics.injectMetricsRoute(app)
+
+app.post('/project/:project_id/doc/:doc_id/flush', HttpController.flushDoc)
+
+app.get('/project/:project_id/doc/:doc_id/diff', HttpController.getDiff)
+
+app.get('/project/:project_id/doc/:doc_id/check', HttpController.checkDoc)
+
+app.get('/project/:project_id/updates', HttpController.getUpdates)
+
+app.post('/project/:project_id/flush', HttpController.flushProject)
+
+app.post(
+  '/project/:project_id/doc/:doc_id/version/:version/restore',
+  HttpController.restore
+)
+
+app.post('/project/:project_id/doc/:doc_id/push', HttpController.pushDocHistory)
+app.post('/project/:project_id/doc/:doc_id/pull', HttpController.pullDocHistory)
+
+app.post('/flush/all', HttpController.flushAll)
+app.post('/check/dangling', HttpController.checkDanglingUpdates)
+
+let packWorker = null // use a single packing worker
+
+app.post('/pack', function(req, res, next) {
+  if (packWorker != null) {
+    return res.send('pack already running')
+  } else {
+    logger.log('running pack')
+    packWorker = child_process.fork(__dirname + '/app/js/PackWorker.js', [
+      req.query.limit || 1000,
+      req.query.delay || 1000,
+      req.query.timeout || 30 * 60 * 1000
+    ])
+    packWorker.on('exit', function(code, signal) {
+      logger.log({ code, signal }, 'history auto pack exited')
+      return (packWorker = null)
+    })
+    return res.send('pack started')
+  }
+})
+
+app.get('/status', (req, res, next) => res.send('track-changes is alive'))
+
+app.get('/oops', function(req, res, next) {
+  throw new Error('dummy test error')
+})
+
+app.get('/check_lock', HttpController.checkLock)
+
+app.get('/health_check', HttpController.healthCheck)
+
+app.use(function(error, req, res, next) {
+  logger.error({ err: error, req }, 'an internal error occured')
+  return res.send(500)
+})
+
+const port =
+  __guard__(
+    Settings.internal != null ? Settings.internal.trackchanges : undefined,
+    x => x.port
+  ) || 3015
+const host =
+  __guard__(
+    Settings.internal != null ? Settings.internal.trackchanges : undefined,
+    x1 => x1.host
+  ) || 'localhost'
+
+if (!module.parent) {
+  // Called directly
+  app.listen(port, host, function(error) {
+    if (error != null) {
+      return logger.error(
+        { err: error },
+        'could not start track-changes server'
+      )
+    } else {
+      return logger.info(
+        `trackchanges starting up, listening on ${host}:${port}`
+      )
+    }
+  })
+}
+
+module.exports = app
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/app/coffee/DiffGenerator.coffee

@@ -1,227 +0,0 @@
-ConsistencyError = (message) ->
-	error = new Error(message)
-	error.name = "ConsistencyError"
-	error.__proto__ = ConsistencyError.prototype
-	return error
-ConsistencyError.prototype.__proto__ = Error.prototype
-
-logger = require "logger-sharelatex"
-
-module.exports = DiffGenerator =
-	ConsistencyError: ConsistencyError
-
-	rewindUpdate: (content, update) ->
-		for op, i in update.op by -1 when op.broken isnt true
-			try
-				content = DiffGenerator.rewindOp content, op
-			catch e
-				if e instanceof ConsistencyError and i = update.op.length - 1
-					# catch known case where the last op in an array has been
-					# merged into a later op
-					logger.error {err: e, update, op: JSON.stringify(op)}, "marking op as broken"
-					op.broken = true
-				else
-					throw e # rethrow the execption
-		return content
-
-	rewindOp: (content, op) ->
-		if op.i?
-			# ShareJS will accept an op where p > content.length when applied,
-			# and it applies as though p == content.length. However, the op is
-			# passed to us with the original p > content.length. Detect if that
-			# is the case with this op, and shift p back appropriately to match
-			# ShareJS if so.
-			p = op.p
-			max_p = content.length - op.i.length
-			if p > max_p
-				logger.warn {max_p, p}, "truncating position to content length"
-				p = max_p
-
-			textToBeRemoved = content.slice(p, p + op.i.length)
-			if op.i != textToBeRemoved
-				throw new ConsistencyError(
-					"Inserted content, '#{op.i}', does not match text to be removed, '#{textToBeRemoved}'"
-				)
-
-			return content.slice(0, p) + content.slice(p + op.i.length)
-
-		else if op.d?
-			return content.slice(0, op.p) + op.d + content.slice(op.p)
-		
-		else
-			return content
-
-	rewindUpdates: (content, updates) ->
-		for update in updates.reverse()
-			try
-				content = DiffGenerator.rewindUpdate(content, update)
-			catch e
-				e.attempted_update = update # keep a record of the attempted update
-				throw e # rethrow the exception
-		return content
-
-	buildDiff: (initialContent, updates) ->
-		diff = [ u: initialContent ]
-		for update in updates
-			diff = DiffGenerator.applyUpdateToDiff diff, update
-		diff = DiffGenerator.compressDiff diff
-		return diff
-
-	compressDiff: (diff) ->
-		newDiff = []
-		for part in diff
-			lastPart = newDiff[newDiff.length - 1]
-			if lastPart? and lastPart.meta?.user? and part.meta?.user?
-				if lastPart.i? and part.i? and lastPart.meta.user.id == part.meta.user.id
-					lastPart.i += part.i
-					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts)
-					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts)
-				else if lastPart.d? and part.d? and lastPart.meta.user.id == part.meta.user.id
-					lastPart.d += part.d
-					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts)
-					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts)
-				else
-					newDiff.push part
-			else
-				newDiff.push part
-		return newDiff
-
-	applyOpToDiff: (diff, op, meta) ->
-		position = 0
-
-		remainingDiff = diff.slice()
-		{consumedDiff, remainingDiff} = DiffGenerator._consumeToOffset(remainingDiff, op.p)
-		newDiff = consumedDiff
-
-		if op.i?
-			newDiff.push
-				i: op.i
-				meta: meta
-		else if op.d?
-			{consumedDiff, remainingDiff} = DiffGenerator._consumeDiffAffectedByDeleteOp remainingDiff, op, meta
-			newDiff.push(consumedDiff...)
-
-		newDiff.push(remainingDiff...)
-
-		return newDiff
-
-	applyUpdateToDiff: (diff, update) ->
-		for op in update.op when op.broken isnt true
-			diff = DiffGenerator.applyOpToDiff diff, op, update.meta
-		return diff
-
-	_consumeToOffset: (remainingDiff, totalOffset) ->
-		consumedDiff = []
-		position = 0
-		while part = remainingDiff.shift()
-			length = DiffGenerator._getLengthOfDiffPart part
-			if part.d?
-				consumedDiff.push part
-			else if position + length >= totalOffset
-				partOffset = totalOffset - position
-				if partOffset > 0
-					consumedDiff.push DiffGenerator._slicePart part, 0, partOffset
-				if partOffset < length
-					remainingDiff.unshift DiffGenerator._slicePart part, partOffset
-				break
-			else
-				position += length
-				consumedDiff.push part
-
-		return {
-			consumedDiff: consumedDiff
-			remainingDiff: remainingDiff
-		}
-
-	_consumeDiffAffectedByDeleteOp: (remainingDiff, deleteOp, meta) ->
-		consumedDiff = []
-		remainingOp = deleteOp
-		while remainingOp and remainingDiff.length > 0
-			{newPart, remainingDiff, remainingOp} = DiffGenerator._consumeDeletedPart remainingDiff, remainingOp, meta
-			consumedDiff.push newPart if newPart?
-		return {
-			consumedDiff: consumedDiff
-			remainingDiff: remainingDiff
-		}
-
-	_consumeDeletedPart: (remainingDiff, op, meta) ->
-		part = remainingDiff.shift()
-		partLength = DiffGenerator._getLengthOfDiffPart part
-
-		if part.d?
-			# Skip existing deletes
-			remainingOp = op
-			newPart = part
-
-		else if partLength > op.d.length
-			# Only the first bit of the part has been deleted
-			remainingPart = DiffGenerator._slicePart part, op.d.length
-			remainingDiff.unshift remainingPart
-
-			deletedContent = DiffGenerator._getContentOfPart(part).slice(0, op.d.length)
-			if deletedContent != op.d
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{op.d}'")
-
-			if part.u?
-				newPart =
-					d: op.d
-					meta: meta
-			else if part.i?
-				newPart = null
-
-			remainingOp = null
-
-		else if partLength == op.d.length
-			# The entire part has been deleted, but it is the last part
-
-			deletedContent = DiffGenerator._getContentOfPart(part)
-			if deletedContent != op.d
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{op.d}'")
-
-			if part.u?
-				newPart =
-					d: op.d
-					meta: meta
-			else if part.i?
-				newPart = null
-
-			remainingOp = null
-
-		else if partLength < op.d.length
-			# The entire part has been deleted and there is more
-
-			deletedContent = DiffGenerator._getContentOfPart(part)
-			opContent = op.d.slice(0, deletedContent.length)
-			if deletedContent != opContent
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{opContent}'")
-
-			if part.u
-				newPart =
-					d: part.u
-					meta: meta
-			else if part.i?
-				newPart = null
-
-			remainingOp =
-				p: op.p, d: op.d.slice(DiffGenerator._getLengthOfDiffPart(part))
-
-		return {
-			newPart: newPart
-			remainingDiff: remainingDiff
-			remainingOp: remainingOp
-		}
-
-	_slicePart: (basePart, from, to) ->
-		if basePart.u?
-			part = { u: basePart.u.slice(from, to) }
-		else if basePart.i?
-			part = { i: basePart.i.slice(from, to) }
-		if basePart.meta?
-			part.meta = basePart.meta
-		return part
-
-	_getLengthOfDiffPart: (part) ->
-		(part.u or part.d or part.i or '').length
-
-	_getContentOfPart: (part) ->
-		part.u or part.d or part.i or ''

services/track-changes/app/coffee/DiffManager.coffee

@@ -1,88 +0,0 @@
-UpdatesManager = require "./UpdatesManager"
-DocumentUpdaterManager = require "./DocumentUpdaterManager"
-DiffGenerator = require "./DiffGenerator"
-logger = require "logger-sharelatex"
-
-module.exports = DiffManager =
-	getLatestDocAndUpdates: (project_id, doc_id, fromVersion, callback = (error, content, version, updates) ->) ->
-		# Get updates last, since then they must be ahead and it
-		# might be possible to rewind to the same version as the doc.
-		DocumentUpdaterManager.getDocument project_id, doc_id, (error, content, version) ->
-			return callback(error) if error?
-			if !fromVersion? # If we haven't been given a version, just return lastest doc and no updates
-				return callback(null, content, version, [])
-			UpdatesManager.getDocUpdatesWithUserInfo project_id, doc_id, from: fromVersion, (error, updates) ->
-				return callback(error) if error?
-				callback(null, content, version, updates)
-	
-	getDiff: (project_id, doc_id, fromVersion, toVersion, callback = (error, diff) ->) ->
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, fromVersion, (error, startingContent, updates) ->
-			if error?
-				if error.message == "broken-history"
-					return callback(null, "history unavailable")
-				else
-					return callback(error)
-
-			updatesToApply = []
-			for update in updates.slice().reverse()
-				if update.v <= toVersion
-					updatesToApply.push update
-
-			try
-				diff = DiffGenerator.buildDiff startingContent, updatesToApply
-			catch e
-				return callback(e)
-			
-			callback(null, diff)
-
-	getDocumentBeforeVersion: (project_id, doc_id, version, _callback = (error, document, rewoundUpdates) ->) ->
-		# Whichever order we get the latest document and the latest updates,
-		# there is potential for updates to be applied between them so that
-		# they do not return the same 'latest' versions.
-		# If this happens, we just retry and hopefully get them at the compatible
-		# versions.
-		retries = 3
-		callback = (error, args...) ->
-			if error?
-				if error.retry and retries > 0
-					logger.warn {error, project_id, doc_id, version, retries}, "retrying getDocumentBeforeVersion"
-					retry()
-				else
-					_callback(error)
-			else
-				_callback(null, args...)
-
-		do retry = () ->
-			retries--
-			DiffManager._tryGetDocumentBeforeVersion(project_id, doc_id, version, callback)
-
-	_tryGetDocumentBeforeVersion: (project_id, doc_id, version, callback = (error, document, rewoundUpdates) ->) ->
-		logger.log project_id: project_id, doc_id: doc_id, version: version, "getting document before version"
-		DiffManager.getLatestDocAndUpdates project_id, doc_id, version, (error, content, version, updates) ->
-			return callback(error) if error?
-
-			# bail out if we hit a broken update
-			for u in updates when u.broken
-				return callback new Error "broken-history"
-
-			# discard any updates which are ahead of this document version
-			while updates[0]?.v >= version
-				updates.shift()
-
-			lastUpdate = updates[0]
-			if lastUpdate? and lastUpdate.v != version - 1
-				error = new Error("latest update version, #{lastUpdate.v}, does not match doc version, #{version}")
-				error.retry = true
-				return callback error
-			
-			logger.log {docVersion: version, lastUpdateVersion: lastUpdate?.v, updateCount: updates.length}, "rewinding updates"
-
-			tryUpdates = updates.slice().reverse()
-
-			try
-				startingContent = DiffGenerator.rewindUpdates content, tryUpdates
-				# tryUpdates is reversed, and any unapplied ops are marked as broken
-			catch e
-				return callback(e)
-
-			callback(null, startingContent, tryUpdates)

services/track-changes/app/coffee/DocumentUpdaterManager.coffee

@@ -1,42 +0,0 @@
-request = require "request"
-logger = require "logger-sharelatex"
-Settings = require "settings-sharelatex"
-
-module.exports = DocumentUpdaterManager =
-	getDocument: (project_id, doc_id, callback = (error, content, version) ->) ->
-		url = "#{Settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}"
-		logger.log project_id:project_id, doc_id: doc_id, "getting doc from document updater"
-		request.get url, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode >= 200 and res.statusCode < 300
-				try
-					body = JSON.parse(body)
-				catch error
-					return callback(error)
-				logger.log {project_id, doc_id, version: body.version}, "got doc from document updater"
-				callback null, body.lines.join("\n"), body.version
-			else
-				error = new Error("doc updater returned a non-success status code: #{res.statusCode}")
-				logger.error err: error, project_id:project_id, doc_id:doc_id, url: url, "error accessing doc updater"
-				callback error
-
-	setDocument: (project_id, doc_id, content, user_id, callback = (error) ->) ->
-		url = "#{Settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}"
-		logger.log project_id:project_id, doc_id: doc_id, "setting doc in document updater"
-		request.post {
-			url: url
-			json:
-				lines: content.split("\n")
-				source: "restore"
-				user_id: user_id
-				undoing: true
-		}, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode >= 200 and res.statusCode < 300
-				callback null
-			else
-				error = new Error("doc updater returned a non-success status code: #{res.statusCode}")
-				logger.error err: error, project_id:project_id, doc_id:doc_id, url: url, "error accessing doc updater"
-				callback error

services/track-changes/app/coffee/HealthChecker.coffee

@@ -1,46 +0,0 @@
-ObjectId = require("mongojs").ObjectId
-request = require("request")
-async = require("async")
-settings = require("settings-sharelatex")
-port = settings.internal.trackchanges.port
-logger = require "logger-sharelatex"
-LockManager = require "./LockManager"
-
-module.exports =
-	check : (callback)->
-		project_id = ObjectId(settings.trackchanges.healthCheck.project_id)
-		url = "http://localhost:#{port}/project/#{project_id}"
-		logger.log project_id:project_id, "running health check"
-		jobs = [
-			(cb)->
-				request.get {url:"http://localhost:#{port}/check_lock", timeout:3000}, (err, res, body) ->
-					if err?
-						logger.err err:err, project_id:project_id, "error checking lock for health check"
-						cb(err)
-					else if res?.statusCode != 200
-						cb("status code not 200, it's #{res.statusCode}")
-					else
-						cb()
-			(cb)->
-				request.post {url:"#{url}/flush", timeout:10000}, (err, res, body) ->
-					if err?
-						logger.err err:err, project_id:project_id, "error flushing for health check"
-						cb(err)
-					else if res?.statusCode != 204
-						cb("status code not 204, it's #{res.statusCode}")
-					else
-						cb()
-			(cb)->
-				request.get {url:"#{url}/updates", timeout:10000}, (err, res, body)->
-					if err?
-						logger.err err:err, project_id:project_id, "error getting updates for health check"
-						cb(err)
-					else if res?.statusCode != 200
-						cb("status code not 200, it's #{res.statusCode}")
-					else
-						cb()
-		]
-		async.series jobs, callback
-
-	checkLock: (callback) ->
-		LockManager.healthCheck callback

services/track-changes/app/coffee/HttpController.coffee

@@ -1,137 +0,0 @@
-UpdatesManager = require "./UpdatesManager"
-DiffManager = require "./DiffManager"
-PackManager = require "./PackManager"
-RestoreManager = require "./RestoreManager"
-logger = require "logger-sharelatex"
-HealthChecker = require "./HealthChecker"
-_ = require "underscore"
-
-module.exports = HttpController =
-	flushDoc: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-		logger.log project_id: project_id, doc_id: doc_id, "compressing doc history"
-		UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	flushProject: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "compressing project history"
-		UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	flushAll: (req, res, next = (error) ->) ->
-		# limit on projects to flush or -1 for all (default)
-		limit = if req.query.limit? then parseInt(req.query.limit, 10) else -1
-		logger.log {limit: limit}, "flushing all projects"
-		UpdatesManager.flushAll limit, (error, result) ->
-			return next(error) if error?
-			{failed, succeeded, all} = result
-			status = "#{succeeded.length} succeeded, #{failed.length} failed"
-			if limit == 0
-				res.status(200).send "#{status}\nwould flush:\n#{all.join('\n')}\n"
-			else if failed.length > 0
-				logger.log {failed: failed, succeeded: succeeded}, "error flushing projects"
-				res.status(500).send "#{status}\nfailed to flush:\n#{failed.join('\n')}\n"
-			else
-				res.status(200).send "#{status}\nflushed #{succeeded.length} projects of #{all.length}\n"
-
-	checkDanglingUpdates: (req, res, next = (error) ->) ->
-		logger.log "checking dangling updates"
-		UpdatesManager.getDanglingUpdates (error, result) ->
-			return next(error) if error?
-			if result.length > 0
-				logger.log {dangling: result}, "found dangling updates"
-				res.status(500).send "dangling updates:\n#{result.join('\n')}\n"
-			else
-				res.status(200).send "no dangling updates found\n"
-
-	checkDoc: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-		logger.log project_id: project_id, doc_id: doc_id, "checking doc history"
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, 1, (error, document, rewoundUpdates) ->
-			return next(error) if error?
-			broken = []
-			for update in rewoundUpdates
-				for op in update.op when op.broken is true
-					broken.push op
-			if broken.length > 0
-				res.send broken
-			else
-				res.send 204
-
-	getDiff: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-
-		if req.query.from?
-			from = parseInt(req.query.from, 10)
-		else
-			from = null
-		if req.query.to?
-			to = parseInt(req.query.to, 10)
-		else
-			to = null
-
-		logger.log {project_id, doc_id, from, to}, "getting diff"
-		DiffManager.getDiff project_id, doc_id, from, to, (error, diff) ->
-			return next(error) if error?
-			res.json {diff: diff}
-
-	getUpdates: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-
-		if req.query.before?
-			before = parseInt(req.query.before, 10)
-		if req.query.min_count?
-			min_count = parseInt(req.query.min_count, 10)
-
-		UpdatesManager.getSummarizedProjectUpdates project_id, before: before, min_count: min_count, (error, updates, nextBeforeTimestamp) ->
-			return next(error) if error?
-			res.json {
-				updates: updates
-				nextBeforeTimestamp: nextBeforeTimestamp
-			}
-
-	restore: (req, res, next = (error) ->) ->
-		{doc_id, project_id, version} = req.params
-		user_id = req.headers["x-user-id"]
-		version = parseInt(version, 10)
-		RestoreManager.restoreToBeforeVersion project_id, doc_id, version, user_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	pushDocHistory: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id = req.params.doc_id
-		logger.log {project_id, doc_id}, "pushing all finalised changes to s3"
-		PackManager.pushOldPacks project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	pullDocHistory: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id = req.params.doc_id
-		logger.log {project_id, doc_id}, "pulling all packs from s3"
-		PackManager.pullOldPacks project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	healthCheck: (req, res)->
-		HealthChecker.check (err)->
-			if err?
-				logger.err err:err, "error performing health check"
-				res.send 500
-			else
-				res.send 200
-
-	checkLock: (req, res)->
-		HealthChecker.checkLock (err) ->
-			if err?
-				logger.err err:err, "error performing lock check"
-				res.send 500
-			else
-				res.send 200

services/track-changes/app/coffee/LockManager.coffee

@@ -1,85 +0,0 @@
-Settings = require "settings-sharelatex"
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.lock)
-os = require "os"
-crypto = require "crypto"
-logger = require "logger-sharelatex"
-
-HOST = os.hostname()
-PID = process.pid
-RND = crypto.randomBytes(4).toString('hex')
-COUNT = 0
-
-module.exports = LockManager =
-	LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock
-	MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock
-	LOCK_TTL: 300 # seconds (allow 5 minutes for any operation to complete)
-
-	# Use a signed lock value as described in
-	# http://redis.io/topics/distlock#correct-implementation-with-a-single-instance
-	# to prevent accidental unlocking by multiple processes
-	randomLock : () ->
-		time = Date.now()
-		return "locked:host=#{HOST}:pid=#{PID}:random=#{RND}:time=#{time}:count=#{COUNT++}"
-
-	unlockScript: 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
-
-	tryLock : (key, callback = (err, gotLock) ->) ->
-		lockValue = LockManager.randomLock()
-		rclient.set key, lockValue, "EX", @LOCK_TTL, "NX", (err, gotLock)->
-			return callback(err) if err?
-			if gotLock == "OK"
-				callback err, true, lockValue
-			else
-				callback err, false
-
-	getLock: (key, callback = (error) ->) ->
-		startTime = Date.now()
-		do attempt = () ->
-			if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
-				e = new Error("Timeout")
-				e.key = key
-				return callback(e)
-
-			LockManager.tryLock key, (error, gotLock, lockValue) ->
-				return callback(error) if error?
-				if gotLock
-					callback(null, lockValue)
-				else
-					setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
-
-	checkLock: (key, callback = (err, isFree) ->) ->
-		rclient.exists key, (err, exists) ->
-			return callback(err) if err?
-			exists = parseInt exists
-			if exists == 1
-				callback err, false
-			else
-				callback err, true
-
-	releaseLock: (key, lockValue, callback) ->
-		rclient.eval LockManager.unlockScript, 1, key, lockValue, (err, result) ->
-			if err?
-				return callback(err)
-			if result? and result isnt 1 # successful unlock should release exactly one key
-				logger.error {key:key, lockValue:lockValue, redis_err:err, redis_result:result}, "unlocking error"
-				return callback(new Error("tried to release timed out lock"))
-			callback(err,result)
-
-	runWithLock: (key, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
-		LockManager.getLock key, (error, lockValue) ->
-			return callback(error) if error?
-			runner (error1) ->
-				LockManager.releaseLock key, lockValue, (error2) ->
-					error = error1 or error2
-					return callback(error) if error?
-					callback()
-
-	healthCheck: (callback) ->
-		action = (releaseLock) ->
-			releaseLock()
-		LockManager.runWithLock	"HistoryLock:HealthCheck:host=#{HOST}:pid=#{PID}:random=#{RND}", action, callback
-
-	close: (callback) ->
-		rclient.quit()
-		rclient.once 'end', callback

services/track-changes/app/coffee/MongoAWS.coffee

@@ -1,118 +0,0 @@
-settings = require "settings-sharelatex"
-logger = require "logger-sharelatex"
-AWS = require 'aws-sdk'
-S3S = require 's3-streams'
-{db, ObjectId} = require "./mongojs"
-JSONStream = require "JSONStream"
-ReadlineStream = require "byline"
-zlib = require "zlib"
-Metrics = require "metrics-sharelatex"
-
-DAYS = 24 * 3600 * 1000 # one day in milliseconds
-
-createStream = (streamConstructor, project_id, doc_id, pack_id) ->
-	AWS_CONFIG =
-		accessKeyId: settings.trackchanges.s3.key
-		secretAccessKey: settings.trackchanges.s3.secret
-		endpoint: settings.trackchanges.s3.endpoint
-		s3ForcePathStyle: settings.trackchanges.s3.pathStyle
-
-	return streamConstructor new AWS.S3(AWS_CONFIG), {
-		"Bucket": settings.trackchanges.stores.doc_history,
-		"Key": project_id+"/changes-"+doc_id+"/pack-"+pack_id
-	}
-
-module.exports = MongoAWS =
-
-	archivePack: (project_id, doc_id, pack_id, _callback = (error) ->) ->
-
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
-
-		query = {
-			_id: ObjectId(pack_id)
-			doc_id: ObjectId(doc_id)
-		}
-
-		return callback new Error("invalid project id") if not project_id?
-		return callback new Error("invalid doc id") if not doc_id?
-		return callback new Error("invalid pack id") if not pack_id?
-
-		logger.log {project_id, doc_id, pack_id}, "uploading data to s3"
-
-		upload = createStream S3S.WriteStream, project_id, doc_id, pack_id
-
-		db.docHistory.findOne query, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("cannot find pack to send to s3") if not result?
-			return callback new Error("refusing to send pack with TTL to s3") if result.expiresAt?
-			uncompressedData = JSON.stringify(result)
-			if uncompressedData.indexOf("\u0000") != -1
-				error = new Error("null bytes found in upload")
-				logger.error err: error, project_id: project_id, doc_id: doc_id, pack_id: pack_id, error.message
-				return callback(error)
-			zlib.gzip uncompressedData, (err, buf) ->
-				logger.log {project_id, doc_id, pack_id, origSize: uncompressedData.length, newSize: buf.length}, "compressed pack"
-				return callback(err) if err?
-				upload.on 'error', (err) ->
-					callback(err)
-				upload.on 'finish', () ->
-					Metrics.inc("archive-pack")
-					logger.log {project_id, doc_id, pack_id}, "upload to s3 completed"
-					callback(null)
-				upload.write buf
-				upload.end()
-
-	readArchivedPack: (project_id, doc_id, pack_id, _callback = (error, result) ->) ->
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
-
-		return callback new Error("invalid project id") if not project_id?
-		return callback new Error("invalid doc id") if not doc_id?
-		return callback new Error("invalid pack id") if not pack_id?
-
-		logger.log {project_id, doc_id, pack_id}, "downloading data from s3"
-
-		download = createStream S3S.ReadStream, project_id, doc_id, pack_id
-
-		inputStream = download
-			.on 'open', (obj) ->
-				return 1
-			.on 'error', (err) ->
-				callback(err)
-
-		gunzip = zlib.createGunzip()
-		gunzip.setEncoding('utf8')
-		gunzip.on 'error', (err) ->
-			logger.log {project_id, doc_id, pack_id, err}, "error uncompressing gzip stream"
-			callback(err)
-
-		outputStream = inputStream.pipe gunzip
-		parts = []
-		outputStream.on 'error', (err) ->
-			return callback(err)
-		outputStream.on 'end', () ->
-			logger.log {project_id, doc_id, pack_id}, "download from s3 completed"
-			try
-				object = JSON.parse parts.join('')
-			catch e
-				return callback(e)
-			object._id = ObjectId(object._id)
-			object.doc_id = ObjectId(object.doc_id)
-			object.project_id = ObjectId(object.project_id)
-			for op in object.pack
-				op._id = ObjectId(op._id) if op._id?
-			callback null, object
-		outputStream.on 'data', (data) ->
-			parts.push data
-
-	unArchivePack: (project_id, doc_id, pack_id, callback = (error) ->) ->
-		MongoAWS.readArchivedPack project_id, doc_id, pack_id, (err, object) ->
-			return callback(err) if err?
-			Metrics.inc("unarchive-pack")
-			# allow the object to expire, we can always retrieve it again
-			object.expiresAt = new Date(Date.now() + 7 * DAYS)
-			logger.log {project_id, doc_id, pack_id}, "inserting object from s3"
-			db.docHistory.insert object, callback

services/track-changes/app/coffee/MongoManager.coffee

@@ -1,104 +0,0 @@
-{db, ObjectId} = require "./mongojs"
-PackManager = require "./PackManager"
-async = require "async"
-_ = require "underscore"
-metrics = require 'metrics-sharelatex'
-logger = require 'logger-sharelatex'
-
-module.exports = MongoManager =
-	getLastCompressedUpdate: (doc_id, callback = (error, update) ->) ->
-		db.docHistory
-			.find(doc_id: ObjectId(doc_id.toString()), {pack: {$slice:-1}}) # only return the last entry in a pack
-			.sort( v: -1 )
-			.limit(1)
-			.toArray (error, compressedUpdates) ->
-				return callback(error) if error?
-				callback null, compressedUpdates[0] or null
-
-	peekLastCompressedUpdate: (doc_id, callback = (error, update, version) ->) ->
-		# under normal use we pass back the last update as
-		# callback(null,update,version).
-		#
-		# when we have an existing last update but want to force a new one
-		# to start, we pass it back as callback(null,null,version), just
-		# giving the version so we can check consistency.
-		MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
-			return callback(error) if error?
-			if update?
-				if update.broken # marked as broken so we will force a new op
-					return callback null, null
-				else if update.pack?
-					if update.finalised # no more ops can be appended
-						return callback null, null, update.pack[0]?.v
-					else
-						return callback null, update, update.pack[0]?.v
-				else
-					return callback null, update, update.v
-			else
-				PackManager.getLastPackFromIndex doc_id, (error, pack) ->
-					return callback(error) if error?
-					return callback(null, null, pack.v_end) if pack?.inS3? and pack?.v_end?
-					callback null, null
-
-	backportProjectId: (project_id, doc_id, callback = (error) ->) ->
-		db.docHistory.update {
-			doc_id: ObjectId(doc_id.toString())
-			project_id: { $exists: false }
-		}, {
-			$set: { project_id: ObjectId(project_id.toString()) }
-		}, {
-			multi: true
-		}, callback
-
-	getProjectMetaData: (project_id, callback = (error, metadata) ->) ->
-		db.projectHistoryMetaData.find {
-			project_id: ObjectId(project_id.toString())
-		}, (error, results) ->
-			return callback(error) if error?
-			callback null, results[0]
-
-	setProjectMetaData: (project_id, metadata, callback = (error) ->) ->
-		db.projectHistoryMetaData.update {
-			project_id: ObjectId(project_id)
-		}, {
-			$set: metadata
-		}, {
-			upsert: true
-		}, callback
-
-	upgradeHistory: (project_id, callback = (error) ->) ->
-		# preserve the project's existing history
-		db.docHistory.update {
-			project_id: ObjectId(project_id)
-			temporary: true
-			expiresAt: {$exists: true}
-		}, {
-			$set: {temporary: false}
-			$unset: {expiresAt: ""}
-		}, {
-			multi: true
-		}, callback
-
-	ensureIndices: () ->
-		# For finding all updates that go into a diff for a doc
-		db.docHistory.ensureIndex { doc_id: 1, v: 1 }, { background: true }
-		# For finding all updates that affect a project
-		db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, { background: true }
-		# For finding updates that don't yet have a project_id and need it inserting
-		db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, { background: true }
-		# For finding project meta-data
-		db.projectHistoryMetaData.ensureIndex { project_id: 1 }, { background: true }
-		# TTL index for auto deleting week old temporary ops
-		db.docHistory.ensureIndex { expiresAt: 1 }, { expireAfterSeconds: 0, background: true }
-		# For finding packs to be checked for archiving
-		db.docHistory.ensureIndex { last_checked: 1 }, { background: true }
-		# For finding archived packs
-		db.docHistoryIndex.ensureIndex { project_id: 1 }, { background: true }
-
-
-[
-	'getLastCompressedUpdate',
-	'getProjectMetaData',
-	'setProjectMetaData'
-].map (method) ->
-	metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger)

services/track-changes/app/coffee/PackManager.coffee

@@ -1,540 +0,0 @@
-async = require "async"
-_ = require "underscore"
-{db, ObjectId, BSON} = require "./mongojs"
-logger = require "logger-sharelatex"
-LockManager = require "./LockManager"
-MongoAWS = require "./MongoAWS"
-Metrics = require "metrics-sharelatex"
-ProjectIterator = require "./ProjectIterator"
-Settings = require "settings-sharelatex"
-keys = Settings.redis.lock.key_schema
-
-# Sharejs operations are stored in a 'pack' object
-#
-#  e.g.  a single sharejs update looks like
-#
-#   {
-#     "doc_id" : 549dae9e0a2a615c0c7f0c98,
-#     "project_id" : 549dae9c0a2a615c0c7f0c8c,
-#     "op" : [ {"p" : 6981,	"d" : "?"	} ],
-#     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
-#     "v" : 17082
-#   }
-#
-#  and a pack looks like this
-#
-#   {
-#     "doc_id" : 549dae9e0a2a615c0c7f0c98,
-#     "project_id" : 549dae9c0a2a615c0c7f0c8c,
-#     "pack" : [ U1, U2, U3, ...., UN],
-#     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
-#     "v" : 17082
-#     "v_end" : ...
-#   }
-#
-#  where U1, U2, U3, .... are single updates stripped of their
-#  doc_id and project_id fields (which are the same for all the
-#  updates in the pack).
-#
-#  The pack itself has v and meta fields, this makes it possible to
-#  treat packs and single updates in a similar way.
-#
-#  The v field of the pack itself is from the first entry U1, the
-#  v_end field from UN.  The meta.end_ts field of the pack itself is
-#  from the last entry UN, the meta.start_ts field from U1.
-
-DAYS = 24 * 3600 * 1000 # one day in milliseconds
-
-module.exports = PackManager =
-
-	MAX_SIZE:  1024*1024 # make these configurable parameters
-	MAX_COUNT: 1024
-
-	insertCompressedUpdates: (project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		return callback() if newUpdates.length == 0
-
-		# never append permanent ops to a pack that will expire
-		lastUpdate = null if lastUpdate?.expiresAt? and not temporary
-
-		updatesToFlush = []
-		updatesRemaining = newUpdates.slice()
-
-		n = lastUpdate?.n || 0
-		sz = lastUpdate?.sz || 0
-
-		while updatesRemaining.length and n < PackManager.MAX_COUNT and sz < PackManager.MAX_SIZE
-			nextUpdate = updatesRemaining[0]
-			nextUpdateSize = BSON.calculateObjectSize(nextUpdate)
-			if nextUpdateSize + sz > PackManager.MAX_SIZE and n > 0
-				break
-			n++
-			sz += nextUpdateSize
-			updatesToFlush.push updatesRemaining.shift()
-
-		PackManager.flushCompressedUpdates project_id, doc_id, lastUpdate, updatesToFlush, temporary, (error) ->
-			return callback(error) if error?
-			PackManager.insertCompressedUpdates project_id, doc_id, null, updatesRemaining, temporary, callback
-
-	flushCompressedUpdates:	(project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		return callback() if newUpdates.length == 0
-
-		canAppend = false
-		# check if it is safe to append to an existing pack
-		if lastUpdate?
-			if not temporary and not lastUpdate.expiresAt?
-				# permanent pack appends to permanent pack
-				canAppend = true
-			age = Date.now() - lastUpdate.meta?.start_ts
-			if temporary and lastUpdate.expiresAt? and age < 1 * DAYS
-				# temporary pack appends to temporary pack if same day
-				canAppend = true
-
-		if canAppend
-			PackManager.appendUpdatesToExistingPack project_id, doc_id, lastUpdate, newUpdates, temporary, callback
-		else
-			PackManager.insertUpdatesIntoNewPack project_id, doc_id, newUpdates, temporary, callback
-
-	insertUpdatesIntoNewPack: (project_id, doc_id, newUpdates, temporary, callback = (error) ->) ->
-		first = newUpdates[0]
-		last = newUpdates[newUpdates.length - 1]
-		n = newUpdates.length
-		sz = BSON.calculateObjectSize(newUpdates)
-		newPack =
-			project_id: ObjectId(project_id.toString())
-			doc_id: ObjectId(doc_id.toString())
-			pack: newUpdates
-			n: n
-			sz: sz
-			meta:
-				start_ts: first.meta.start_ts
-				end_ts: last.meta.end_ts
-			v: first.v
-			v_end: last.v
-			temporary: temporary
-		if temporary
-			newPack.expiresAt = new Date(Date.now() + 7 * DAYS)
-			newPack.last_checked = new Date(Date.now() + 30 * DAYS) # never check temporary packs
-		logger.log {project_id, doc_id, newUpdates}, "inserting updates into new pack"
-		db.docHistory.save newPack, (err, result) ->
-			return callback(err) if err?
-			Metrics.inc("insert-pack-" + if temporary then "temporary" else "permanent")
-			if temporary
-				return callback()
-			else
-				PackManager.updateIndex project_id, doc_id, callback
-
-	appendUpdatesToExistingPack: (project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		first = newUpdates[0]
-		last = newUpdates[newUpdates.length - 1]
-		n = newUpdates.length
-		sz = BSON.calculateObjectSize(newUpdates)
-		query =
-			_id: lastUpdate._id
-			project_id: ObjectId(project_id.toString())
-			doc_id: ObjectId(doc_id.toString())
-			pack: {$exists: true}
-		update =
-			$push:
-				"pack": {$each: newUpdates}
-			$inc:
-				"n": n
-				"sz":  sz
-			$set:
-				"meta.end_ts": last.meta.end_ts
-				"v_end": last.v
-		if lastUpdate.expiresAt and temporary
-			update.$set.expiresAt = new Date(Date.now() + 7 * DAYS)
-		logger.log {project_id, doc_id, lastUpdate, newUpdates}, "appending updates to existing pack"
-		Metrics.inc("append-pack-" + if temporary then "temporary" else "permanent")
-		db.docHistory.findAndModify {query, update,	new:true, fields:{meta:1,v_end:1}}, callback
-
-	# Retrieve all changes for a document
-
-	getOpsByVersionRange: (project_id, doc_id, fromVersion, toVersion, callback = (error, updates) ->) ->
-		PackManager.loadPacksByVersionRange project_id, doc_id, fromVersion, toVersion, (error) ->
-			query = {doc_id:ObjectId(doc_id.toString())}
-			query.v = {$lte:toVersion} if toVersion?
-			query.v_end = {$gte:fromVersion} if fromVersion?
-			#console.log "query:", query
-			db.docHistory.find(query).sort {v:-1}, (err, result) ->
-				return callback(err) if err?
-				#console.log "getOpsByVersionRange:", err, result
-				updates = []
-				opInRange = (op, from, to) ->
-					return false if fromVersion? and op.v < fromVersion
-					return false if toVersion? and op.v > toVersion
-					return true
-				for docHistory in result
-					#console.log 'adding', docHistory.pack
-					for op in docHistory.pack.reverse() when opInRange(op, fromVersion, toVersion)
-						op.project_id = docHistory.project_id
-						op.doc_id = docHistory.doc_id
-						#console.log "added op", op.v, fromVersion, toVersion
-						updates.push op
-				callback(null, updates)
-
-	loadPacksByVersionRange: (project_id, doc_id, fromVersion, toVersion, callback) ->
-		PackManager.getIndex doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			indexPacks = indexResult?.packs or []
-			packInRange = (pack, from, to) ->
-				return false if fromVersion? and pack.v_end < fromVersion
-				return false if toVersion? and pack.v > toVersion
-				return true
-			neededIds = (pack._id for pack in indexPacks when packInRange(pack, fromVersion, toVersion))
-			if neededIds.length
-				PackManager.fetchPacksIfNeeded project_id, doc_id, neededIds, callback
-			else
-				callback()
-
-	fetchPacksIfNeeded: (project_id, doc_id, pack_ids, callback) ->
-		db.docHistory.find {_id: {$in: (ObjectId(id) for id in pack_ids)}}, {_id:1}, (err, loadedPacks) ->
-			return callback(err) if err?
-			allPackIds = (id.toString() for id in pack_ids)
-			loadedPackIds = (pack._id.toString() for pack in loadedPacks)
-			packIdsToFetch = _.difference allPackIds, loadedPackIds
-			logger.log {project_id, doc_id, loadedPackIds, allPackIds, packIdsToFetch}, "analysed packs"
-			return callback() if packIdsToFetch.length is 0
-			async.eachLimit packIdsToFetch, 4, (pack_id, cb) ->
-				MongoAWS.unArchivePack project_id, doc_id, pack_id, cb
-			, (err) ->
-				return callback(err) if err?
-				logger.log {project_id, doc_id}, "done unarchiving"
-				callback()
-
-	# Retrieve all changes across a project
-
-	makeProjectIterator: (project_id, before, callback) ->
-		# get all the docHistory Entries
-		db.docHistory.find({project_id: ObjectId(project_id)},{pack:false}).sort {"meta.end_ts":-1}, (err, packs) ->
-			return callback(err) if err?
-			allPacks = []
-			seenIds = {}
-			for pack in packs
-				allPacks.push pack
-				seenIds[pack._id] = true
-			db.docHistoryIndex.find {project_id: ObjectId(project_id)}, (err, indexes) ->
-				return callback(err) if err?
-				for index in indexes
-					for pack in index.packs when not seenIds[pack._id]
-						pack.project_id = index.project_id
-						pack.doc_id = index._id
-						pack.fromIndex = true
-						allPacks.push pack
-						seenIds[pack._id] = true
-				callback(null, new ProjectIterator(allPacks, before, PackManager.getPackById))
-
-	getPackById: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findOne {_id: pack_id}, (err, pack) ->
-			return callback(err) if err?
-			if not pack?
-				MongoAWS.unArchivePack project_id, doc_id, pack_id, callback
-			else if pack.expiresAt? and pack.temporary is false
-				# we only need to touch the TTL when listing the changes in the project
-				# because diffs on individual documents are always done after that
-				PackManager.increaseTTL pack, callback
-				# only do this for cached packs, not temporary ones to avoid older packs
-				# being kept longer than newer ones (which messes up the last update version)
-			else
-				callback(null, pack)
-
-	increaseTTL: (pack, callback) ->
-		if pack.expiresAt < new Date(Date.now() + 6 * DAYS)
-			# update cache expiry since we are using this pack
-			db.docHistory.findAndModify {
-				query: {_id: pack._id}
-				update: {$set: {expiresAt: new Date(Date.now() + 7 * DAYS)}}
-			}, (err) ->
-				return callback(err, pack)
-		else
-			callback(null, pack)
-
-	# Manage docHistoryIndex collection
-
-	getIndex: (doc_id, callback) ->
-		db.docHistoryIndex.findOne {_id:ObjectId(doc_id.toString())}, callback
-
-	getPackFromIndex: (doc_id, pack_id, callback) ->
-		db.docHistoryIndex.findOne {_id:ObjectId(doc_id.toString()), "packs._id": pack_id}, {"packs.$":1}, callback
-
-	getLastPackFromIndex: (doc_id, callback) ->
-		db.docHistoryIndex.findOne {_id: ObjectId(doc_id.toString())}, {packs:{$slice:-1}}, (err, indexPack) ->
-			return callback(err) if err?
-			return callback() if not indexPack?
-			callback(null,indexPack[0])
-
-	getIndexWithKeys: (doc_id, callback) ->
-		PackManager.getIndex doc_id, (err, index) ->
-			return callback(err) if err?
-			return callback() if not index?
-			for pack in index?.packs or []
-				index[pack._id] = pack
-			callback(null, index)
-
-	initialiseIndex: (project_id, doc_id, callback) ->
-		PackManager.findCompletedPacks project_id, doc_id, (err, packs) ->
-			#console.log 'err', err, 'packs', packs, packs?.length
-			return callback(err) if err?
-			return callback() if not packs?
-			PackManager.insertPacksIntoIndexWithLock project_id, doc_id, packs, callback
-
-	updateIndex: (project_id, doc_id, callback) ->
-		# find all packs prior to current pack
-		PackManager.findUnindexedPacks project_id, doc_id, (err, newPacks) ->
-			return callback(err) if err?
-			return callback() if not newPacks? or newPacks.length is 0
-			PackManager.insertPacksIntoIndexWithLock project_id, doc_id, newPacks, (err) ->
-				return callback(err) if err?
-				logger.log {project_id, doc_id, newPacks}, "added new packs to index"
-				callback()
-
-	findCompletedPacks: (project_id, doc_id, callback) ->
-		query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} }
-		db.docHistory.find(query, {pack:false}).sort {v:1}, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?
-			return callback() if not packs?.length
-			last = packs.pop() # discard the last pack, if it's still in progress
-			packs.push(last) if last.finalised # it's finalised so we push it back to archive it
-			callback(null, packs)
-
-	findPacks: (project_id, doc_id, callback) ->
-		query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} }
-		db.docHistory.find(query, {pack:false}).sort {v:1}, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?
-			return callback() if not packs?.length
-			callback(null, packs)
-
-	findUnindexedPacks: (project_id, doc_id, callback) ->
-		PackManager.getIndexWithKeys doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			PackManager.findCompletedPacks project_id, doc_id, (err, historyPacks) ->
-				return callback(err) if err?
-				return callback() if not historyPacks?
-				# select only the new packs not already in the index
-				newPacks = (pack for pack in historyPacks when not indexResult?[pack._id]?)
-				newPacks = (_.omit(pack, 'doc_id', 'project_id', 'n', 'sz', 'last_checked', 'finalised') for pack in newPacks)
-				if newPacks.length
-					logger.log {project_id, doc_id, n: newPacks.length}, "found new packs"
-				callback(null, newPacks)
-
-	insertPacksIntoIndexWithLock: (project_id, doc_id, newPacks, callback) ->
-		LockManager.runWithLock(
-			keys.historyIndexLock({doc_id}),
-			(releaseLock) ->
-				PackManager._insertPacksIntoIndex project_id, doc_id, newPacks, releaseLock
-			callback
-		)
-
-	_insertPacksIntoIndex: (project_id, doc_id, newPacks, callback) ->
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString())}
-			update:
-				$setOnInsert: project_id: ObjectId(project_id.toString())
-				$push:
-					packs: {$each: newPacks, $sort: {v: 1}}
-			upsert: true
-		}, callback
-
-	# Archiving packs to S3
-
-	archivePack: (project_id, doc_id, pack_id, callback) ->
-		clearFlagOnError = (err, cb) ->
-			if err? # clear the inS3 flag on error
-				PackManager.clearPackAsArchiveInProgress project_id, doc_id, pack_id, (err2) ->
-					return cb(err2) if err2?
-					return cb(err)
-			else
-				cb()
-		async.series [
-			(cb) ->
-				PackManager.checkArchiveNotInProgress project_id, doc_id, pack_id, cb
-			(cb) ->
-				PackManager.markPackAsArchiveInProgress project_id, doc_id, pack_id, cb
-			(cb) ->
-				MongoAWS.archivePack project_id, doc_id, pack_id, (err) ->
-					clearFlagOnError(err, cb)
-			(cb) ->
-				PackManager.checkArchivedPack project_id, doc_id, pack_id, (err) ->
-					clearFlagOnError(err, cb)
-			(cb) ->
-				PackManager.markPackAsArchived project_id, doc_id, pack_id, cb
-			(cb) ->
-				PackManager.setTTLOnArchivedPack project_id, doc_id, pack_id, callback
-		], callback
-
-
-	checkArchivedPack: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findOne {_id: pack_id}, (err, pack) ->
-			return callback(err) if err?
-			return callback new Error("pack not found") if not pack?
-			MongoAWS.readArchivedPack project_id, doc_id, pack_id, (err, result) ->
-				delete result.last_checked
-				delete pack.last_checked
-				# need to compare ids as ObjectIds with .equals()
-				for key in ['_id', 'project_id', 'doc_id']
-					result[key] = pack[key] if result[key].equals(pack[key])
-				for op, i in result.pack
-					op._id = pack.pack[i]._id if op._id? and op._id.equals(pack.pack[i]._id)
-				if _.isEqual pack, result
-					callback()
-				else
-					logger.err {pack, result, jsondiff: JSON.stringify(pack) is JSON.stringify(result)}, "difference when comparing packs"
-					callback new Error("pack retrieved from s3 does not match pack in mongo")
-	# Extra methods to test archive/unarchive for a doc_id
-
-	pushOldPacks: (project_id, doc_id, callback) ->
-		PackManager.findPacks project_id, doc_id, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?.length
-			PackManager.processOldPack project_id, doc_id, packs[0]._id, callback
-
-	pullOldPacks: (project_id, doc_id, callback) ->
-		PackManager.loadPacksByVersionRange project_id, doc_id, null, null, callback
-
-
-	# Processing old packs via worker
-
-	processOldPack: (project_id, doc_id, pack_id, callback) ->
-		markAsChecked = (err) ->
-			PackManager.markPackAsChecked project_id, doc_id, pack_id, (err2) ->
-				return callback(err2) if err2?
-				callback(err)
-		logger.log {project_id, doc_id}, "processing old packs"
-		db.docHistory.findOne {_id:pack_id}, (err, pack) ->
-			return markAsChecked(err) if err?
-			return markAsChecked() if not pack?
-			return callback() if pack.expiresAt? # return directly
-			PackManager.finaliseIfNeeded project_id, doc_id, pack._id, pack, (err) ->
-				return markAsChecked(err) if err?
-				PackManager.updateIndexIfNeeded project_id, doc_id, (err) ->
-					return markAsChecked(err) if err?
-					PackManager.findUnarchivedPacks project_id, doc_id, (err, unarchivedPacks) ->
-						return markAsChecked(err) if err?
-						if not unarchivedPacks?.length
-							logger.log {project_id, doc_id}, "no packs need archiving"
-							return markAsChecked()
-						async.eachSeries unarchivedPacks, (pack, cb) ->
-							PackManager.archivePack project_id, doc_id, pack._id, cb
-						, (err) ->
-							return markAsChecked(err) if err?
-							logger.log  {project_id, doc_id}, "done processing"
-							markAsChecked()
-
-	finaliseIfNeeded: (project_id, doc_id, pack_id, pack, callback) ->
-		sz = pack.sz / (1024 * 1024) # in fractions of a megabyte
-		n = pack.n / 1024 # in fraction of 1024 ops
-		age = (Date.now() - pack.meta.end_ts) / DAYS
-		if age < 30 # always keep if less than 1 month old
-			logger.log {project_id, doc_id, pack_id, age}, "less than 30 days old"
-			return callback()
-		# compute an archiving threshold which decreases for each month of age
-		archive_threshold = 30 / age
-		if sz > archive_threshold or n > archive_threshold or age > 90
-			logger.log {project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "meets archive threshold"
-			PackManager.markPackAsFinalisedWithLock project_id, doc_id, pack_id, callback
-		else
-			logger.log {project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "does not meet archive threshold"
-			callback()
-
-	markPackAsFinalisedWithLock: (project_id, doc_id, pack_id, callback) ->
-		LockManager.runWithLock(
-			keys.historyLock({doc_id}),
-			(releaseLock) ->
-				PackManager._markPackAsFinalised project_id, doc_id, pack_id, releaseLock
-			callback
-		)
-
-	_markPackAsFinalised: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as finalised"
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
-			update: {$set: {finalised: true}}
-		}, callback
-
-
-	updateIndexIfNeeded: (project_id, doc_id, callback) ->
-		logger.log {project_id, doc_id}, "archiving old packs"
-		PackManager.getIndexWithKeys doc_id, (err, index) ->
-			return callback(err) if err?
-			if not index?
-				PackManager.initialiseIndex project_id, doc_id, callback
-			else
-				PackManager.updateIndex project_id, doc_id, callback
-
-	markPackAsChecked: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as checked"
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
-			update: {$currentDate: {"last_checked":true}}
-		}, callback
-
-	findUnarchivedPacks: (project_id, doc_id, callback) ->
-		PackManager.getIndex doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			indexPacks = indexResult?.packs or []
-			unArchivedPacks = (pack for pack in indexPacks when not pack.inS3?)
-			if unArchivedPacks.length
-				logger.log {project_id, doc_id, n: unArchivedPacks.length}, "find unarchived packs"
-			callback(null, unArchivedPacks)
-
-	# Archive locking flags
-
-	checkArchiveNotInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "checking if archive in progress"
-		PackManager.getPackFromIndex doc_id, pack_id, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("pack not found in index") if not result?
-			if result.inS3
-				return callback new Error("pack archiving already done")
-			else if result.inS3?
-				return callback new Error("pack archiving already in progress")
-			else
-				return callback()
-
-	markPackAsArchiveInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id}, "marking pack as archive in progress status"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()),  packs: {$elemMatch: {"_id": pack_id, inS3: {$exists:false}}}}
-			fields:  { "packs.$": 1 }
-			update: {$set: {"packs.$.inS3":false}}
-		}, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("archive is already in progress") if not result?
-			logger.log {project_id, doc_id, pack_id}, "marked as archive in progress"
-			callback()
-
-	clearPackAsArchiveInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "clearing as archive in progress"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}}
-			fields:  { "packs.$": 1 }
-			update: {$unset: {"packs.$.inS3":true}}
-		}, callback
-
-	markPackAsArchived: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as archived"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}}
-			fields: { "packs.$": 1 }
-			update: {$set: {"packs.$.inS3":true}}
-		}, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("archive is not marked as progress") if not result?
-			logger.log {project_id, doc_id, pack_id}, "marked as archived"
-			callback()
-
-	setTTLOnArchivedPack: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
-			update: {$set: {expiresAt: new Date(Date.now() + 1*DAYS)}}
-		}, (err) ->
-			logger.log {project_id, doc_id, pack_id}, "set expiry on pack"
-			callback()
-
-
-#	_getOneDayInFutureWithRandomDelay: ->
-#		thirtyMins = 1000 * 60 * 30
-#		randomThirtyMinMax = Math.ceil(Math.random() * thirtyMins)
-#		return new Date(Date.now() + randomThirtyMinMax + 1*DAYS)

services/track-changes/app/coffee/PackWorker.coffee

@@ -1,139 +0,0 @@
-Settings = require "settings-sharelatex"
-async = require "async"
-_ = require "underscore"
-{db, ObjectId, BSON} = require "./mongojs"
-fs = require "fs"
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("track-changes")
-logger = require "logger-sharelatex"
-logger.initialize("track-changes-packworker")
-if Settings.sentry?.dsn?
-	logger.initializeErrorReporting(Settings.sentry.dsn)
-
-DAYS = 24 * 3600 * 1000
-
-LockManager = require "./LockManager"
-PackManager = require "./PackManager"
-
-# this worker script is forked by the main process to look for
-# document histories which can be archived
-
-source = process.argv[2]
-DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
-TIMEOUT = Number(process.argv[4]) || 30*60*1000
-COUNT = 0  # number processed
-TOTAL = 0  # total number to process
-
-if !source.match(/^[0-9]+$/)
-	file = fs.readFileSync source
-	result = for line in file.toString().split('\n')
-		[project_id, doc_id] = line.split(' ')
-		{doc_id, project_id}
-	pending = _.filter result, (row) -> row?.doc_id?.match(/^[a-f0-9]{24}$/)
-else
-	LIMIT = Number(process.argv[2]) || 1000
-
-shutDownRequested = false
-shutDownTimer = setTimeout () ->
-	logger.log "pack timed out, requesting shutdown"
-	# start the shutdown on the next pack
-	shutDownRequested = true
-	# do a hard shutdown after a further 5 minutes
-	hardTimeout = setTimeout () ->
-		logger.error "HARD TIMEOUT in pack archive worker"
-		process.exit()
-	, 5*60*1000
-	hardTimeout.unref()
-, TIMEOUT
-
-logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}"
-
-# work around for https://github.com/mafintosh/mongojs/issues/224
-db.close =  (callback) ->
-	this._getServer (err, server) ->
-		return callback(err) if err?
-		server = if server.destroy? then server else server.topology
-		server.destroy(true, true)
-		callback()
-
-finish = () ->
-	if shutDownTimer?
-		logger.log 'cancelling timeout'
-		clearTimeout shutDownTimer
-	logger.log 'closing db'
-	db.close () ->
-		logger.log 'closing LockManager Redis Connection'
-		LockManager.close () ->
-			logger.log {processedCount: COUNT, allCount: TOTAL}, 'ready to exit from pack archive worker'
-			hardTimeout = setTimeout () ->
-				logger.error 'hard exit from pack archive worker'
-				process.exit(1)
-			, 5*1000
-			hardTimeout.unref()
-
-process.on 'exit', (code) ->
-	logger.log {code}, 'pack archive worker exited'
-
-processUpdates = (pending) ->
-	async.eachSeries pending,	(result, callback) ->
-		{_id, project_id, doc_id} = result
-		COUNT++
-		logger.log {project_id, doc_id}, "processing #{COUNT}/#{TOTAL}"
-		if not project_id? or not doc_id?
-			logger.log {project_id, doc_id}, "skipping pack, missing project/doc id"
-			return callback()
-		handler = (err, result) ->
-			if err? and err.code is "InternalError" and err.retryable
-				logger.warn {err, result}, "ignoring S3 error in pack archive worker"
-				# Ignore any s3 errors due to random problems
-				err = null
-			if err?
-				logger.error {err, result}, "error in pack archive worker"
-				return callback(err)
-			if shutDownRequested
-				logger.warn "shutting down pack archive worker"
-				return callback(new Error("shutdown"))
-			setTimeout () ->
-				callback(err, result)
-			, DOCUMENT_PACK_DELAY
-		if not _id?
-			PackManager.pushOldPacks project_id, doc_id, handler
-		else
-			PackManager.processOldPack project_id, doc_id, _id, handler
-	, (err, results) ->
-		if err? and err.message != "shutdown"
-			logger.error {err}, 'error in pack archive worker processUpdates'
-		finish()
-
-# find the packs which can be archived
-
-ObjectIdFromDate =  (date) ->
-	id = Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
-	return ObjectId(id)
-
-# new approach, two passes
-# find packs to be marked as finalised:true, those which have a newer pack present
-# then only consider finalised:true packs for archiving
-
-if pending?
-	logger.log "got #{pending.length} entries from #{source}"
-	processUpdates pending
-else
-	oneWeekAgo = new Date(Date.now() - 7 * DAYS)
-	db.docHistory.find({
-		expiresAt: {$exists: false}
-		project_id: {$exists: true}
-		v_end: {$exists: true}
-		_id: {$lt: ObjectIdFromDate(oneWeekAgo)}
-		last_checked: {$lt: oneWeekAgo}
-	}, {_id:1, doc_id:1, project_id:1}).sort({
-		last_checked:1
-	}).limit LIMIT, (err, results) ->
-		if err?
-			logger.log {err}, 'error checking for updates'
-			finish()
-			return
-		pending = _.uniq results, false, (result) -> result.doc_id.toString()
-		TOTAL = pending.length
-		logger.log "found #{TOTAL} documents to archive"
-		processUpdates pending

services/track-changes/app/coffee/ProjectIterator.coffee

@@ -1,62 +0,0 @@
-Heap = require "heap"
-
-module.exports = ProjectIterator =
-
-	class ProjectIterator
-		constructor: (packs, @before, @getPackByIdFn) ->
-			byEndTs = (a,b) -> (b.meta.end_ts - a.meta.end_ts) || (a.fromIndex - b.fromIndex)
-			@packs = packs.slice().sort byEndTs
-			@queue = new Heap(byEndTs)
-
-		next: (callback) ->
-			#  what's up next
-			#console.log ">>> top item", iterator.packs[0]
-			iterator = this
-			before = @before
-			queue = iterator.queue
-			opsToReturn = []
-			nextPack = iterator.packs[0]
-			lowWaterMark = nextPack?.meta.end_ts || 0
-			nextItem = queue.peek()
-
-			#console.log "queue empty?", queue.empty()
-			#console.log "nextItem", nextItem
-			#console.log "nextItem.meta.end_ts", nextItem?.meta.end_ts
-			#console.log "lowWaterMark", lowWaterMark
-
-			while before? and nextPack?.meta.start_ts > before
-				# discard pack that is outside range
-				iterator.packs.shift()
-				nextPack = iterator.packs[0]
-				lowWaterMark = nextPack?.meta.end_ts || 0
-
-			if (queue.empty() or nextItem?.meta.end_ts <= lowWaterMark) and nextPack?
-				# retrieve the next pack and populate the queue
-				return @getPackByIdFn nextPack.project_id, nextPack.doc_id, nextPack._id, (err, pack) ->
-					return callback(err) if err?
-					iterator.packs.shift() # have now retrieved this pack, remove it
-					#console.log "got pack", pack
-					for op in pack.pack when (not before? or op.meta.end_ts < before)
-						#console.log "adding op", op
-						op.doc_id = nextPack.doc_id
-						op.project_id = nextPack.project_id
-						queue.push op
-					# now try again
-					return iterator.next(callback)
-
-			#console.log "nextItem", nextItem, "lowWaterMark", lowWaterMark
-			while nextItem? and (nextItem?.meta.end_ts > lowWaterMark)
-				opsToReturn.push nextItem
-				queue.pop()
-				nextItem = queue.peek()
-
-			#console.log "queue empty?", queue.empty()
-			#console.log "nextPack", nextPack?
-
-			if queue.empty() and not nextPack? # got everything
-				iterator._done = true
-
-			callback(null, opsToReturn)
-
-		done: () ->
-			return @_done

services/track-changes/app/coffee/RedisManager.coffee

@@ -1,80 +0,0 @@
-Settings = require "settings-sharelatex"
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.history)
-Keys = Settings.redis.history.key_schema
-async = require "async"
-
-module.exports = RedisManager =
-
-	getOldestDocUpdates: (doc_id, batchSize, callback = (error, jsonUpdates) ->) ->
-		key = Keys.uncompressedHistoryOps({doc_id})
-		rclient.lrange key, 0, batchSize - 1, callback
-
-	expandDocUpdates: (jsonUpdates, callback = (error, rawUpdates) ->) ->
-		try
-			rawUpdates = ( JSON.parse(update) for update in jsonUpdates or [] )
-		catch e
-			return callback(e)
-		callback null, rawUpdates
-
-	deleteAppliedDocUpdates: (project_id, doc_id, docUpdates, callback = (error) ->) ->
-		multi = rclient.multi()
-		# Delete all the updates which have been applied (exact match)
-		for update in docUpdates or []
-			multi.lrem Keys.uncompressedHistoryOps({doc_id}), 1, update
-		multi.exec (error, results) ->
-			return callback(error) if error?
-			# It's ok to delete the doc_id from the set here. Even though the list
-			# of updates may not be empty, we will continue to process it until it is.
-			rclient.srem Keys.docsWithHistoryOps({project_id}), doc_id, (error) ->
-				return callback(error) if error?
-				callback null
-
-	getDocIdsWithHistoryOps: (project_id, callback = (error, doc_ids) ->) ->
-		rclient.smembers Keys.docsWithHistoryOps({project_id}), callback
-
-	# iterate over keys asynchronously using redis scan (non-blocking)
-	# handle all the cluster nodes or single redis server
-	_getKeys: (pattern, callback) ->
-		nodes = rclient.nodes?('master') || [ rclient ];
-		doKeyLookupForNode = (node, cb) ->
-			RedisManager._getKeysFromNode node, pattern, cb
-		async.concatSeries nodes, doKeyLookupForNode, callback
-
-	_getKeysFromNode: (node, pattern, callback) ->
-		cursor = 0  # redis iterator
-		keySet = {} # use hash to avoid duplicate results
-		# scan over all keys looking for pattern
-		doIteration = (cb) ->
-			node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) ->
-				return callback(error) if error?
-				[cursor, keys] = reply
-				for key in keys
-					keySet[key] = true
-				if cursor == '0'  # note redis returns string result not numeric
-					return callback(null, Object.keys(keySet))
-				else
-					doIteration()
-		doIteration()
-
-	# extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b
-	# or DocsWithHistoryOps:{57fd0b1f53a8396d22b2c24b} (for redis cluster)
-	_extractIds: (keyList) ->
-		ids = for key in keyList
-			m = key.match(/:\{?([0-9a-f]{24})\}?/) # extract object id
-			m[1]
-		return ids
-
-	getProjectIdsWithHistoryOps: (callback = (error, project_ids) ->) ->
-		RedisManager._getKeys Keys.docsWithHistoryOps({project_id:"*"}), (error, project_keys) ->
-			return callback(error) if error?
-			project_ids = RedisManager._extractIds project_keys
-			callback(error, project_ids)
-
-	getAllDocIdsWithHistoryOps: (callback = (error, doc_ids) ->) ->
-		# return all the docids, to find dangling history entries after
-		# everything is flushed.
-		RedisManager._getKeys Keys.uncompressedHistoryOps({doc_id:"*"}), (error, doc_keys) ->
-			return callback(error) if error?
-			doc_ids = RedisManager._extractIds doc_keys
-			callback(error, doc_ids)

services/track-changes/app/coffee/RestoreManager.coffee

@@ -1,12 +0,0 @@
-DocumentUpdaterManager = require "./DocumentUpdaterManager"
-DiffManager = require "./DiffManager"
-logger = require "logger-sharelatex"
-
-module.exports = RestoreManager =
-	restoreToBeforeVersion: (project_id, doc_id, version, user_id, callback = (error) ->) ->
-		logger.log project_id: project_id, doc_id: doc_id, version: version, user_id: user_id, "restoring document"
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, version, (error, content) ->
-			return callback(error) if error?
-			DocumentUpdaterManager.setDocument project_id, doc_id, content, user_id, (error) ->
-				return callback(error) if error?
-				callback()

services/track-changes/app/coffee/UpdateCompressor.coffee

@@ -1,218 +0,0 @@
-strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..]
-strRemove = (s1, pos, length) -> s1[...pos] + s1[(pos + length)..]
-
-diff_match_patch = require("../lib/diff_match_patch").diff_match_patch
-dmp = new diff_match_patch()
-
-module.exports = UpdateCompressor =
-	NOOP: "noop"
-
-	# Updates come from the doc updater in format
-	# {
-	# 	op:   [ { ... op1 ... }, { ... op2 ... } ]
-	# 	meta: { ts: ..., user_id: ... }
-	# }
-	# but it's easier to work with on op per update, so convert these updates to
-	# our compressed format
-	# [{
-	# 	op: op1
-	# 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
-	# }, {
-	# 	op: op2
-	# 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
-	# }]
-	convertToSingleOpUpdates: (updates) ->
-		splitUpdates = []
-		for update in updates
-			# Reject any non-insert or delete ops, i.e. comments
-			ops = update.op.filter (o) -> o.i? or o.d?
-			if ops.length == 0
-				splitUpdates.push
-					op: UpdateCompressor.NOOP
-					meta:
-						start_ts: update.meta.start_ts or update.meta.ts
-						end_ts:   update.meta.end_ts   or update.meta.ts
-						user_id:  update.meta.user_id
-					v: update.v
-			else
-				for op in ops
-					splitUpdates.push
-						op: op
-						meta:
-							start_ts: update.meta.start_ts or update.meta.ts
-							end_ts:   update.meta.end_ts   or update.meta.ts
-							user_id:  update.meta.user_id
-						v: update.v
-		return splitUpdates
-
-	concatUpdatesWithSameVersion: (updates) ->
-		concattedUpdates = []
-		for update in updates
-			lastUpdate = concattedUpdates[concattedUpdates.length - 1]
-			if lastUpdate? and lastUpdate.v == update.v
-				lastUpdate.op.push update.op unless update.op == UpdateCompressor.NOOP
-			else
-				nextUpdate =
-					op:   []
-					meta: update.meta
-					v:    update.v
-				nextUpdate.op.push update.op unless update.op == UpdateCompressor.NOOP
-				concattedUpdates.push nextUpdate
-		return concattedUpdates
-
-	compressRawUpdates: (lastPreviousUpdate, rawUpdates) ->
-		if lastPreviousUpdate?.op?.length > 1
-			# if the last previous update was an array op, don't compress onto it.
-			# The avoids cases where array length changes but version number doesn't
-			return [lastPreviousUpdate].concat UpdateCompressor.compressRawUpdates(null,rawUpdates)
-		if lastPreviousUpdate?
-			rawUpdates = [lastPreviousUpdate].concat(rawUpdates)
-		updates = UpdateCompressor.convertToSingleOpUpdates(rawUpdates)
-		updates = UpdateCompressor.compressUpdates(updates)
-		return UpdateCompressor.concatUpdatesWithSameVersion(updates)
-
-	compressUpdates: (updates) ->
-		return [] if updates.length == 0
-
-		compressedUpdates = [updates.shift()]
-		for update in updates
-			lastCompressedUpdate = compressedUpdates.pop()
-			if lastCompressedUpdate?
-				compressedUpdates = compressedUpdates.concat UpdateCompressor._concatTwoUpdates lastCompressedUpdate, update
-			else
-				compressedUpdates.push update
-
-		return compressedUpdates
-
-	MAX_TIME_BETWEEN_UPDATES: oneMinute = 60 * 1000
-	MAX_UPDATE_SIZE: twoMegabytes = 2* 1024 * 1024
-
-	_concatTwoUpdates: (firstUpdate, secondUpdate) ->
-		firstUpdate =
-			op: firstUpdate.op
-			meta:
-				user_id:  firstUpdate.meta.user_id or null
-				start_ts: firstUpdate.meta.start_ts or firstUpdate.meta.ts
-				end_ts:   firstUpdate.meta.end_ts   or firstUpdate.meta.ts
-			v: firstUpdate.v
-		secondUpdate =
-			op: secondUpdate.op
-			meta:
-				user_id:  secondUpdate.meta.user_id or null
-				start_ts: secondUpdate.meta.start_ts or secondUpdate.meta.ts
-				end_ts:   secondUpdate.meta.end_ts   or secondUpdate.meta.ts
-			v: secondUpdate.v
-
-		if firstUpdate.meta.user_id != secondUpdate.meta.user_id
-			return [firstUpdate, secondUpdate]
-
-		if secondUpdate.meta.start_ts - firstUpdate.meta.end_ts > UpdateCompressor.MAX_TIME_BETWEEN_UPDATES
-			return [firstUpdate, secondUpdate]
-
-		firstOp = firstUpdate.op
-		secondOp = secondUpdate.op
-
-		firstSize = firstOp.i?.length or firstOp.d?.length
-		secondSize = secondOp.i?.length or secondOp.d?.length
-
-		# Two inserts
-		if firstOp.i? and secondOp.i? and firstOp.p <= secondOp.p <= (firstOp.p + firstOp.i.length) and firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
-			return [
-				meta:
-					start_ts: firstUpdate.meta.start_ts
-					end_ts:   secondUpdate.meta.end_ts
-					user_id:  firstUpdate.meta.user_id
-				op:
-					p: firstOp.p
-					i: strInject(firstOp.i, secondOp.p - firstOp.p, secondOp.i)
-				v: secondUpdate.v
-			]
-		# Two deletes
-		else if firstOp.d? and secondOp.d? and secondOp.p <= firstOp.p <= (secondOp.p + secondOp.d.length) and firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
-			return [
-				meta:
-					start_ts: firstUpdate.meta.start_ts
-					end_ts:   secondUpdate.meta.end_ts
-					user_id:  firstUpdate.meta.user_id
-				op:
-					p: secondOp.p
-					d: strInject(secondOp.d, firstOp.p - secondOp.p, firstOp.d)
-				v: secondUpdate.v
-			]
-		# An insert and then a delete
-		else if firstOp.i? and secondOp.d? and firstOp.p <= secondOp.p <= (firstOp.p + firstOp.i.length)
-			offset = secondOp.p - firstOp.p
-			insertedText = firstOp.i.slice(offset, offset + secondOp.d.length)
-			# Only trim the insert when the delete is fully contained within in it
-			if insertedText == secondOp.d
-				insert = strRemove(firstOp.i, offset, secondOp.d.length)
-				return [
-					meta:
-						start_ts: firstUpdate.meta.start_ts
-						end_ts:   secondUpdate.meta.end_ts
-						user_id:  firstUpdate.meta.user_id
-					op:
-						p: firstOp.p
-						i: insert
-					v: secondUpdate.v
-				]
-			else
-				# This will only happen if the delete extends outside the insert
-				return [firstUpdate, secondUpdate]
-
-		# A delete then an insert at the same place, likely a copy-paste of a chunk of content
-		else if firstOp.d? and secondOp.i? and firstOp.p == secondOp.p
-			offset = firstOp.p
-			diff_ops = @diffAsShareJsOps(firstOp.d, secondOp.i)
-			if diff_ops.length == 0
-				return [{ # Noop
-					meta:
-						start_ts: firstUpdate.meta.start_ts
-						end_ts:   secondUpdate.meta.end_ts
-						user_id:  firstUpdate.meta.user_id
-					op:
-						p: firstOp.p
-						i: ""
-					v: secondUpdate.v
-				}]
-			else
-				return diff_ops.map (op) ->
-					op.p += offset
-					return {
-						meta:
-							start_ts: firstUpdate.meta.start_ts
-							end_ts:   secondUpdate.meta.end_ts
-							user_id:  firstUpdate.meta.user_id
-						op: op
-						v: secondUpdate.v
-					}
-
-		else
-			return [firstUpdate, secondUpdate]
-
-	ADDED: 1
-	REMOVED: -1
-	UNCHANGED: 0
-	diffAsShareJsOps: (before, after, callback = (error, ops) ->) ->
-		diffs = dmp.diff_main(before, after)
-		dmp.diff_cleanupSemantic(diffs)
-
-		ops = []
-		position = 0
-		for diff in diffs
-			type = diff[0]
-			content = diff[1]
-			if type == @ADDED
-				ops.push
-					i: content
-					p: position
-				position += content.length
-			else if type == @REMOVED
-				ops.push
-					d: content
-					p: position
-			else if type == @UNCHANGED
-				position += content.length
-			else
-				throw "Unknown type"
-		return ops

services/track-changes/app/coffee/UpdateTrimmer.coffee

@@ -1,23 +0,0 @@
-MongoManager = require "./MongoManager"
-WebApiManager = require "./WebApiManager"
-logger = require "logger-sharelatex"
-
-module.exports = UpdateTrimmer =
-	shouldTrimUpdates: (project_id, callback = (error, shouldTrim) ->) ->
-		MongoManager.getProjectMetaData project_id, (error, metadata) ->
-			return callback(error) if error?
-			if metadata?.preserveHistory
-				return callback null, false
-			else
-				WebApiManager.getProjectDetails project_id, (error, details) ->
-					return callback(error) if error?
-					logger.log project_id: project_id, details: details, "got details"
-					if details?.features?.versioning
-						MongoManager.setProjectMetaData project_id, preserveHistory: true, (error) ->
-							return callback(error) if error?
-							MongoManager.upgradeHistory project_id, (error) ->
-								return callback(error) if error?
-								callback null, false
-					else
-						callback null, true
-

services/track-changes/app/coffee/UpdatesManager.coffee

@@ -1,344 +0,0 @@
-MongoManager = require "./MongoManager"
-PackManager = require "./PackManager"
-RedisManager = require "./RedisManager"
-UpdateCompressor = require "./UpdateCompressor"
-LockManager = require "./LockManager"
-WebApiManager = require "./WebApiManager"
-UpdateTrimmer = require "./UpdateTrimmer"
-logger = require "logger-sharelatex"
-async = require "async"
-_ = require "underscore"
-Settings = require "settings-sharelatex"
-keys = Settings.redis.lock.key_schema
-
-module.exports = UpdatesManager =
-	compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) ->
-		length = rawUpdates.length
-		if length == 0
-			return callback()
-
-		# check that ops are in the correct order
-		for op, i in rawUpdates when i > 0
-			thisVersion = op?.v
-			prevVersion = rawUpdates[i-1]?.v
-			if not (prevVersion < thisVersion)
-				logger.error project_id: project_id, doc_id: doc_id, rawUpdates:rawUpdates, temporary: temporary, thisVersion:thisVersion, prevVersion:prevVersion, "op versions out of order"
-
-		# FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
-		# CORRECTION:  we do use it to log the time in case of error
-		MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
-			# lastCompressedUpdate is the most recent update in Mongo, and
-			# lastVersion is its sharejs version number.
-			#
-			# The peekLastCompressedUpdate method may pass the update back
-			# as 'null' (for example if the previous compressed update has
-			# been archived).  In this case it can still pass back the
-			# lastVersion from the update to allow us to check consistency.
-			return callback(error) if error?
-
-			# Ensure that raw updates start where lastVersion left off
-			if lastVersion?
-				discardedUpdates = []
-				rawUpdates = rawUpdates.slice(0)
-				while rawUpdates[0]? and rawUpdates[0].v <= lastVersion
-					discardedUpdates.push rawUpdates.shift()
-				if discardedUpdates.length
-					logger.error project_id: project_id, doc_id: doc_id, discardedUpdates: discardedUpdates, temporary: temporary, lastVersion: lastVersion, "discarded updates already present"
-
-				if rawUpdates[0]? and rawUpdates[0].v != lastVersion + 1
-					ts = lastCompressedUpdate?.meta?.end_ts
-					last_timestamp = if ts? then new Date(ts) else 'unknown time'
-					error = new Error("Tried to apply raw op at version #{rawUpdates[0].v} to last compressed update with version #{lastVersion} from #{last_timestamp}")
-					logger.error err: error, doc_id: doc_id, project_id: project_id, prev_end_ts: ts, temporary: temporary, lastCompressedUpdate: lastCompressedUpdate, "inconsistent doc versions"
-					if Settings.trackchanges?.continueOnError and rawUpdates[0].v > lastVersion + 1
-						# we have lost some ops - continue to write into the database, we can't recover at this point
-						lastCompressedUpdate = null
-					else
-						return callback error
-
-			if rawUpdates.length == 0
-				return callback()
-
-			# some old large ops in redis need to be rejected, they predate
-			# the size limit that now prevents them going through the system
-			REJECT_LARGE_OP_SIZE = 4 * 1024 * 1024
-			for rawUpdate in rawUpdates
-				opSizes = ((op.i?.length || op.d?.length) for op in rawUpdate?.op or [])
-				size = _.max opSizes
-				if size > REJECT_LARGE_OP_SIZE
-					error = new Error("dropped op exceeding maximum allowed size of #{REJECT_LARGE_OP_SIZE}")
-					logger.error err: error, doc_id: doc_id, project_id: project_id, size: size, rawUpdate: rawUpdate, "dropped op - too big"
-					rawUpdate.op = []
-
-			compressedUpdates = UpdateCompressor.compressRawUpdates null, rawUpdates
-			PackManager.insertCompressedUpdates project_id, doc_id, lastCompressedUpdate, compressedUpdates, temporary, (error, result) ->
-				return callback(error) if error?
-				logger.log {project_id, doc_id, orig_v: lastCompressedUpdate?.v, new_v: result.v}, "inserted updates into pack"	if result?
-				callback()
-
-	# Check whether the updates are temporary (per-project property)
-	_prepareProjectForUpdates: (project_id, callback = (error, temporary) ->) ->
-		UpdateTrimmer.shouldTrimUpdates project_id, (error, temporary) ->
-			return callback(error) if error?
-			callback(null, temporary)
-
-	# Check for project id on document history (per-document property)
-	_prepareDocForUpdates: (project_id, doc_id, callback = (error) ->) ->
-		MongoManager.backportProjectId project_id, doc_id, (error) ->
-			return callback(error) if error?
-			callback(null)
-
-	# Apply updates for specific project/doc after preparing at project and doc level
-	REDIS_READ_BATCH_SIZE: 100
-	processUncompressedUpdates: (project_id, doc_id, temporary, callback = (error) ->) ->
-		# get the updates as strings from redis (so we can delete them after they are applied)
-		RedisManager.getOldestDocUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, docUpdates) ->
-			return callback(error) if error?
-			length = docUpdates.length
-			# parse the redis strings into ShareJs updates
-			RedisManager.expandDocUpdates docUpdates, (error, rawUpdates) ->
-				if error?
-					logger.err project_id: project_id, doc_id: doc_id, docUpdates: docUpdates, "failed to parse docUpdates"
-					return callback(error)
-				logger.log project_id: project_id, doc_id: doc_id, rawUpdates: rawUpdates, "retrieved raw updates from redis"
-				UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, temporary, (error) ->
-					return callback(error) if error?
-					logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates"
-					# delete the applied updates from redis
-					RedisManager.deleteAppliedDocUpdates project_id, doc_id, docUpdates, (error) ->
-						return callback(error) if error?
-						if length == UpdatesManager.REDIS_READ_BATCH_SIZE
-							# There might be more updates
-							logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates"
-							setTimeout () ->
-								UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, callback
-							, 0
-						else
-							logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed"
-							callback()
-
-	# Process updates for a doc when we flush it individually
-	processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
-		UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
-			return callback(error) if error?
-			UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, callback
-
-
-	# Process updates for a doc when the whole project is flushed (internal method)
-	_processUncompressedUpdatesForDocWithLock: (project_id, doc_id, temporary, callback = (error) ->) ->
-		UpdatesManager._prepareDocForUpdates project_id, doc_id, (error) ->
-			return callback(error) if error?
-			LockManager.runWithLock(
-				keys.historyLock({doc_id}),
-				(releaseLock) ->
-					UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, releaseLock
-				callback
-			)
-
-	# Process all updates for a project, only check project-level information once
-	processUncompressedUpdatesForProject: (project_id, callback = (error) ->) ->
-		RedisManager.getDocIdsWithHistoryOps project_id, (error, doc_ids) ->
-			return callback(error) if error?
-			UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
-				jobs = []
-				for doc_id in doc_ids
-					do (doc_id) ->
-						jobs.push (cb) ->
-							UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, cb
-				async.parallelLimit jobs, 5, callback
-
-	# flush all outstanding changes
-	flushAll: (limit, callback = (error, result) ->) ->
-		RedisManager.getProjectIdsWithHistoryOps (error, project_ids) ->
-			return callback(error) if error?
-			logger.log {count: project_ids?.length, project_ids: project_ids}, "found projects"
-			jobs = []
-			project_ids = _.shuffle project_ids # randomise to avoid hitting same projects each time
-			selectedProjects = if limit < 0 then project_ids else project_ids[0...limit]
-			for project_id in selectedProjects
-				do (project_id) ->
-					jobs.push (cb) ->
-						UpdatesManager.processUncompressedUpdatesForProject project_id, (err) ->
-							return cb(null, {failed: err?, project_id: project_id})
-			async.series jobs, (error, result) ->
-				return callback(error) if error?
-				failedProjects = (x.project_id for x in result when x.failed)
-				succeededProjects = (x.project_id for x in result when not x.failed)
-				callback(null, {failed: failedProjects, succeeded: succeededProjects, all: project_ids})
-
-	getDanglingUpdates: (callback = (error, doc_ids) ->) ->
-		RedisManager.getAllDocIdsWithHistoryOps (error, all_doc_ids) ->
-			return callback(error) if error?
-			RedisManager.getProjectIdsWithHistoryOps (error, all_project_ids) ->
-				return callback(error) if error?
-				# function to get doc_ids for each project
-				task = (cb) -> async.concatSeries all_project_ids, RedisManager.getDocIdsWithHistoryOps, cb
-				# find the dangling doc ids
-				task (error, project_doc_ids) ->
-					dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids)
-					logger.log {all_doc_ids: all_doc_ids, all_project_ids: all_project_ids, project_doc_ids: project_doc_ids, dangling_doc_ids: dangling_doc_ids}, "checking for dangling doc ids"
-					callback(null, dangling_doc_ids)
-
-	getDocUpdates: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
-		UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
-			return callback(error) if error?
-			#console.log "options", options
-			PackManager.getOpsByVersionRange project_id, doc_id, options.from, options.to, (error, updates) ->
-				return callback(error) if error?
-				callback null, updates
-
-	getDocUpdatesWithUserInfo: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
-		UpdatesManager.getDocUpdates project_id, doc_id, options, (error, updates) ->
-			return callback(error) if error?
-			UpdatesManager.fillUserInfo updates, (error, updates) ->
-				return callback(error) if error?
-				callback null, updates
-
-	getSummarizedProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
-		options.min_count ||= 25
-		summarizedUpdates = []
-		before = options.before
-		nextBeforeTimestamp = null
-		UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
-			return callback(error) if error?
-			PackManager.makeProjectIterator project_id, before, (err, iterator) ->
-				return callback(err) if err?
-				# repeatedly get updates and pass them through the summariser to get an final output with user info
-				async.whilst () ->
-					#console.log "checking iterator.done", iterator.done()
-					return summarizedUpdates.length < options.min_count and not iterator.done()
-				, (cb) ->
-					iterator.next (err, partialUpdates) ->
-						return callback(err) if err?
-						#logger.log {partialUpdates}, 'got partialUpdates'
-						return cb() if partialUpdates.length is 0 ## FIXME should try to avoid this happening
-						nextBeforeTimestamp = partialUpdates[partialUpdates.length - 1].meta.end_ts
-						# add the updates to the summary list
-						summarizedUpdates = UpdatesManager._summarizeUpdates partialUpdates, summarizedUpdates
-						cb()
-				, () ->
-					# finally done all updates
-					#console.log 'summarized Updates', summarizedUpdates
-					UpdatesManager.fillSummarizedUserInfo summarizedUpdates, (err, results) ->
-						return callback(err) if err?
-						callback null, results,	if not iterator.done() then nextBeforeTimestamp else undefined
-
-	fetchUserInfo: (users, callback = (error, fetchedUserInfo) ->) ->
-		jobs = []
-		fetchedUserInfo = {}
-		for user_id of users
-			do (user_id) ->
-				jobs.push (callback) ->
-					WebApiManager.getUserInfo user_id, (error, userInfo) ->
-						return callback(error) if error?
-						fetchedUserInfo[user_id] = userInfo
-						callback()
-
-		async.series jobs, (err) ->
-			return callback(err) if err?
-			callback(null, fetchedUserInfo)
-
-	fillUserInfo: (updates, callback = (error, updates) ->) ->
-		users = {}
-		for update in updates
-			user_id = update.meta.user_id
-			if UpdatesManager._validUserId(user_id)
-				users[user_id] = true
-
-		UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
-			return callback(error) if error?
-			for update in updates
-				user_id = update.meta.user_id
-				delete update.meta.user_id
-				if UpdatesManager._validUserId(user_id)
-					update.meta.user = fetchedUserInfo[user_id]
-			callback null, updates
-
-	fillSummarizedUserInfo: (updates, callback = (error, updates) ->) ->
-		users = {}
-		for update in updates
-			user_ids = update.meta.user_ids or []
-			for user_id in user_ids
-				if UpdatesManager._validUserId(user_id)
-					users[user_id] = true
-
-		UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
-			return callback(error) if error?
-			for update in updates
-				user_ids = update.meta.user_ids or []
-				update.meta.users = []
-				delete update.meta.user_ids
-				for user_id in user_ids
-					if UpdatesManager._validUserId(user_id)
-						update.meta.users.push fetchedUserInfo[user_id]
-					else
-						update.meta.users.push null
-			callback null, updates
-
-	_validUserId: (user_id) ->
-		if !user_id?
-			return false
-		else
-			return !!user_id.match(/^[a-f0-9]{24}$/)
-
-	TIME_BETWEEN_DISTINCT_UPDATES: fiveMinutes = 5 * 60 * 1000
-	SPLIT_ON_DELETE_SIZE: 16 # characters
-	_summarizeUpdates: (updates, existingSummarizedUpdates = []) ->
-		summarizedUpdates = existingSummarizedUpdates.slice()
-		previousUpdateWasBigDelete = false
-		for update in updates
-			earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
-			shouldConcat = false
-
-			# If a user inserts some text, then deletes a big chunk including that text,
-			# the update we show might concat the insert and delete, and there will be no sign
-			# of that insert having happened, or be able to restore to it (restoring after a big delete is common).
-			# So, we split the summary on 'big' deletes. However, we've stepping backwards in time with
-			# most recent changes considered first, so if this update is a big delete, we want to start
-			# a new summarized update next timge, hence we monitor the previous update.
-			if previousUpdateWasBigDelete
-				shouldConcat = false
-			else if earliestUpdate and earliestUpdate.meta.end_ts - update.meta.start_ts < @TIME_BETWEEN_DISTINCT_UPDATES
-				# We're going backwards in time through the updates, so only combine if this update starts less than 5 minutes before
-				# the end of current summarized block, so no block spans more than 5 minutes.
-				shouldConcat = true
-
-			isBigDelete = false
-			for op in update.op or []
-				if op.d? and op.d.length > @SPLIT_ON_DELETE_SIZE
-					isBigDelete = true
-
-			previousUpdateWasBigDelete = isBigDelete
-
-			if shouldConcat
-				# check if the user in this update is already present in the earliest update,
-				# if not, add them to the users list of the earliest update
-				earliestUpdate.meta.user_ids = _.union earliestUpdate.meta.user_ids, [update.meta.user_id]
-
-				doc_id = update.doc_id.toString()
-				doc = earliestUpdate.docs[doc_id]
-				if doc?
-					doc.fromV = Math.min(doc.fromV, update.v)
-					doc.toV = Math.max(doc.toV, update.v)
-				else
-					earliestUpdate.docs[doc_id] =
-						fromV: update.v
-						toV: update.v
-
-				earliestUpdate.meta.start_ts = Math.min(earliestUpdate.meta.start_ts, update.meta.start_ts)
-				earliestUpdate.meta.end_ts   = Math.max(earliestUpdate.meta.end_ts, update.meta.end_ts)
-			else
-				newUpdate =
-					meta:
-						user_ids: []
-						start_ts: update.meta.start_ts
-						end_ts: update.meta.end_ts
-					docs: {}
-
-				newUpdate.docs[update.doc_id.toString()] =
-					fromV: update.v
-					toV: update.v
-				newUpdate.meta.user_ids.push update.meta.user_id
-				summarizedUpdates.push newUpdate
-
-		return summarizedUpdates

services/track-changes/app/coffee/WebApiManager.coffee

@@ -1,69 +0,0 @@
-request = require "requestretry"  # allow retry on error https://github.com/FGRibreau/node-request-retry
-logger = require "logger-sharelatex"
-Settings = require "settings-sharelatex"
-
-# Don't let HTTP calls hang for a long time
-MAX_HTTP_REQUEST_LENGTH = 15000 # 15 seconds
-
-# DEPRECATED! This method of getting user details via track-changes is deprecated
-# in the way we lay out our services.
-# Instead, web should be responsible for collecting the raw data (user_ids) and
-# filling it out with calls to other services. All API calls should create a
-# tree-like structure as much as possible, with web as the root.
-module.exports = WebApiManager =
-	sendRequest: (url, callback = (error, body) ->) ->
-		request.get {
-			url: "#{Settings.apis.web.url}#{url}"
-			timeout: MAX_HTTP_REQUEST_LENGTH
-			maxAttempts: 2 # for node-request-retry
-			auth:
-				user: Settings.apis.web.user
-				pass: Settings.apis.web.pass
-				sendImmediately: true
-		}, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode == 404
-				logger.log url: url, "got 404 from web api"
-				return callback null, null
-			if res.statusCode >= 200 and res.statusCode < 300
-				return callback null, body
-			else
-				error = new Error("web returned a non-success status code: #{res.statusCode} (attempts: #{res.attempts})")
-				callback error
-
-	getUserInfo: (user_id, callback = (error, userInfo) ->) ->
-		url = "/user/#{user_id}/personal_info"
-		logger.log user_id: user_id, "getting user info from web"
-		WebApiManager.sendRequest url, (error, body) ->
-			if error?
-				logger.error err: error, user_id: user_id, url: url, "error accessing web"
-				return callback error
-
-			if body == null
-				logger.error user_id: user_id, url: url, "no user found"
-				return callback null, null
-			try
-				user = JSON.parse(body)
-			catch error
-				return callback(error)
-			callback null, {
-				id: user.id
-				email: user.email
-				first_name: user.first_name
-				last_name: user.last_name
-			}
-
-	getProjectDetails: (project_id, callback = (error, details) ->) ->
-		url = "/project/#{project_id}/details"
-		logger.log project_id: project_id, "getting project details from web"
-		WebApiManager.sendRequest url, (error, body) ->
-			if error?
-				logger.error err: error, project_id: project_id, url: url, "error accessing web"
-				return callback error
-
-			try
-				project = JSON.parse(body)
-			catch error
-				return callback(error)
-			callback null, project

services/track-changes/app/coffee/mongojs.coffee

@@ -1,9 +0,0 @@
-Settings = require "settings-sharelatex"
-mongojs = require "mongojs"
-bson = require "bson"
-db = mongojs(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryIndex"])
-module.exports =
-	db: db
-	ObjectId: mongojs.ObjectId
-	BSON: new bson.BSONPure()
-

services/track-changes/app/js/DiffGenerator.js

@@ -0,0 +1,345 @@
+/* eslint-disable
+    camelcase,
+    no-proto,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DiffGenerator
+var ConsistencyError = function(message) {
+  const error = new Error(message)
+  error.name = 'ConsistencyError'
+  error.__proto__ = ConsistencyError.prototype
+  return error
+}
+ConsistencyError.prototype.__proto__ = Error.prototype
+
+const logger = require('logger-sharelatex')
+
+module.exports = DiffGenerator = {
+  ConsistencyError,
+
+  rewindUpdate(content, update) {
+    for (let j = update.op.length - 1, i = j; j >= 0; j--, i = j) {
+      const op = update.op[i]
+      if (op.broken !== true) {
+        try {
+          content = DiffGenerator.rewindOp(content, op)
+        } catch (e) {
+          if (e instanceof ConsistencyError && (i = update.op.length - 1)) {
+            // catch known case where the last op in an array has been
+            // merged into a later op
+            logger.error(
+              { err: e, update, op: JSON.stringify(op) },
+              'marking op as broken'
+            )
+            op.broken = true
+          } else {
+            throw e // rethrow the execption
+          }
+        }
+      }
+    }
+    return content
+  },
+
+  rewindOp(content, op) {
+    let p
+    if (op.i != null) {
+      // ShareJS will accept an op where p > content.length when applied,
+      // and it applies as though p == content.length. However, the op is
+      // passed to us with the original p > content.length. Detect if that
+      // is the case with this op, and shift p back appropriately to match
+      // ShareJS if so.
+      ;({ p } = op)
+      const max_p = content.length - op.i.length
+      if (p > max_p) {
+        logger.warn({ max_p, p }, 'truncating position to content length')
+        p = max_p
+      }
+
+      const textToBeRemoved = content.slice(p, p + op.i.length)
+      if (op.i !== textToBeRemoved) {
+        throw new ConsistencyError(
+          `Inserted content, '${op.i}', does not match text to be removed, '${textToBeRemoved}'`
+        )
+      }
+
+      return content.slice(0, p) + content.slice(p + op.i.length)
+    } else if (op.d != null) {
+      return content.slice(0, op.p) + op.d + content.slice(op.p)
+    } else {
+      return content
+    }
+  },
+
+  rewindUpdates(content, updates) {
+    for (const update of Array.from(updates.reverse())) {
+      try {
+        content = DiffGenerator.rewindUpdate(content, update)
+      } catch (e) {
+        e.attempted_update = update // keep a record of the attempted update
+        throw e // rethrow the exception
+      }
+    }
+    return content
+  },
+
+  buildDiff(initialContent, updates) {
+    let diff = [{ u: initialContent }]
+    for (const update of Array.from(updates)) {
+      diff = DiffGenerator.applyUpdateToDiff(diff, update)
+    }
+    diff = DiffGenerator.compressDiff(diff)
+    return diff
+  },
+
+  compressDiff(diff) {
+    const newDiff = []
+    for (const part of Array.from(diff)) {
+      const lastPart = newDiff[newDiff.length - 1]
+      if (
+        lastPart != null &&
+        (lastPart.meta != null ? lastPart.meta.user : undefined) != null &&
+        (part.meta != null ? part.meta.user : undefined) != null
+      ) {
+        if (
+          lastPart.i != null &&
+          part.i != null &&
+          lastPart.meta.user.id === part.meta.user.id
+        ) {
+          lastPart.i += part.i
+          lastPart.meta.start_ts = Math.min(
+            lastPart.meta.start_ts,
+            part.meta.start_ts
+          )
+          lastPart.meta.end_ts = Math.max(
+            lastPart.meta.end_ts,
+            part.meta.end_ts
+          )
+        } else if (
+          lastPart.d != null &&
+          part.d != null &&
+          lastPart.meta.user.id === part.meta.user.id
+        ) {
+          lastPart.d += part.d
+          lastPart.meta.start_ts = Math.min(
+            lastPart.meta.start_ts,
+            part.meta.start_ts
+          )
+          lastPart.meta.end_ts = Math.max(
+            lastPart.meta.end_ts,
+            part.meta.end_ts
+          )
+        } else {
+          newDiff.push(part)
+        }
+      } else {
+        newDiff.push(part)
+      }
+    }
+    return newDiff
+  },
+
+  applyOpToDiff(diff, op, meta) {
+    let consumedDiff
+    const position = 0
+
+    let remainingDiff = diff.slice()
+    ;({ consumedDiff, remainingDiff } = DiffGenerator._consumeToOffset(
+      remainingDiff,
+      op.p
+    ))
+    const newDiff = consumedDiff
+
+    if (op.i != null) {
+      newDiff.push({
+        i: op.i,
+        meta
+      })
+    } else if (op.d != null) {
+      ;({
+        consumedDiff,
+        remainingDiff
+      } = DiffGenerator._consumeDiffAffectedByDeleteOp(remainingDiff, op, meta))
+      newDiff.push(...Array.from(consumedDiff || []))
+    }
+
+    newDiff.push(...Array.from(remainingDiff || []))
+
+    return newDiff
+  },
+
+  applyUpdateToDiff(diff, update) {
+    for (const op of Array.from(update.op)) {
+      if (op.broken !== true) {
+        diff = DiffGenerator.applyOpToDiff(diff, op, update.meta)
+      }
+    }
+    return diff
+  },
+
+  _consumeToOffset(remainingDiff, totalOffset) {
+    let part
+    const consumedDiff = []
+    let position = 0
+    while ((part = remainingDiff.shift())) {
+      const length = DiffGenerator._getLengthOfDiffPart(part)
+      if (part.d != null) {
+        consumedDiff.push(part)
+      } else if (position + length >= totalOffset) {
+        const partOffset = totalOffset - position
+        if (partOffset > 0) {
+          consumedDiff.push(DiffGenerator._slicePart(part, 0, partOffset))
+        }
+        if (partOffset < length) {
+          remainingDiff.unshift(DiffGenerator._slicePart(part, partOffset))
+        }
+        break
+      } else {
+        position += length
+        consumedDiff.push(part)
+      }
+    }
+
+    return {
+      consumedDiff,
+      remainingDiff
+    }
+  },
+
+  _consumeDiffAffectedByDeleteOp(remainingDiff, deleteOp, meta) {
+    const consumedDiff = []
+    let remainingOp = deleteOp
+    while (remainingOp && remainingDiff.length > 0) {
+      let newPart
+      ;({
+        newPart,
+        remainingDiff,
+        remainingOp
+      } = DiffGenerator._consumeDeletedPart(remainingDiff, remainingOp, meta))
+      if (newPart != null) {
+        consumedDiff.push(newPart)
+      }
+    }
+    return {
+      consumedDiff,
+      remainingDiff
+    }
+  },
+
+  _consumeDeletedPart(remainingDiff, op, meta) {
+    let deletedContent, newPart, remainingOp
+    const part = remainingDiff.shift()
+    const partLength = DiffGenerator._getLengthOfDiffPart(part)
+
+    if (part.d != null) {
+      // Skip existing deletes
+      remainingOp = op
+      newPart = part
+    } else if (partLength > op.d.length) {
+      // Only the first bit of the part has been deleted
+      const remainingPart = DiffGenerator._slicePart(part, op.d.length)
+      remainingDiff.unshift(remainingPart)
+
+      deletedContent = DiffGenerator._getContentOfPart(part).slice(
+        0,
+        op.d.length
+      )
+      if (deletedContent !== op.d) {
+        throw new ConsistencyError(
+          `deleted content, '${deletedContent}', does not match delete op, '${op.d}'`
+        )
+      }
+
+      if (part.u != null) {
+        newPart = {
+          d: op.d,
+          meta
+        }
+      } else if (part.i != null) {
+        newPart = null
+      }
+
+      remainingOp = null
+    } else if (partLength === op.d.length) {
+      // The entire part has been deleted, but it is the last part
+
+      deletedContent = DiffGenerator._getContentOfPart(part)
+      if (deletedContent !== op.d) {
+        throw new ConsistencyError(
+          `deleted content, '${deletedContent}', does not match delete op, '${op.d}'`
+        )
+      }
+
+      if (part.u != null) {
+        newPart = {
+          d: op.d,
+          meta
+        }
+      } else if (part.i != null) {
+        newPart = null
+      }
+
+      remainingOp = null
+    } else if (partLength < op.d.length) {
+      // The entire part has been deleted and there is more
+
+      deletedContent = DiffGenerator._getContentOfPart(part)
+      const opContent = op.d.slice(0, deletedContent.length)
+      if (deletedContent !== opContent) {
+        throw new ConsistencyError(
+          `deleted content, '${deletedContent}', does not match delete op, '${opContent}'`
+        )
+      }
+
+      if (part.u) {
+        newPart = {
+          d: part.u,
+          meta
+        }
+      } else if (part.i != null) {
+        newPart = null
+      }
+
+      remainingOp = {
+        p: op.p,
+        d: op.d.slice(DiffGenerator._getLengthOfDiffPart(part))
+      }
+    }
+
+    return {
+      newPart,
+      remainingDiff,
+      remainingOp
+    }
+  },
+
+  _slicePart(basePart, from, to) {
+    let part
+    if (basePart.u != null) {
+      part = { u: basePart.u.slice(from, to) }
+    } else if (basePart.i != null) {
+      part = { i: basePart.i.slice(from, to) }
+    }
+    if (basePart.meta != null) {
+      part.meta = basePart.meta
+    }
+    return part
+  },
+
+  _getLengthOfDiffPart(part) {
+    return (part.u || part.d || part.i || '').length
+  },
+
+  _getContentOfPart(part) {
+    return part.u || part.d || part.i || ''
+  }
+}

services/track-changes/app/js/DiffManager.js

@@ -0,0 +1,188 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DiffManager
+const UpdatesManager = require('./UpdatesManager')
+const DocumentUpdaterManager = require('./DocumentUpdaterManager')
+const DiffGenerator = require('./DiffGenerator')
+const logger = require('logger-sharelatex')
+
+module.exports = DiffManager = {
+  getLatestDocAndUpdates(project_id, doc_id, fromVersion, callback) {
+    // Get updates last, since then they must be ahead and it
+    // might be possible to rewind to the same version as the doc.
+    if (callback == null) {
+      callback = function(error, content, version, updates) {}
+    }
+    return DocumentUpdaterManager.getDocument(project_id, doc_id, function(
+      error,
+      content,
+      version
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (fromVersion == null) {
+        // If we haven't been given a version, just return lastest doc and no updates
+        return callback(null, content, version, [])
+      }
+      return UpdatesManager.getDocUpdatesWithUserInfo(
+        project_id,
+        doc_id,
+        { from: fromVersion },
+        function(error, updates) {
+          if (error != null) {
+            return callback(error)
+          }
+          return callback(null, content, version, updates)
+        }
+      )
+    })
+  },
+
+  getDiff(project_id, doc_id, fromVersion, toVersion, callback) {
+    if (callback == null) {
+      callback = function(error, diff) {}
+    }
+    return DiffManager.getDocumentBeforeVersion(
+      project_id,
+      doc_id,
+      fromVersion,
+      function(error, startingContent, updates) {
+        let diff
+        if (error != null) {
+          if (error.message === 'broken-history') {
+            return callback(null, 'history unavailable')
+          } else {
+            return callback(error)
+          }
+        }
+
+        const updatesToApply = []
+        for (const update of Array.from(updates.slice().reverse())) {
+          if (update.v <= toVersion) {
+            updatesToApply.push(update)
+          }
+        }
+
+        try {
+          diff = DiffGenerator.buildDiff(startingContent, updatesToApply)
+        } catch (e) {
+          return callback(e)
+        }
+
+        return callback(null, diff)
+      }
+    )
+  },
+
+  getDocumentBeforeVersion(project_id, doc_id, version, _callback) {
+    // Whichever order we get the latest document and the latest updates,
+    // there is potential for updates to be applied between them so that
+    // they do not return the same 'latest' versions.
+    // If this happens, we just retry and hopefully get them at the compatible
+    // versions.
+    let retry
+    if (_callback == null) {
+      _callback = function(error, document, rewoundUpdates) {}
+    }
+    let retries = 3
+    const callback = function(error, ...args) {
+      if (error != null) {
+        if (error.retry && retries > 0) {
+          logger.warn(
+            { error, project_id, doc_id, version, retries },
+            'retrying getDocumentBeforeVersion'
+          )
+          return retry()
+        } else {
+          return _callback(error)
+        }
+      } else {
+        return _callback(null, ...Array.from(args))
+      }
+    }
+
+    return (retry = function() {
+      retries--
+      return DiffManager._tryGetDocumentBeforeVersion(
+        project_id,
+        doc_id,
+        version,
+        callback
+      )
+    })()
+  },
+
+  _tryGetDocumentBeforeVersion(project_id, doc_id, version, callback) {
+    if (callback == null) {
+      callback = function(error, document, rewoundUpdates) {}
+    }
+    logger.log(
+      { project_id, doc_id, version },
+      'getting document before version'
+    )
+    return DiffManager.getLatestDocAndUpdates(
+      project_id,
+      doc_id,
+      version,
+      function(error, content, version, updates) {
+        let startingContent
+        if (error != null) {
+          return callback(error)
+        }
+
+        // bail out if we hit a broken update
+        for (const u of Array.from(updates)) {
+          if (u.broken) {
+            return callback(new Error('broken-history'))
+          }
+        }
+
+        // discard any updates which are ahead of this document version
+        while ((updates[0] != null ? updates[0].v : undefined) >= version) {
+          updates.shift()
+        }
+
+        const lastUpdate = updates[0]
+        if (lastUpdate != null && lastUpdate.v !== version - 1) {
+          error = new Error(
+            `latest update version, ${lastUpdate.v}, does not match doc version, ${version}`
+          )
+          error.retry = true
+          return callback(error)
+        }
+
+        logger.log(
+          {
+            docVersion: version,
+            lastUpdateVersion: lastUpdate != null ? lastUpdate.v : undefined,
+            updateCount: updates.length
+          },
+          'rewinding updates'
+        )
+
+        const tryUpdates = updates.slice().reverse()
+
+        try {
+          startingContent = DiffGenerator.rewindUpdates(content, tryUpdates)
+          // tryUpdates is reversed, and any unapplied ops are marked as broken
+        } catch (e) {
+          return callback(e)
+        }
+
+        return callback(null, startingContent, tryUpdates)
+      }
+    )
+  }
+}

services/track-changes/app/js/DocumentUpdaterManager.js

@@ -0,0 +1,90 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocumentUpdaterManager
+const request = require('request')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+
+module.exports = DocumentUpdaterManager = {
+  getDocument(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, content, version) {}
+    }
+    const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`
+    logger.log({ project_id, doc_id }, 'getting doc from document updater')
+    return request.get(url, function(error, res, body) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (res.statusCode >= 200 && res.statusCode < 300) {
+        try {
+          body = JSON.parse(body)
+        } catch (error1) {
+          error = error1
+          return callback(error)
+        }
+        logger.log(
+          { project_id, doc_id, version: body.version },
+          'got doc from document updater'
+        )
+        return callback(null, body.lines.join('\n'), body.version)
+      } else {
+        error = new Error(
+          `doc updater returned a non-success status code: ${res.statusCode}`
+        )
+        logger.error(
+          { err: error, project_id, doc_id, url },
+          'error accessing doc updater'
+        )
+        return callback(error)
+      }
+    })
+  },
+
+  setDocument(project_id, doc_id, content, user_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`
+    logger.log({ project_id, doc_id }, 'setting doc in document updater')
+    return request.post(
+      {
+        url,
+        json: {
+          lines: content.split('\n'),
+          source: 'restore',
+          user_id,
+          undoing: true
+        }
+      },
+      function(error, res, body) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          return callback(null)
+        } else {
+          error = new Error(
+            `doc updater returned a non-success status code: ${res.statusCode}`
+          )
+          logger.error(
+            { err: error, project_id, doc_id, url },
+            'error accessing doc updater'
+          )
+          return callback(error)
+        }
+      }
+    )
+  }
+}

services/track-changes/app/js/HealthChecker.js

@@ -0,0 +1,84 @@
+/* eslint-disable
+    camelcase,
+    standard/no-callback-literal,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { ObjectId } = require('mongojs')
+const request = require('request')
+const async = require('async')
+const settings = require('settings-sharelatex')
+const { port } = settings.internal.trackchanges
+const logger = require('logger-sharelatex')
+const LockManager = require('./LockManager')
+
+module.exports = {
+  check(callback) {
+    const project_id = ObjectId(settings.trackchanges.healthCheck.project_id)
+    const url = `http://localhost:${port}/project/${project_id}`
+    logger.log({ project_id }, 'running health check')
+    const jobs = [
+      cb =>
+        request.get(
+          { url: `http://localhost:${port}/check_lock`, timeout: 3000 },
+          function(err, res, body) {
+            if (err != null) {
+              logger.err(
+                { err, project_id },
+                'error checking lock for health check'
+              )
+              return cb(err)
+            } else if ((res != null ? res.statusCode : undefined) !== 200) {
+              return cb(`status code not 200, it's ${res.statusCode}`)
+            } else {
+              return cb()
+            }
+          }
+        ),
+      cb =>
+        request.post({ url: `${url}/flush`, timeout: 10000 }, function(
+          err,
+          res,
+          body
+        ) {
+          if (err != null) {
+            logger.err({ err, project_id }, 'error flushing for health check')
+            return cb(err)
+          } else if ((res != null ? res.statusCode : undefined) !== 204) {
+            return cb(`status code not 204, it's ${res.statusCode}`)
+          } else {
+            return cb()
+          }
+        }),
+      cb =>
+        request.get({ url: `${url}/updates`, timeout: 10000 }, function(
+          err,
+          res,
+          body
+        ) {
+          if (err != null) {
+            logger.err(
+              { err, project_id },
+              'error getting updates for health check'
+            )
+            return cb(err)
+          } else if ((res != null ? res.statusCode : undefined) !== 200) {
+            return cb(`status code not 200, it's ${res.statusCode}`)
+          } else {
+            return cb()
+          }
+        })
+    ]
+    return async.series(jobs, callback)
+  },
+
+  checkLock(callback) {
+    return LockManager.healthCheck(callback)
+  }
+}

services/track-changes/app/js/HttpController.js

@@ -0,0 +1,274 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HttpController
+const UpdatesManager = require('./UpdatesManager')
+const DiffManager = require('./DiffManager')
+const PackManager = require('./PackManager')
+const RestoreManager = require('./RestoreManager')
+const logger = require('logger-sharelatex')
+const HealthChecker = require('./HealthChecker')
+const _ = require('underscore')
+
+module.exports = HttpController = {
+  flushDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { doc_id } = req.params
+    const { project_id } = req.params
+    logger.log({ project_id, doc_id }, 'compressing doc history')
+    return UpdatesManager.processUncompressedUpdatesWithLock(
+      project_id,
+      doc_id,
+      function(error) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.send(204)
+      }
+    )
+  },
+
+  flushProject(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    logger.log({ project_id }, 'compressing project history')
+    return UpdatesManager.processUncompressedUpdatesForProject(
+      project_id,
+      function(error) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.send(204)
+      }
+    )
+  },
+
+  flushAll(req, res, next) {
+    // limit on projects to flush or -1 for all (default)
+    if (next == null) {
+      next = function(error) {}
+    }
+    const limit = req.query.limit != null ? parseInt(req.query.limit, 10) : -1
+    logger.log({ limit }, 'flushing all projects')
+    return UpdatesManager.flushAll(limit, function(error, result) {
+      if (error != null) {
+        return next(error)
+      }
+      const { failed, succeeded, all } = result
+      const status = `${succeeded.length} succeeded, ${failed.length} failed`
+      if (limit === 0) {
+        return res
+          .status(200)
+          .send(`${status}\nwould flush:\n${all.join('\n')}\n`)
+      } else if (failed.length > 0) {
+        logger.log({ failed, succeeded }, 'error flushing projects')
+        return res
+          .status(500)
+          .send(`${status}\nfailed to flush:\n${failed.join('\n')}\n`)
+      } else {
+        return res
+          .status(200)
+          .send(
+            `${status}\nflushed ${succeeded.length} projects of ${all.length}\n`
+          )
+      }
+    })
+  },
+
+  checkDanglingUpdates(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    logger.log('checking dangling updates')
+    return UpdatesManager.getDanglingUpdates(function(error, result) {
+      if (error != null) {
+        return next(error)
+      }
+      if (result.length > 0) {
+        logger.log({ dangling: result }, 'found dangling updates')
+        return res.status(500).send(`dangling updates:\n${result.join('\n')}\n`)
+      } else {
+        return res.status(200).send('no dangling updates found\n')
+      }
+    })
+  },
+
+  checkDoc(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { doc_id } = req.params
+    const { project_id } = req.params
+    logger.log({ project_id, doc_id }, 'checking doc history')
+    return DiffManager.getDocumentBeforeVersion(project_id, doc_id, 1, function(
+      error,
+      document,
+      rewoundUpdates
+    ) {
+      if (error != null) {
+        return next(error)
+      }
+      const broken = []
+      for (const update of Array.from(rewoundUpdates)) {
+        for (const op of Array.from(update.op)) {
+          if (op.broken === true) {
+            broken.push(op)
+          }
+        }
+      }
+      if (broken.length > 0) {
+        return res.send(broken)
+      } else {
+        return res.send(204)
+      }
+    })
+  },
+
+  getDiff(req, res, next) {
+    let from, to
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { doc_id } = req.params
+    const { project_id } = req.params
+
+    if (req.query.from != null) {
+      from = parseInt(req.query.from, 10)
+    } else {
+      from = null
+    }
+    if (req.query.to != null) {
+      to = parseInt(req.query.to, 10)
+    } else {
+      to = null
+    }
+
+    logger.log({ project_id, doc_id, from, to }, 'getting diff')
+    return DiffManager.getDiff(project_id, doc_id, from, to, function(
+      error,
+      diff
+    ) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.json({ diff })
+    })
+  },
+
+  getUpdates(req, res, next) {
+    let before, min_count
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+
+    if (req.query.before != null) {
+      before = parseInt(req.query.before, 10)
+    }
+    if (req.query.min_count != null) {
+      min_count = parseInt(req.query.min_count, 10)
+    }
+
+    return UpdatesManager.getSummarizedProjectUpdates(
+      project_id,
+      { before, min_count },
+      function(error, updates, nextBeforeTimestamp) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.json({
+          updates,
+          nextBeforeTimestamp
+        })
+      }
+    )
+  },
+
+  restore(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    let { doc_id, project_id, version } = req.params
+    const user_id = req.headers['x-user-id']
+    version = parseInt(version, 10)
+    return RestoreManager.restoreToBeforeVersion(
+      project_id,
+      doc_id,
+      version,
+      user_id,
+      function(error) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.send(204)
+      }
+    )
+  },
+
+  pushDocHistory(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    logger.log({ project_id, doc_id }, 'pushing all finalised changes to s3')
+    return PackManager.pushOldPacks(project_id, doc_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(204)
+    })
+  },
+
+  pullDocHistory(req, res, next) {
+    if (next == null) {
+      next = function(error) {}
+    }
+    const { project_id } = req.params
+    const { doc_id } = req.params
+    logger.log({ project_id, doc_id }, 'pulling all packs from s3')
+    return PackManager.pullOldPacks(project_id, doc_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(204)
+    })
+  },
+
+  healthCheck(req, res) {
+    return HealthChecker.check(function(err) {
+      if (err != null) {
+        logger.err({ err }, 'error performing health check')
+        return res.send(500)
+      } else {
+        return res.send(200)
+      }
+    })
+  },
+
+  checkLock(req, res) {
+    return HealthChecker.checkLock(function(err) {
+      if (err != null) {
+        logger.err({ err }, 'error performing lock check')
+        return res.send(500)
+      } else {
+        return res.send(200)
+      }
+    })
+  }
+}

services/track-changes/app/js/LockManager.js

@@ -0,0 +1,157 @@
+/* eslint-disable
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let LockManager
+const Settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(Settings.redis.lock)
+const os = require('os')
+const crypto = require('crypto')
+const logger = require('logger-sharelatex')
+
+const HOST = os.hostname()
+const PID = process.pid
+const RND = crypto.randomBytes(4).toString('hex')
+let COUNT = 0
+
+module.exports = LockManager = {
+  LOCK_TEST_INTERVAL: 50, // 50ms between each test of the lock
+  MAX_LOCK_WAIT_TIME: 10000, // 10s maximum time to spend trying to get the lock
+  LOCK_TTL: 300, // seconds (allow 5 minutes for any operation to complete)
+
+  // Use a signed lock value as described in
+  // http://redis.io/topics/distlock#correct-implementation-with-a-single-instance
+  // to prevent accidental unlocking by multiple processes
+  randomLock() {
+    const time = Date.now()
+    return `locked:host=${HOST}:pid=${PID}:random=${RND}:time=${time}:count=${COUNT++}`
+  },
+
+  unlockScript:
+    'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end',
+
+  tryLock(key, callback) {
+    if (callback == null) {
+      callback = function(err, gotLock) {}
+    }
+    const lockValue = LockManager.randomLock()
+    return rclient.set(key, lockValue, 'EX', this.LOCK_TTL, 'NX', function(
+      err,
+      gotLock
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (gotLock === 'OK') {
+        return callback(err, true, lockValue)
+      } else {
+        return callback(err, false)
+      }
+    })
+  },
+
+  getLock(key, callback) {
+    let attempt
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const startTime = Date.now()
+    return (attempt = function() {
+      if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) {
+        const e = new Error('Timeout')
+        e.key = key
+        return callback(e)
+      }
+
+      return LockManager.tryLock(key, function(error, gotLock, lockValue) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (gotLock) {
+          return callback(null, lockValue)
+        } else {
+          return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL)
+        }
+      })
+    })()
+  },
+
+  checkLock(key, callback) {
+    if (callback == null) {
+      callback = function(err, isFree) {}
+    }
+    return rclient.exists(key, function(err, exists) {
+      if (err != null) {
+        return callback(err)
+      }
+      exists = parseInt(exists)
+      if (exists === 1) {
+        return callback(err, false)
+      } else {
+        return callback(err, true)
+      }
+    })
+  },
+
+  releaseLock(key, lockValue, callback) {
+    return rclient.eval(LockManager.unlockScript, 1, key, lockValue, function(
+      err,
+      result
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (result != null && result !== 1) {
+        // successful unlock should release exactly one key
+        logger.error(
+          { key, lockValue, redis_err: err, redis_result: result },
+          'unlocking error'
+        )
+        return callback(new Error('tried to release timed out lock'))
+      }
+      return callback(err, result)
+    })
+  },
+
+  runWithLock(key, runner, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return LockManager.getLock(key, function(error, lockValue) {
+      if (error != null) {
+        return callback(error)
+      }
+      return runner(error1 =>
+        LockManager.releaseLock(key, lockValue, function(error2) {
+          error = error1 || error2
+          if (error != null) {
+            return callback(error)
+          }
+          return callback()
+        })
+      )
+    })
+  },
+
+  healthCheck(callback) {
+    const action = releaseLock => releaseLock()
+    return LockManager.runWithLock(
+      `HistoryLock:HealthCheck:host=${HOST}:pid=${PID}:random=${RND}`,
+      action,
+      callback
+    )
+  },
+
+  close(callback) {
+    rclient.quit()
+    return rclient.once('end', callback)
+  }
+}

services/track-changes/app/js/MongoAWS.js

@@ -0,0 +1,193 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MongoAWS
+const settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const AWS = require('aws-sdk')
+const S3S = require('s3-streams')
+const { db, ObjectId } = require('./mongojs')
+const JSONStream = require('JSONStream')
+const ReadlineStream = require('byline')
+const zlib = require('zlib')
+const Metrics = require('metrics-sharelatex')
+
+const DAYS = 24 * 3600 * 1000 // one day in milliseconds
+
+const createStream = function(streamConstructor, project_id, doc_id, pack_id) {
+  const AWS_CONFIG = {
+    accessKeyId: settings.trackchanges.s3.key,
+    secretAccessKey: settings.trackchanges.s3.secret,
+    endpoint: settings.trackchanges.s3.endpoint,
+    s3ForcePathStyle: settings.trackchanges.s3.pathStyle
+  }
+
+  return streamConstructor(new AWS.S3(AWS_CONFIG), {
+    Bucket: settings.trackchanges.stores.doc_history,
+    Key: project_id + '/changes-' + doc_id + '/pack-' + pack_id
+  })
+}
+
+module.exports = MongoAWS = {
+  archivePack(project_id, doc_id, pack_id, _callback) {
+    if (_callback == null) {
+      _callback = function(error) {}
+    }
+    const callback = function(...args) {
+      _callback(...Array.from(args || []))
+      return (_callback = function() {})
+    }
+
+    const query = {
+      _id: ObjectId(pack_id),
+      doc_id: ObjectId(doc_id)
+    }
+
+    if (project_id == null) {
+      return callback(new Error('invalid project id'))
+    }
+    if (doc_id == null) {
+      return callback(new Error('invalid doc id'))
+    }
+    if (pack_id == null) {
+      return callback(new Error('invalid pack id'))
+    }
+
+    logger.log({ project_id, doc_id, pack_id }, 'uploading data to s3')
+
+    const upload = createStream(S3S.WriteStream, project_id, doc_id, pack_id)
+
+    return db.docHistory.findOne(query, function(err, result) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (result == null) {
+        return callback(new Error('cannot find pack to send to s3'))
+      }
+      if (result.expiresAt != null) {
+        return callback(new Error('refusing to send pack with TTL to s3'))
+      }
+      const uncompressedData = JSON.stringify(result)
+      if (uncompressedData.indexOf('\u0000') !== -1) {
+        const error = new Error('null bytes found in upload')
+        logger.error({ err: error, project_id, doc_id, pack_id }, error.message)
+        return callback(error)
+      }
+      return zlib.gzip(uncompressedData, function(err, buf) {
+        logger.log(
+          {
+            project_id,
+            doc_id,
+            pack_id,
+            origSize: uncompressedData.length,
+            newSize: buf.length
+          },
+          'compressed pack'
+        )
+        if (err != null) {
+          return callback(err)
+        }
+        upload.on('error', err => callback(err))
+        upload.on('finish', function() {
+          Metrics.inc('archive-pack')
+          logger.log({ project_id, doc_id, pack_id }, 'upload to s3 completed')
+          return callback(null)
+        })
+        upload.write(buf)
+        return upload.end()
+      })
+    })
+  },
+
+  readArchivedPack(project_id, doc_id, pack_id, _callback) {
+    if (_callback == null) {
+      _callback = function(error, result) {}
+    }
+    const callback = function(...args) {
+      _callback(...Array.from(args || []))
+      return (_callback = function() {})
+    }
+
+    if (project_id == null) {
+      return callback(new Error('invalid project id'))
+    }
+    if (doc_id == null) {
+      return callback(new Error('invalid doc id'))
+    }
+    if (pack_id == null) {
+      return callback(new Error('invalid pack id'))
+    }
+
+    logger.log({ project_id, doc_id, pack_id }, 'downloading data from s3')
+
+    const download = createStream(S3S.ReadStream, project_id, doc_id, pack_id)
+
+    const inputStream = download
+      .on('open', obj => 1)
+      .on('error', err => callback(err))
+
+    const gunzip = zlib.createGunzip()
+    gunzip.setEncoding('utf8')
+    gunzip.on('error', function(err) {
+      logger.log(
+        { project_id, doc_id, pack_id, err },
+        'error uncompressing gzip stream'
+      )
+      return callback(err)
+    })
+
+    const outputStream = inputStream.pipe(gunzip)
+    const parts = []
+    outputStream.on('error', err => callback(err))
+    outputStream.on('end', function() {
+      let object
+      logger.log({ project_id, doc_id, pack_id }, 'download from s3 completed')
+      try {
+        object = JSON.parse(parts.join(''))
+      } catch (e) {
+        return callback(e)
+      }
+      object._id = ObjectId(object._id)
+      object.doc_id = ObjectId(object.doc_id)
+      object.project_id = ObjectId(object.project_id)
+      for (const op of Array.from(object.pack)) {
+        if (op._id != null) {
+          op._id = ObjectId(op._id)
+        }
+      }
+      return callback(null, object)
+    })
+    return outputStream.on('data', data => parts.push(data))
+  },
+
+  unArchivePack(project_id, doc_id, pack_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return MongoAWS.readArchivedPack(project_id, doc_id, pack_id, function(
+      err,
+      object
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      Metrics.inc('unarchive-pack')
+      // allow the object to expire, we can always retrieve it again
+      object.expiresAt = new Date(Date.now() + 7 * DAYS)
+      logger.log({ project_id, doc_id, pack_id }, 'inserting object from s3')
+      return db.docHistory.insert(object, callback)
+    })
+  }
+}

services/track-changes/app/js/MongoManager.js

@@ -0,0 +1,210 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MongoManager
+const { db, ObjectId } = require('./mongojs')
+const PackManager = require('./PackManager')
+const async = require('async')
+const _ = require('underscore')
+const metrics = require('metrics-sharelatex')
+const logger = require('logger-sharelatex')
+
+module.exports = MongoManager = {
+  getLastCompressedUpdate(doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, update) {}
+    }
+    return db.docHistory
+      .find({ doc_id: ObjectId(doc_id.toString()) }, { pack: { $slice: -1 } }) // only return the last entry in a pack
+      .sort({ v: -1 })
+      .limit(1)
+      .toArray(function(error, compressedUpdates) {
+        if (error != null) {
+          return callback(error)
+        }
+        return callback(null, compressedUpdates[0] || null)
+      })
+  },
+
+  peekLastCompressedUpdate(doc_id, callback) {
+    // under normal use we pass back the last update as
+    // callback(null,update,version).
+    //
+    // when we have an existing last update but want to force a new one
+    // to start, we pass it back as callback(null,null,version), just
+    // giving the version so we can check consistency.
+    if (callback == null) {
+      callback = function(error, update, version) {}
+    }
+    return MongoManager.getLastCompressedUpdate(doc_id, function(
+      error,
+      update
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (update != null) {
+        if (update.broken) {
+          // marked as broken so we will force a new op
+          return callback(null, null)
+        } else if (update.pack != null) {
+          if (update.finalised) {
+            // no more ops can be appended
+            return callback(
+              null,
+              null,
+              update.pack[0] != null ? update.pack[0].v : undefined
+            )
+          } else {
+            return callback(
+              null,
+              update,
+              update.pack[0] != null ? update.pack[0].v : undefined
+            )
+          }
+        } else {
+          return callback(null, update, update.v)
+        }
+      } else {
+        return PackManager.getLastPackFromIndex(doc_id, function(error, pack) {
+          if (error != null) {
+            return callback(error)
+          }
+          if (
+            (pack != null ? pack.inS3 : undefined) != null &&
+            (pack != null ? pack.v_end : undefined) != null
+          ) {
+            return callback(null, null, pack.v_end)
+          }
+          return callback(null, null)
+        })
+      }
+    })
+  },
+
+  backportProjectId(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return db.docHistory.update(
+      {
+        doc_id: ObjectId(doc_id.toString()),
+        project_id: { $exists: false }
+      },
+      {
+        $set: { project_id: ObjectId(project_id.toString()) }
+      },
+      {
+        multi: true
+      },
+      callback
+    )
+  },
+
+  getProjectMetaData(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, metadata) {}
+    }
+    return db.projectHistoryMetaData.find(
+      {
+        project_id: ObjectId(project_id.toString())
+      },
+      function(error, results) {
+        if (error != null) {
+          return callback(error)
+        }
+        return callback(null, results[0])
+      }
+    )
+  },
+
+  setProjectMetaData(project_id, metadata, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return db.projectHistoryMetaData.update(
+      {
+        project_id: ObjectId(project_id)
+      },
+      {
+        $set: metadata
+      },
+      {
+        upsert: true
+      },
+      callback
+    )
+  },
+
+  upgradeHistory(project_id, callback) {
+    // preserve the project's existing history
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return db.docHistory.update(
+      {
+        project_id: ObjectId(project_id),
+        temporary: true,
+        expiresAt: { $exists: true }
+      },
+      {
+        $set: { temporary: false },
+        $unset: { expiresAt: '' }
+      },
+      {
+        multi: true
+      },
+      callback
+    )
+  },
+
+  ensureIndices() {
+    // For finding all updates that go into a diff for a doc
+    db.docHistory.ensureIndex({ doc_id: 1, v: 1 }, { background: true })
+    // For finding all updates that affect a project
+    db.docHistory.ensureIndex(
+      { project_id: 1, 'meta.end_ts': 1 },
+      { background: true }
+    )
+    // For finding updates that don't yet have a project_id and need it inserting
+    db.docHistory.ensureIndex(
+      { doc_id: 1, project_id: 1 },
+      { background: true }
+    )
+    // For finding project meta-data
+    db.projectHistoryMetaData.ensureIndex(
+      { project_id: 1 },
+      { background: true }
+    )
+    // TTL index for auto deleting week old temporary ops
+    db.docHistory.ensureIndex(
+      { expiresAt: 1 },
+      { expireAfterSeconds: 0, background: true }
+    )
+    // For finding packs to be checked for archiving
+    db.docHistory.ensureIndex({ last_checked: 1 }, { background: true })
+    // For finding archived packs
+    return db.docHistoryIndex.ensureIndex(
+      { project_id: 1 },
+      { background: true }
+    )
+  }
+}
+
+;[
+  'getLastCompressedUpdate',
+  'getProjectMetaData',
+  'setProjectMetaData'
+].map(method =>
+  metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger)
+)

services/track-changes/app/js/PackManager.js

@@ -0,0 +1,1164 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let PackManager
+const async = require('async')
+const _ = require('underscore')
+const { db, ObjectId, BSON } = require('./mongojs')
+const logger = require('logger-sharelatex')
+const LockManager = require('./LockManager')
+const MongoAWS = require('./MongoAWS')
+const Metrics = require('metrics-sharelatex')
+const ProjectIterator = require('./ProjectIterator')
+const Settings = require('settings-sharelatex')
+const keys = Settings.redis.lock.key_schema
+
+// Sharejs operations are stored in a 'pack' object
+//
+//  e.g.  a single sharejs update looks like
+//
+//   {
+//     "doc_id" : 549dae9e0a2a615c0c7f0c98,
+//     "project_id" : 549dae9c0a2a615c0c7f0c8c,
+//     "op" : [ {"p" : 6981,	"d" : "?"	} ],
+//     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
+//     "v" : 17082
+//   }
+//
+//  and a pack looks like this
+//
+//   {
+//     "doc_id" : 549dae9e0a2a615c0c7f0c98,
+//     "project_id" : 549dae9c0a2a615c0c7f0c8c,
+//     "pack" : [ U1, U2, U3, ...., UN],
+//     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
+//     "v" : 17082
+//     "v_end" : ...
+//   }
+//
+//  where U1, U2, U3, .... are single updates stripped of their
+//  doc_id and project_id fields (which are the same for all the
+//  updates in the pack).
+//
+//  The pack itself has v and meta fields, this makes it possible to
+//  treat packs and single updates in a similar way.
+//
+//  The v field of the pack itself is from the first entry U1, the
+//  v_end field from UN.  The meta.end_ts field of the pack itself is
+//  from the last entry UN, the meta.start_ts field from U1.
+
+const DAYS = 24 * 3600 * 1000 // one day in milliseconds
+
+module.exports = PackManager = {
+  MAX_SIZE: 1024 * 1024, // make these configurable parameters
+  MAX_COUNT: 1024,
+
+  insertCompressedUpdates(
+    project_id,
+    doc_id,
+    lastUpdate,
+    newUpdates,
+    temporary,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (newUpdates.length === 0) {
+      return callback()
+    }
+
+    // never append permanent ops to a pack that will expire
+    if (
+      (lastUpdate != null ? lastUpdate.expiresAt : undefined) != null &&
+      !temporary
+    ) {
+      lastUpdate = null
+    }
+
+    const updatesToFlush = []
+    const updatesRemaining = newUpdates.slice()
+
+    let n = (lastUpdate != null ? lastUpdate.n : undefined) || 0
+    let sz = (lastUpdate != null ? lastUpdate.sz : undefined) || 0
+
+    while (
+      updatesRemaining.length &&
+      n < PackManager.MAX_COUNT &&
+      sz < PackManager.MAX_SIZE
+    ) {
+      const nextUpdate = updatesRemaining[0]
+      const nextUpdateSize = BSON.calculateObjectSize(nextUpdate)
+      if (nextUpdateSize + sz > PackManager.MAX_SIZE && n > 0) {
+        break
+      }
+      n++
+      sz += nextUpdateSize
+      updatesToFlush.push(updatesRemaining.shift())
+    }
+
+    return PackManager.flushCompressedUpdates(
+      project_id,
+      doc_id,
+      lastUpdate,
+      updatesToFlush,
+      temporary,
+      function(error) {
+        if (error != null) {
+          return callback(error)
+        }
+        return PackManager.insertCompressedUpdates(
+          project_id,
+          doc_id,
+          null,
+          updatesRemaining,
+          temporary,
+          callback
+        )
+      }
+    )
+  },
+
+  flushCompressedUpdates(
+    project_id,
+    doc_id,
+    lastUpdate,
+    newUpdates,
+    temporary,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (newUpdates.length === 0) {
+      return callback()
+    }
+
+    let canAppend = false
+    // check if it is safe to append to an existing pack
+    if (lastUpdate != null) {
+      if (!temporary && lastUpdate.expiresAt == null) {
+        // permanent pack appends to permanent pack
+        canAppend = true
+      }
+      const age =
+        Date.now() -
+        (lastUpdate.meta != null ? lastUpdate.meta.start_ts : undefined)
+      if (temporary && lastUpdate.expiresAt != null && age < 1 * DAYS) {
+        // temporary pack appends to temporary pack if same day
+        canAppend = true
+      }
+    }
+
+    if (canAppend) {
+      return PackManager.appendUpdatesToExistingPack(
+        project_id,
+        doc_id,
+        lastUpdate,
+        newUpdates,
+        temporary,
+        callback
+      )
+    } else {
+      return PackManager.insertUpdatesIntoNewPack(
+        project_id,
+        doc_id,
+        newUpdates,
+        temporary,
+        callback
+      )
+    }
+  },
+
+  insertUpdatesIntoNewPack(
+    project_id,
+    doc_id,
+    newUpdates,
+    temporary,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const first = newUpdates[0]
+    const last = newUpdates[newUpdates.length - 1]
+    const n = newUpdates.length
+    const sz = BSON.calculateObjectSize(newUpdates)
+    const newPack = {
+      project_id: ObjectId(project_id.toString()),
+      doc_id: ObjectId(doc_id.toString()),
+      pack: newUpdates,
+      n,
+      sz,
+      meta: {
+        start_ts: first.meta.start_ts,
+        end_ts: last.meta.end_ts
+      },
+      v: first.v,
+      v_end: last.v,
+      temporary
+    }
+    if (temporary) {
+      newPack.expiresAt = new Date(Date.now() + 7 * DAYS)
+      newPack.last_checked = new Date(Date.now() + 30 * DAYS) // never check temporary packs
+    }
+    logger.log(
+      { project_id, doc_id, newUpdates },
+      'inserting updates into new pack'
+    )
+    return db.docHistory.save(newPack, function(err, result) {
+      if (err != null) {
+        return callback(err)
+      }
+      Metrics.inc(`insert-pack-${temporary ? 'temporary' : 'permanent'}`)
+      if (temporary) {
+        return callback()
+      } else {
+        return PackManager.updateIndex(project_id, doc_id, callback)
+      }
+    })
+  },
+
+  appendUpdatesToExistingPack(
+    project_id,
+    doc_id,
+    lastUpdate,
+    newUpdates,
+    temporary,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const first = newUpdates[0]
+    const last = newUpdates[newUpdates.length - 1]
+    const n = newUpdates.length
+    const sz = BSON.calculateObjectSize(newUpdates)
+    const query = {
+      _id: lastUpdate._id,
+      project_id: ObjectId(project_id.toString()),
+      doc_id: ObjectId(doc_id.toString()),
+      pack: { $exists: true }
+    }
+    const update = {
+      $push: {
+        pack: { $each: newUpdates }
+      },
+      $inc: {
+        n: n,
+        sz: sz
+      },
+      $set: {
+        'meta.end_ts': last.meta.end_ts,
+        v_end: last.v
+      }
+    }
+    if (lastUpdate.expiresAt && temporary) {
+      update.$set.expiresAt = new Date(Date.now() + 7 * DAYS)
+    }
+    logger.log(
+      { project_id, doc_id, lastUpdate, newUpdates },
+      'appending updates to existing pack'
+    )
+    Metrics.inc(`append-pack-${temporary ? 'temporary' : 'permanent'}`)
+    return db.docHistory.findAndModify(
+      { query, update, new: true, fields: { meta: 1, v_end: 1 } },
+      callback
+    )
+  },
+
+  // Retrieve all changes for a document
+
+  getOpsByVersionRange(project_id, doc_id, fromVersion, toVersion, callback) {
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return PackManager.loadPacksByVersionRange(
+      project_id,
+      doc_id,
+      fromVersion,
+      toVersion,
+      function(error) {
+        const query = { doc_id: ObjectId(doc_id.toString()) }
+        if (toVersion != null) {
+          query.v = { $lte: toVersion }
+        }
+        if (fromVersion != null) {
+          query.v_end = { $gte: fromVersion }
+        }
+        // console.log "query:", query
+        return db.docHistory.find(query).sort({ v: -1 }, function(err, result) {
+          if (err != null) {
+            return callback(err)
+          }
+          // console.log "getOpsByVersionRange:", err, result
+          const updates = []
+          const opInRange = function(op, from, to) {
+            if (fromVersion != null && op.v < fromVersion) {
+              return false
+            }
+            if (toVersion != null && op.v > toVersion) {
+              return false
+            }
+            return true
+          }
+          for (const docHistory of Array.from(result)) {
+            // console.log 'adding', docHistory.pack
+            for (const op of Array.from(docHistory.pack.reverse())) {
+              if (opInRange(op, fromVersion, toVersion)) {
+                op.project_id = docHistory.project_id
+                op.doc_id = docHistory.doc_id
+                // console.log "added op", op.v, fromVersion, toVersion
+                updates.push(op)
+              }
+            }
+          }
+          return callback(null, updates)
+        })
+      }
+    )
+  },
+
+  loadPacksByVersionRange(
+    project_id,
+    doc_id,
+    fromVersion,
+    toVersion,
+    callback
+  ) {
+    return PackManager.getIndex(doc_id, function(err, indexResult) {
+      let pack
+      if (err != null) {
+        return callback(err)
+      }
+      const indexPacks =
+        (indexResult != null ? indexResult.packs : undefined) || []
+      const packInRange = function(pack, from, to) {
+        if (fromVersion != null && pack.v_end < fromVersion) {
+          return false
+        }
+        if (toVersion != null && pack.v > toVersion) {
+          return false
+        }
+        return true
+      }
+      const neededIds = (() => {
+        const result = []
+        for (pack of Array.from(indexPacks)) {
+          if (packInRange(pack, fromVersion, toVersion)) {
+            result.push(pack._id)
+          }
+        }
+        return result
+      })()
+      if (neededIds.length) {
+        return PackManager.fetchPacksIfNeeded(
+          project_id,
+          doc_id,
+          neededIds,
+          callback
+        )
+      } else {
+        return callback()
+      }
+    })
+  },
+
+  fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
+    let id
+    return db.docHistory.find(
+      {
+        _id: {
+          $in: (() => {
+            const result = []
+            for (id of Array.from(pack_ids)) {
+              result.push(ObjectId(id))
+            }
+            return result
+          })()
+        }
+      },
+      { _id: 1 },
+      function(err, loadedPacks) {
+        if (err != null) {
+          return callback(err)
+        }
+        const allPackIds = (() => {
+          const result1 = []
+          for (id of Array.from(pack_ids)) {
+            result1.push(id.toString())
+          }
+          return result1
+        })()
+        const loadedPackIds = Array.from(loadedPacks).map(pack =>
+          pack._id.toString()
+        )
+        const packIdsToFetch = _.difference(allPackIds, loadedPackIds)
+        logger.log(
+          { project_id, doc_id, loadedPackIds, allPackIds, packIdsToFetch },
+          'analysed packs'
+        )
+        if (packIdsToFetch.length === 0) {
+          return callback()
+        }
+        return async.eachLimit(
+          packIdsToFetch,
+          4,
+          (pack_id, cb) =>
+            MongoAWS.unArchivePack(project_id, doc_id, pack_id, cb),
+          function(err) {
+            if (err != null) {
+              return callback(err)
+            }
+            logger.log({ project_id, doc_id }, 'done unarchiving')
+            return callback()
+          }
+        )
+      }
+    )
+  },
+
+  // Retrieve all changes across a project
+
+  makeProjectIterator(project_id, before, callback) {
+    // get all the docHistory Entries
+    return db.docHistory
+      .find({ project_id: ObjectId(project_id) }, { pack: false })
+      .sort({ 'meta.end_ts': -1 }, function(err, packs) {
+        let pack
+        if (err != null) {
+          return callback(err)
+        }
+        const allPacks = []
+        const seenIds = {}
+        for (pack of Array.from(packs)) {
+          allPacks.push(pack)
+          seenIds[pack._id] = true
+        }
+        return db.docHistoryIndex.find(
+          { project_id: ObjectId(project_id) },
+          function(err, indexes) {
+            if (err != null) {
+              return callback(err)
+            }
+            for (const index of Array.from(indexes)) {
+              for (pack of Array.from(index.packs)) {
+                if (!seenIds[pack._id]) {
+                  pack.project_id = index.project_id
+                  pack.doc_id = index._id
+                  pack.fromIndex = true
+                  allPacks.push(pack)
+                  seenIds[pack._id] = true
+                }
+              }
+            }
+            return callback(
+              null,
+              new ProjectIterator(allPacks, before, PackManager.getPackById)
+            )
+          }
+        )
+      })
+  },
+
+  getPackById(project_id, doc_id, pack_id, callback) {
+    return db.docHistory.findOne({ _id: pack_id }, function(err, pack) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (pack == null) {
+        return MongoAWS.unArchivePack(project_id, doc_id, pack_id, callback)
+      } else if (pack.expiresAt != null && pack.temporary === false) {
+        // we only need to touch the TTL when listing the changes in the project
+        // because diffs on individual documents are always done after that
+        return PackManager.increaseTTL(pack, callback)
+        // only do this for cached packs, not temporary ones to avoid older packs
+        // being kept longer than newer ones (which messes up the last update version)
+      } else {
+        return callback(null, pack)
+      }
+    })
+  },
+
+  increaseTTL(pack, callback) {
+    if (pack.expiresAt < new Date(Date.now() + 6 * DAYS)) {
+      // update cache expiry since we are using this pack
+      return db.docHistory.findAndModify(
+        {
+          query: { _id: pack._id },
+          update: { $set: { expiresAt: new Date(Date.now() + 7 * DAYS) } }
+        },
+        err => callback(err, pack)
+      )
+    } else {
+      return callback(null, pack)
+    }
+  },
+
+  // Manage docHistoryIndex collection
+
+  getIndex(doc_id, callback) {
+    return db.docHistoryIndex.findOne(
+      { _id: ObjectId(doc_id.toString()) },
+      callback
+    )
+  },
+
+  getPackFromIndex(doc_id, pack_id, callback) {
+    return db.docHistoryIndex.findOne(
+      { _id: ObjectId(doc_id.toString()), 'packs._id': pack_id },
+      { 'packs.$': 1 },
+      callback
+    )
+  },
+
+  getLastPackFromIndex(doc_id, callback) {
+    return db.docHistoryIndex.findOne(
+      { _id: ObjectId(doc_id.toString()) },
+      { packs: { $slice: -1 } },
+      function(err, indexPack) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (indexPack == null) {
+          return callback()
+        }
+        return callback(null, indexPack[0])
+      }
+    )
+  },
+
+  getIndexWithKeys(doc_id, callback) {
+    return PackManager.getIndex(doc_id, function(err, index) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (index == null) {
+        return callback()
+      }
+      for (const pack of Array.from(
+        (index != null ? index.packs : undefined) || []
+      )) {
+        index[pack._id] = pack
+      }
+      return callback(null, index)
+    })
+  },
+
+  initialiseIndex(project_id, doc_id, callback) {
+    return PackManager.findCompletedPacks(project_id, doc_id, function(
+      err,
+      packs
+    ) {
+      // console.log 'err', err, 'packs', packs, packs?.length
+      if (err != null) {
+        return callback(err)
+      }
+      if (packs == null) {
+        return callback()
+      }
+      return PackManager.insertPacksIntoIndexWithLock(
+        project_id,
+        doc_id,
+        packs,
+        callback
+      )
+    })
+  },
+
+  updateIndex(project_id, doc_id, callback) {
+    // find all packs prior to current pack
+    return PackManager.findUnindexedPacks(project_id, doc_id, function(
+      err,
+      newPacks
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (newPacks == null || newPacks.length === 0) {
+        return callback()
+      }
+      return PackManager.insertPacksIntoIndexWithLock(
+        project_id,
+        doc_id,
+        newPacks,
+        function(err) {
+          if (err != null) {
+            return callback(err)
+          }
+          logger.log(
+            { project_id, doc_id, newPacks },
+            'added new packs to index'
+          )
+          return callback()
+        }
+      )
+    })
+  },
+
+  findCompletedPacks(project_id, doc_id, callback) {
+    const query = {
+      doc_id: ObjectId(doc_id.toString()),
+      expiresAt: { $exists: false }
+    }
+    return db.docHistory
+      .find(query, { pack: false })
+      .sort({ v: 1 }, function(err, packs) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (packs == null) {
+          return callback()
+        }
+        if (!(packs != null ? packs.length : undefined)) {
+          return callback()
+        }
+        const last = packs.pop() // discard the last pack, if it's still in progress
+        if (last.finalised) {
+          packs.push(last)
+        } // it's finalised so we push it back to archive it
+        return callback(null, packs)
+      })
+  },
+
+  findPacks(project_id, doc_id, callback) {
+    const query = {
+      doc_id: ObjectId(doc_id.toString()),
+      expiresAt: { $exists: false }
+    }
+    return db.docHistory
+      .find(query, { pack: false })
+      .sort({ v: 1 }, function(err, packs) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (packs == null) {
+          return callback()
+        }
+        if (!(packs != null ? packs.length : undefined)) {
+          return callback()
+        }
+        return callback(null, packs)
+      })
+  },
+
+  findUnindexedPacks(project_id, doc_id, callback) {
+    return PackManager.getIndexWithKeys(doc_id, function(err, indexResult) {
+      if (err != null) {
+        return callback(err)
+      }
+      return PackManager.findCompletedPacks(project_id, doc_id, function(
+        err,
+        historyPacks
+      ) {
+        let pack
+        if (err != null) {
+          return callback(err)
+        }
+        if (historyPacks == null) {
+          return callback()
+        }
+        // select only the new packs not already in the index
+        let newPacks = (() => {
+          const result = []
+          for (pack of Array.from(historyPacks)) {
+            if (
+              (indexResult != null ? indexResult[pack._id] : undefined) == null
+            ) {
+              result.push(pack)
+            }
+          }
+          return result
+        })()
+        newPacks = (() => {
+          const result1 = []
+          for (pack of Array.from(newPacks)) {
+            result1.push(
+              _.omit(
+                pack,
+                'doc_id',
+                'project_id',
+                'n',
+                'sz',
+                'last_checked',
+                'finalised'
+              )
+            )
+          }
+          return result1
+        })()
+        if (newPacks.length) {
+          logger.log(
+            { project_id, doc_id, n: newPacks.length },
+            'found new packs'
+          )
+        }
+        return callback(null, newPacks)
+      })
+    })
+  },
+
+  insertPacksIntoIndexWithLock(project_id, doc_id, newPacks, callback) {
+    return LockManager.runWithLock(
+      keys.historyIndexLock({ doc_id }),
+      releaseLock =>
+        PackManager._insertPacksIntoIndex(
+          project_id,
+          doc_id,
+          newPacks,
+          releaseLock
+        ),
+      callback
+    )
+  },
+
+  _insertPacksIntoIndex(project_id, doc_id, newPacks, callback) {
+    return db.docHistoryIndex.findAndModify(
+      {
+        query: { _id: ObjectId(doc_id.toString()) },
+        update: {
+          $setOnInsert: { project_id: ObjectId(project_id.toString()) },
+          $push: {
+            packs: { $each: newPacks, $sort: { v: 1 } }
+          }
+        },
+        upsert: true
+      },
+      callback
+    )
+  },
+
+  // Archiving packs to S3
+
+  archivePack(project_id, doc_id, pack_id, callback) {
+    const clearFlagOnError = function(err, cb) {
+      if (err != null) {
+        // clear the inS3 flag on error
+        return PackManager.clearPackAsArchiveInProgress(
+          project_id,
+          doc_id,
+          pack_id,
+          function(err2) {
+            if (err2 != null) {
+              return cb(err2)
+            }
+            return cb(err)
+          }
+        )
+      } else {
+        return cb()
+      }
+    }
+    return async.series(
+      [
+        cb =>
+          PackManager.checkArchiveNotInProgress(
+            project_id,
+            doc_id,
+            pack_id,
+            cb
+          ),
+        cb =>
+          PackManager.markPackAsArchiveInProgress(
+            project_id,
+            doc_id,
+            pack_id,
+            cb
+          ),
+        cb =>
+          MongoAWS.archivePack(project_id, doc_id, pack_id, err =>
+            clearFlagOnError(err, cb)
+          ),
+        cb =>
+          PackManager.checkArchivedPack(project_id, doc_id, pack_id, err =>
+            clearFlagOnError(err, cb)
+          ),
+        cb => PackManager.markPackAsArchived(project_id, doc_id, pack_id, cb),
+        cb =>
+          PackManager.setTTLOnArchivedPack(
+            project_id,
+            doc_id,
+            pack_id,
+            callback
+          )
+      ],
+      callback
+    )
+  },
+
+  checkArchivedPack(project_id, doc_id, pack_id, callback) {
+    return db.docHistory.findOne({ _id: pack_id }, function(err, pack) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (pack == null) {
+        return callback(new Error('pack not found'))
+      }
+      return MongoAWS.readArchivedPack(project_id, doc_id, pack_id, function(
+        err,
+        result
+      ) {
+        delete result.last_checked
+        delete pack.last_checked
+        // need to compare ids as ObjectIds with .equals()
+        for (const key of ['_id', 'project_id', 'doc_id']) {
+          if (result[key].equals(pack[key])) {
+            result[key] = pack[key]
+          }
+        }
+        for (let i = 0; i < result.pack.length; i++) {
+          const op = result.pack[i]
+          if (op._id != null && op._id.equals(pack.pack[i]._id)) {
+            op._id = pack.pack[i]._id
+          }
+        }
+        if (_.isEqual(pack, result)) {
+          return callback()
+        } else {
+          logger.err(
+            {
+              pack,
+              result,
+              jsondiff: JSON.stringify(pack) === JSON.stringify(result)
+            },
+            'difference when comparing packs'
+          )
+          return callback(
+            new Error('pack retrieved from s3 does not match pack in mongo')
+          )
+        }
+      })
+    })
+  },
+  // Extra methods to test archive/unarchive for a doc_id
+
+  pushOldPacks(project_id, doc_id, callback) {
+    return PackManager.findPacks(project_id, doc_id, function(err, packs) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (!(packs != null ? packs.length : undefined)) {
+        return callback()
+      }
+      return PackManager.processOldPack(
+        project_id,
+        doc_id,
+        packs[0]._id,
+        callback
+      )
+    })
+  },
+
+  pullOldPacks(project_id, doc_id, callback) {
+    return PackManager.loadPacksByVersionRange(
+      project_id,
+      doc_id,
+      null,
+      null,
+      callback
+    )
+  },
+
+  // Processing old packs via worker
+
+  processOldPack(project_id, doc_id, pack_id, callback) {
+    const markAsChecked = err =>
+      PackManager.markPackAsChecked(project_id, doc_id, pack_id, function(
+        err2
+      ) {
+        if (err2 != null) {
+          return callback(err2)
+        }
+        return callback(err)
+      })
+    logger.log({ project_id, doc_id }, 'processing old packs')
+    return db.docHistory.findOne({ _id: pack_id }, function(err, pack) {
+      if (err != null) {
+        return markAsChecked(err)
+      }
+      if (pack == null) {
+        return markAsChecked()
+      }
+      if (pack.expiresAt != null) {
+        return callback()
+      } // return directly
+      return PackManager.finaliseIfNeeded(
+        project_id,
+        doc_id,
+        pack._id,
+        pack,
+        function(err) {
+          if (err != null) {
+            return markAsChecked(err)
+          }
+          return PackManager.updateIndexIfNeeded(project_id, doc_id, function(
+            err
+          ) {
+            if (err != null) {
+              return markAsChecked(err)
+            }
+            return PackManager.findUnarchivedPacks(project_id, doc_id, function(
+              err,
+              unarchivedPacks
+            ) {
+              if (err != null) {
+                return markAsChecked(err)
+              }
+              if (
+                !(unarchivedPacks != null ? unarchivedPacks.length : undefined)
+              ) {
+                logger.log({ project_id, doc_id }, 'no packs need archiving')
+                return markAsChecked()
+              }
+              return async.eachSeries(
+                unarchivedPacks,
+                (pack, cb) =>
+                  PackManager.archivePack(project_id, doc_id, pack._id, cb),
+                function(err) {
+                  if (err != null) {
+                    return markAsChecked(err)
+                  }
+                  logger.log({ project_id, doc_id }, 'done processing')
+                  return markAsChecked()
+                }
+              )
+            })
+          })
+        }
+      )
+    })
+  },
+
+  finaliseIfNeeded(project_id, doc_id, pack_id, pack, callback) {
+    const sz = pack.sz / (1024 * 1024) // in fractions of a megabyte
+    const n = pack.n / 1024 // in fraction of 1024 ops
+    const age = (Date.now() - pack.meta.end_ts) / DAYS
+    if (age < 30) {
+      // always keep if less than 1 month old
+      logger.log({ project_id, doc_id, pack_id, age }, 'less than 30 days old')
+      return callback()
+    }
+    // compute an archiving threshold which decreases for each month of age
+    const archive_threshold = 30 / age
+    if (sz > archive_threshold || n > archive_threshold || age > 90) {
+      logger.log(
+        { project_id, doc_id, pack_id, age, archive_threshold, sz, n },
+        'meets archive threshold'
+      )
+      return PackManager.markPackAsFinalisedWithLock(
+        project_id,
+        doc_id,
+        pack_id,
+        callback
+      )
+    } else {
+      logger.log(
+        { project_id, doc_id, pack_id, age, archive_threshold, sz, n },
+        'does not meet archive threshold'
+      )
+      return callback()
+    }
+  },
+
+  markPackAsFinalisedWithLock(project_id, doc_id, pack_id, callback) {
+    return LockManager.runWithLock(
+      keys.historyLock({ doc_id }),
+      releaseLock =>
+        PackManager._markPackAsFinalised(
+          project_id,
+          doc_id,
+          pack_id,
+          releaseLock
+        ),
+      callback
+    )
+  },
+
+  _markPackAsFinalised(project_id, doc_id, pack_id, callback) {
+    logger.log({ project_id, doc_id, pack_id }, 'marking pack as finalised')
+    return db.docHistory.findAndModify(
+      {
+        query: { _id: pack_id },
+        update: { $set: { finalised: true } }
+      },
+      callback
+    )
+  },
+
+  updateIndexIfNeeded(project_id, doc_id, callback) {
+    logger.log({ project_id, doc_id }, 'archiving old packs')
+    return PackManager.getIndexWithKeys(doc_id, function(err, index) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (index == null) {
+        return PackManager.initialiseIndex(project_id, doc_id, callback)
+      } else {
+        return PackManager.updateIndex(project_id, doc_id, callback)
+      }
+    })
+  },
+
+  markPackAsChecked(project_id, doc_id, pack_id, callback) {
+    logger.log({ project_id, doc_id, pack_id }, 'marking pack as checked')
+    return db.docHistory.findAndModify(
+      {
+        query: { _id: pack_id },
+        update: { $currentDate: { last_checked: true } }
+      },
+      callback
+    )
+  },
+
+  findUnarchivedPacks(project_id, doc_id, callback) {
+    return PackManager.getIndex(doc_id, function(err, indexResult) {
+      if (err != null) {
+        return callback(err)
+      }
+      const indexPacks =
+        (indexResult != null ? indexResult.packs : undefined) || []
+      const unArchivedPacks = (() => {
+        const result = []
+        for (const pack of Array.from(indexPacks)) {
+          if (pack.inS3 == null) {
+            result.push(pack)
+          }
+        }
+        return result
+      })()
+      if (unArchivedPacks.length) {
+        logger.log(
+          { project_id, doc_id, n: unArchivedPacks.length },
+          'find unarchived packs'
+        )
+      }
+      return callback(null, unArchivedPacks)
+    })
+  },
+
+  // Archive locking flags
+
+  checkArchiveNotInProgress(project_id, doc_id, pack_id, callback) {
+    logger.log(
+      { project_id, doc_id, pack_id },
+      'checking if archive in progress'
+    )
+    return PackManager.getPackFromIndex(doc_id, pack_id, function(err, result) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (result == null) {
+        return callback(new Error('pack not found in index'))
+      }
+      if (result.inS3) {
+        return callback(new Error('pack archiving already done'))
+      } else if (result.inS3 != null) {
+        return callback(new Error('pack archiving already in progress'))
+      } else {
+        return callback()
+      }
+    })
+  },
+
+  markPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
+    logger.log(
+      { project_id, doc_id },
+      'marking pack as archive in progress status'
+    )
+    return db.docHistoryIndex.findAndModify(
+      {
+        query: {
+          _id: ObjectId(doc_id.toString()),
+          packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
+        },
+        fields: { 'packs.$': 1 },
+        update: { $set: { 'packs.$.inS3': false } }
+      },
+      function(err, result) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (result == null) {
+          return callback(new Error('archive is already in progress'))
+        }
+        logger.log(
+          { project_id, doc_id, pack_id },
+          'marked as archive in progress'
+        )
+        return callback()
+      }
+    )
+  },
+
+  clearPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
+    logger.log(
+      { project_id, doc_id, pack_id },
+      'clearing as archive in progress'
+    )
+    return db.docHistoryIndex.findAndModify(
+      {
+        query: {
+          _id: ObjectId(doc_id.toString()),
+          packs: { $elemMatch: { _id: pack_id, inS3: false } }
+        },
+        fields: { 'packs.$': 1 },
+        update: { $unset: { 'packs.$.inS3': true } }
+      },
+      callback
+    )
+  },
+
+  markPackAsArchived(project_id, doc_id, pack_id, callback) {
+    logger.log({ project_id, doc_id, pack_id }, 'marking pack as archived')
+    return db.docHistoryIndex.findAndModify(
+      {
+        query: {
+          _id: ObjectId(doc_id.toString()),
+          packs: { $elemMatch: { _id: pack_id, inS3: false } }
+        },
+        fields: { 'packs.$': 1 },
+        update: { $set: { 'packs.$.inS3': true } }
+      },
+      function(err, result) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (result == null) {
+          return callback(new Error('archive is not marked as progress'))
+        }
+        logger.log({ project_id, doc_id, pack_id }, 'marked as archived')
+        return callback()
+      }
+    )
+  },
+
+  setTTLOnArchivedPack(project_id, doc_id, pack_id, callback) {
+    return db.docHistory.findAndModify(
+      {
+        query: { _id: pack_id },
+        update: { $set: { expiresAt: new Date(Date.now() + 1 * DAYS) } }
+      },
+      function(err) {
+        logger.log({ project_id, doc_id, pack_id }, 'set expiry on pack')
+        return callback()
+      }
+    )
+  }
+}
+
+//	_getOneDayInFutureWithRandomDelay: ->
+//		thirtyMins = 1000 * 60 * 30
+//		randomThirtyMinMax = Math.ceil(Math.random() * thirtyMins)
+//		return new Date(Date.now() + randomThirtyMinMax + 1*DAYS)

services/track-changes/app/js/PackWorker.js

@@ -0,0 +1,211 @@
+/* eslint-disable
+    camelcase,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let LIMIT, pending
+let project_id, doc_id
+const Settings = require('settings-sharelatex')
+const async = require('async')
+const _ = require('underscore')
+const { db, ObjectId, BSON } = require('./mongojs')
+const fs = require('fs')
+const Metrics = require('metrics-sharelatex')
+Metrics.initialize('track-changes')
+const logger = require('logger-sharelatex')
+logger.initialize('track-changes-packworker')
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
+  logger.initializeErrorReporting(Settings.sentry.dsn)
+}
+
+const DAYS = 24 * 3600 * 1000
+
+const LockManager = require('./LockManager')
+const PackManager = require('./PackManager')
+
+// this worker script is forked by the main process to look for
+// document histories which can be archived
+
+const source = process.argv[2]
+const DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
+const TIMEOUT = Number(process.argv[4]) || 30 * 60 * 1000
+let COUNT = 0 // number processed
+let TOTAL = 0 // total number to process
+
+if (!source.match(/^[0-9]+$/)) {
+  const file = fs.readFileSync(source)
+  const result = (() => {
+    const result1 = []
+    for (const line of Array.from(file.toString().split('\n'))) {
+      ;[project_id, doc_id] = Array.from(line.split(' '))
+      result1.push({ doc_id, project_id })
+    }
+    return result1
+  })()
+  pending = _.filter(result, row =>
+    __guard__(row != null ? row.doc_id : undefined, x =>
+      x.match(/^[a-f0-9]{24}$/)
+    )
+  )
+} else {
+  LIMIT = Number(process.argv[2]) || 1000
+}
+
+let shutDownRequested = false
+const shutDownTimer = setTimeout(function() {
+  logger.log('pack timed out, requesting shutdown')
+  // start the shutdown on the next pack
+  shutDownRequested = true
+  // do a hard shutdown after a further 5 minutes
+  const hardTimeout = setTimeout(function() {
+    logger.error('HARD TIMEOUT in pack archive worker')
+    return process.exit()
+  }, 5 * 60 * 1000)
+  return hardTimeout.unref()
+}, TIMEOUT)
+
+logger.log(
+  `checking for updates, limit=${LIMIT}, delay=${DOCUMENT_PACK_DELAY}, timeout=${TIMEOUT}`
+)
+
+// work around for https://github.com/mafintosh/mongojs/issues/224
+db.close = function(callback) {
+  return this._getServer(function(err, server) {
+    if (err != null) {
+      return callback(err)
+    }
+    server = server.destroy != null ? server : server.topology
+    server.destroy(true, true)
+    return callback()
+  })
+}
+
+const finish = function() {
+  if (shutDownTimer != null) {
+    logger.log('cancelling timeout')
+    clearTimeout(shutDownTimer)
+  }
+  logger.log('closing db')
+  return db.close(function() {
+    logger.log('closing LockManager Redis Connection')
+    return LockManager.close(function() {
+      logger.log(
+        { processedCount: COUNT, allCount: TOTAL },
+        'ready to exit from pack archive worker'
+      )
+      const hardTimeout = setTimeout(function() {
+        logger.error('hard exit from pack archive worker')
+        return process.exit(1)
+      }, 5 * 1000)
+      return hardTimeout.unref()
+    })
+  })
+}
+
+process.on('exit', code => logger.log({ code }, 'pack archive worker exited'))
+
+const processUpdates = pending =>
+  async.eachSeries(
+    pending,
+    function(result, callback) {
+      let _id
+      ;({ _id, project_id, doc_id } = result)
+      COUNT++
+      logger.log({ project_id, doc_id }, `processing ${COUNT}/${TOTAL}`)
+      if (project_id == null || doc_id == null) {
+        logger.log(
+          { project_id, doc_id },
+          'skipping pack, missing project/doc id'
+        )
+        return callback()
+      }
+      const handler = function(err, result) {
+        if (err != null && err.code === 'InternalError' && err.retryable) {
+          logger.warn(
+            { err, result },
+            'ignoring S3 error in pack archive worker'
+          )
+          // Ignore any s3 errors due to random problems
+          err = null
+        }
+        if (err != null) {
+          logger.error({ err, result }, 'error in pack archive worker')
+          return callback(err)
+        }
+        if (shutDownRequested) {
+          logger.warn('shutting down pack archive worker')
+          return callback(new Error('shutdown'))
+        }
+        return setTimeout(() => callback(err, result), DOCUMENT_PACK_DELAY)
+      }
+      if (_id == null) {
+        return PackManager.pushOldPacks(project_id, doc_id, handler)
+      } else {
+        return PackManager.processOldPack(project_id, doc_id, _id, handler)
+      }
+    },
+    function(err, results) {
+      if (err != null && err.message !== 'shutdown') {
+        logger.error({ err }, 'error in pack archive worker processUpdates')
+      }
+      return finish()
+    }
+  )
+// find the packs which can be archived
+
+const ObjectIdFromDate = function(date) {
+  const id = Math.floor(date.getTime() / 1000).toString(16) + '0000000000000000'
+  return ObjectId(id)
+}
+
+// new approach, two passes
+// find packs to be marked as finalised:true, those which have a newer pack present
+// then only consider finalised:true packs for archiving
+
+if (pending != null) {
+  logger.log(`got ${pending.length} entries from ${source}`)
+  processUpdates(pending)
+} else {
+  const oneWeekAgo = new Date(Date.now() - 7 * DAYS)
+  db.docHistory
+    .find(
+      {
+        expiresAt: { $exists: false },
+        project_id: { $exists: true },
+        v_end: { $exists: true },
+        _id: { $lt: ObjectIdFromDate(oneWeekAgo) },
+        last_checked: { $lt: oneWeekAgo }
+      },
+      { _id: 1, doc_id: 1, project_id: 1 }
+    )
+    .sort({
+      last_checked: 1
+    })
+    .limit(LIMIT, function(err, results) {
+      if (err != null) {
+        logger.log({ err }, 'error checking for updates')
+        finish()
+        return
+      }
+      pending = _.uniq(results, false, result => result.doc_id.toString())
+      TOTAL = pending.length
+      logger.log(`found ${TOTAL} documents to archive`)
+      return processUpdates(pending)
+    })
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/app/js/ProjectIterator.js

@@ -0,0 +1,109 @@
+/* eslint-disable
+    no-unmodified-loop-condition,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let ProjectIterator
+const Heap = require('heap')
+
+module.exports = ProjectIterator = ProjectIterator = class ProjectIterator {
+  constructor(packs, before, getPackByIdFn) {
+    this.before = before
+    this.getPackByIdFn = getPackByIdFn
+    const byEndTs = (a, b) =>
+      b.meta.end_ts - a.meta.end_ts || a.fromIndex - b.fromIndex
+    this.packs = packs.slice().sort(byEndTs)
+    this.queue = new Heap(byEndTs)
+  }
+
+  next(callback) {
+    //  what's up next
+    // console.log ">>> top item", iterator.packs[0]
+    const iterator = this
+    const { before } = this
+    const { queue } = iterator
+    const opsToReturn = []
+    let nextPack = iterator.packs[0]
+    let lowWaterMark =
+      (nextPack != null ? nextPack.meta.end_ts : undefined) || 0
+    let nextItem = queue.peek()
+
+    // console.log "queue empty?", queue.empty()
+    // console.log "nextItem", nextItem
+    // console.log "nextItem.meta.end_ts", nextItem?.meta.end_ts
+    // console.log "lowWaterMark", lowWaterMark
+
+    while (
+      before != null &&
+      (nextPack != null ? nextPack.meta.start_ts : undefined) > before
+    ) {
+      // discard pack that is outside range
+      iterator.packs.shift()
+      nextPack = iterator.packs[0]
+      lowWaterMark = (nextPack != null ? nextPack.meta.end_ts : undefined) || 0
+    }
+
+    if (
+      (queue.empty() ||
+        (nextItem != null ? nextItem.meta.end_ts : undefined) <=
+          lowWaterMark) &&
+      nextPack != null
+    ) {
+      // retrieve the next pack and populate the queue
+      return this.getPackByIdFn(
+        nextPack.project_id,
+        nextPack.doc_id,
+        nextPack._id,
+        function(err, pack) {
+          if (err != null) {
+            return callback(err)
+          }
+          iterator.packs.shift() // have now retrieved this pack, remove it
+          // console.log "got pack", pack
+          for (const op of Array.from(pack.pack)) {
+            // console.log "adding op", op
+            if (before == null || op.meta.end_ts < before) {
+              op.doc_id = nextPack.doc_id
+              op.project_id = nextPack.project_id
+              queue.push(op)
+            }
+          }
+          // now try again
+          return iterator.next(callback)
+        }
+      )
+    }
+
+    // console.log "nextItem", nextItem, "lowWaterMark", lowWaterMark
+    while (
+      nextItem != null &&
+      (nextItem != null ? nextItem.meta.end_ts : undefined) > lowWaterMark
+    ) {
+      opsToReturn.push(nextItem)
+      queue.pop()
+      nextItem = queue.peek()
+    }
+
+    // console.log "queue empty?", queue.empty()
+    // console.log "nextPack", nextPack?
+
+    if (queue.empty() && nextPack == null) {
+      // got everything
+      iterator._done = true
+    }
+
+    return callback(null, opsToReturn)
+  }
+
+  done() {
+    return this._done
+  }
+}

services/track-changes/app/js/RedisManager.js

@@ -0,0 +1,166 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RedisManager
+const Settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(Settings.redis.history)
+const Keys = Settings.redis.history.key_schema
+const async = require('async')
+
+module.exports = RedisManager = {
+  getOldestDocUpdates(doc_id, batchSize, callback) {
+    if (callback == null) {
+      callback = function(error, jsonUpdates) {}
+    }
+    const key = Keys.uncompressedHistoryOps({ doc_id })
+    return rclient.lrange(key, 0, batchSize - 1, callback)
+  },
+
+  expandDocUpdates(jsonUpdates, callback) {
+    let rawUpdates
+    if (callback == null) {
+      callback = function(error, rawUpdates) {}
+    }
+    try {
+      rawUpdates = Array.from(jsonUpdates || []).map(update =>
+        JSON.parse(update)
+      )
+    } catch (e) {
+      return callback(e)
+    }
+    return callback(null, rawUpdates)
+  },
+
+  deleteAppliedDocUpdates(project_id, doc_id, docUpdates, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const multi = rclient.multi()
+    // Delete all the updates which have been applied (exact match)
+    for (const update of Array.from(docUpdates || [])) {
+      multi.lrem(Keys.uncompressedHistoryOps({ doc_id }), 1, update)
+    }
+    return multi.exec(function(error, results) {
+      if (error != null) {
+        return callback(error)
+      }
+      // It's ok to delete the doc_id from the set here. Even though the list
+      // of updates may not be empty, we will continue to process it until it is.
+      return rclient.srem(
+        Keys.docsWithHistoryOps({ project_id }),
+        doc_id,
+        function(error) {
+          if (error != null) {
+            return callback(error)
+          }
+          return callback(null)
+        }
+      )
+    })
+  },
+
+  getDocIdsWithHistoryOps(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, doc_ids) {}
+    }
+    return rclient.smembers(Keys.docsWithHistoryOps({ project_id }), callback)
+  },
+
+  // iterate over keys asynchronously using redis scan (non-blocking)
+  // handle all the cluster nodes or single redis server
+  _getKeys(pattern, callback) {
+    const nodes = (typeof rclient.nodes === 'function'
+      ? rclient.nodes('master')
+      : undefined) || [rclient]
+    const doKeyLookupForNode = (node, cb) =>
+      RedisManager._getKeysFromNode(node, pattern, cb)
+    return async.concatSeries(nodes, doKeyLookupForNode, callback)
+  },
+
+  _getKeysFromNode(node, pattern, callback) {
+    let cursor = 0 // redis iterator
+    const keySet = {} // use hash to avoid duplicate results
+    // scan over all keys looking for pattern
+    var doIteration = cb =>
+      node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function(
+        error,
+        reply
+      ) {
+        let keys
+        if (error != null) {
+          return callback(error)
+        }
+        ;[cursor, keys] = Array.from(reply)
+        for (const key of Array.from(keys)) {
+          keySet[key] = true
+        }
+        if (cursor === '0') {
+          // note redis returns string result not numeric
+          return callback(null, Object.keys(keySet))
+        } else {
+          return doIteration()
+        }
+      })
+    return doIteration()
+  },
+
+  // extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b
+  // or DocsWithHistoryOps:{57fd0b1f53a8396d22b2c24b} (for redis cluster)
+  _extractIds(keyList) {
+    const ids = (() => {
+      const result = []
+      for (const key of Array.from(keyList)) {
+        const m = key.match(/:\{?([0-9a-f]{24})\}?/) // extract object id
+        result.push(m[1])
+      }
+      return result
+    })()
+    return ids
+  },
+
+  getProjectIdsWithHistoryOps(callback) {
+    if (callback == null) {
+      callback = function(error, project_ids) {}
+    }
+    return RedisManager._getKeys(
+      Keys.docsWithHistoryOps({ project_id: '*' }),
+      function(error, project_keys) {
+        if (error != null) {
+          return callback(error)
+        }
+        const project_ids = RedisManager._extractIds(project_keys)
+        return callback(error, project_ids)
+      }
+    )
+  },
+
+  getAllDocIdsWithHistoryOps(callback) {
+    // return all the docids, to find dangling history entries after
+    // everything is flushed.
+    if (callback == null) {
+      callback = function(error, doc_ids) {}
+    }
+    return RedisManager._getKeys(
+      Keys.uncompressedHistoryOps({ doc_id: '*' }),
+      function(error, doc_keys) {
+        if (error != null) {
+          return callback(error)
+        }
+        const doc_ids = RedisManager._extractIds(doc_keys)
+        return callback(error, doc_ids)
+      }
+    )
+  }
+}

services/track-changes/app/js/RestoreManager.js

@@ -0,0 +1,48 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RestoreManager
+const DocumentUpdaterManager = require('./DocumentUpdaterManager')
+const DiffManager = require('./DiffManager')
+const logger = require('logger-sharelatex')
+
+module.exports = RestoreManager = {
+  restoreToBeforeVersion(project_id, doc_id, version, user_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    logger.log({ project_id, doc_id, version, user_id }, 'restoring document')
+    return DiffManager.getDocumentBeforeVersion(
+      project_id,
+      doc_id,
+      version,
+      function(error, content) {
+        if (error != null) {
+          return callback(error)
+        }
+        return DocumentUpdaterManager.setDocument(
+          project_id,
+          doc_id,
+          content,
+          user_id,
+          function(error) {
+            if (error != null) {
+              return callback(error)
+            }
+            return callback()
+          }
+        )
+      }
+    )
+  }
+}

services/track-changes/app/js/UpdateCompressor.js

@@ -0,0 +1,337 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    new-cap,
+    no-throw-literal,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let oneMinute, twoMegabytes, UpdateCompressor
+const strInject = (s1, pos, s2) => s1.slice(0, pos) + s2 + s1.slice(pos)
+const strRemove = (s1, pos, length) => s1.slice(0, pos) + s1.slice(pos + length)
+
+const { diff_match_patch } = require('../lib/diff_match_patch')
+const dmp = new diff_match_patch()
+
+module.exports = UpdateCompressor = {
+  NOOP: 'noop',
+
+  // Updates come from the doc updater in format
+  // {
+  // 	op:   [ { ... op1 ... }, { ... op2 ... } ]
+  // 	meta: { ts: ..., user_id: ... }
+  // }
+  // but it's easier to work with on op per update, so convert these updates to
+  // our compressed format
+  // [{
+  // 	op: op1
+  // 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
+  // }, {
+  // 	op: op2
+  // 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
+  // }]
+  convertToSingleOpUpdates(updates) {
+    const splitUpdates = []
+    for (const update of Array.from(updates)) {
+      // Reject any non-insert or delete ops, i.e. comments
+      const ops = update.op.filter(o => o.i != null || o.d != null)
+      if (ops.length === 0) {
+        splitUpdates.push({
+          op: UpdateCompressor.NOOP,
+          meta: {
+            start_ts: update.meta.start_ts || update.meta.ts,
+            end_ts: update.meta.end_ts || update.meta.ts,
+            user_id: update.meta.user_id
+          },
+          v: update.v
+        })
+      } else {
+        for (const op of Array.from(ops)) {
+          splitUpdates.push({
+            op,
+            meta: {
+              start_ts: update.meta.start_ts || update.meta.ts,
+              end_ts: update.meta.end_ts || update.meta.ts,
+              user_id: update.meta.user_id
+            },
+            v: update.v
+          })
+        }
+      }
+    }
+    return splitUpdates
+  },
+
+  concatUpdatesWithSameVersion(updates) {
+    const concattedUpdates = []
+    for (const update of Array.from(updates)) {
+      const lastUpdate = concattedUpdates[concattedUpdates.length - 1]
+      if (lastUpdate != null && lastUpdate.v === update.v) {
+        if (update.op !== UpdateCompressor.NOOP) {
+          lastUpdate.op.push(update.op)
+        }
+      } else {
+        const nextUpdate = {
+          op: [],
+          meta: update.meta,
+          v: update.v
+        }
+        if (update.op !== UpdateCompressor.NOOP) {
+          nextUpdate.op.push(update.op)
+        }
+        concattedUpdates.push(nextUpdate)
+      }
+    }
+    return concattedUpdates
+  },
+
+  compressRawUpdates(lastPreviousUpdate, rawUpdates) {
+    if (
+      __guard__(
+        lastPreviousUpdate != null ? lastPreviousUpdate.op : undefined,
+        x => x.length
+      ) > 1
+    ) {
+      // if the last previous update was an array op, don't compress onto it.
+      // The avoids cases where array length changes but version number doesn't
+      return [lastPreviousUpdate].concat(
+        UpdateCompressor.compressRawUpdates(null, rawUpdates)
+      )
+    }
+    if (lastPreviousUpdate != null) {
+      rawUpdates = [lastPreviousUpdate].concat(rawUpdates)
+    }
+    let updates = UpdateCompressor.convertToSingleOpUpdates(rawUpdates)
+    updates = UpdateCompressor.compressUpdates(updates)
+    return UpdateCompressor.concatUpdatesWithSameVersion(updates)
+  },
+
+  compressUpdates(updates) {
+    if (updates.length === 0) {
+      return []
+    }
+
+    let compressedUpdates = [updates.shift()]
+    for (const update of Array.from(updates)) {
+      const lastCompressedUpdate = compressedUpdates.pop()
+      if (lastCompressedUpdate != null) {
+        compressedUpdates = compressedUpdates.concat(
+          UpdateCompressor._concatTwoUpdates(lastCompressedUpdate, update)
+        )
+      } else {
+        compressedUpdates.push(update)
+      }
+    }
+
+    return compressedUpdates
+  },
+
+  MAX_TIME_BETWEEN_UPDATES: (oneMinute = 60 * 1000),
+  MAX_UPDATE_SIZE: (twoMegabytes = 2 * 1024 * 1024),
+
+  _concatTwoUpdates(firstUpdate, secondUpdate) {
+    let offset
+    firstUpdate = {
+      op: firstUpdate.op,
+      meta: {
+        user_id: firstUpdate.meta.user_id || null,
+        start_ts: firstUpdate.meta.start_ts || firstUpdate.meta.ts,
+        end_ts: firstUpdate.meta.end_ts || firstUpdate.meta.ts
+      },
+      v: firstUpdate.v
+    }
+    secondUpdate = {
+      op: secondUpdate.op,
+      meta: {
+        user_id: secondUpdate.meta.user_id || null,
+        start_ts: secondUpdate.meta.start_ts || secondUpdate.meta.ts,
+        end_ts: secondUpdate.meta.end_ts || secondUpdate.meta.ts
+      },
+      v: secondUpdate.v
+    }
+
+    if (firstUpdate.meta.user_id !== secondUpdate.meta.user_id) {
+      return [firstUpdate, secondUpdate]
+    }
+
+    if (
+      secondUpdate.meta.start_ts - firstUpdate.meta.end_ts >
+      UpdateCompressor.MAX_TIME_BETWEEN_UPDATES
+    ) {
+      return [firstUpdate, secondUpdate]
+    }
+
+    const firstOp = firstUpdate.op
+    const secondOp = secondUpdate.op
+
+    const firstSize =
+      (firstOp.i != null ? firstOp.i.length : undefined) ||
+      (firstOp.d != null ? firstOp.d.length : undefined)
+    const secondSize =
+      (secondOp.i != null ? secondOp.i.length : undefined) ||
+      (secondOp.d != null ? secondOp.d.length : undefined)
+
+    // Two inserts
+    if (
+      firstOp.i != null &&
+      secondOp.i != null &&
+      firstOp.p <= secondOp.p && secondOp.p <= firstOp.p + firstOp.i.length &&
+      firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
+    ) {
+      return [
+        {
+          meta: {
+            start_ts: firstUpdate.meta.start_ts,
+            end_ts: secondUpdate.meta.end_ts,
+            user_id: firstUpdate.meta.user_id
+          },
+          op: {
+            p: firstOp.p,
+            i: strInject(firstOp.i, secondOp.p - firstOp.p, secondOp.i)
+          },
+          v: secondUpdate.v
+        }
+      ]
+      // Two deletes
+    } else if (
+      firstOp.d != null &&
+      secondOp.d != null &&
+      secondOp.p <= firstOp.p && firstOp.p <= secondOp.p + secondOp.d.length &&
+      firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
+    ) {
+      return [
+        {
+          meta: {
+            start_ts: firstUpdate.meta.start_ts,
+            end_ts: secondUpdate.meta.end_ts,
+            user_id: firstUpdate.meta.user_id
+          },
+          op: {
+            p: secondOp.p,
+            d: strInject(secondOp.d, firstOp.p - secondOp.p, firstOp.d)
+          },
+          v: secondUpdate.v
+        }
+      ]
+      // An insert and then a delete
+    } else if (
+      firstOp.i != null &&
+      secondOp.d != null &&
+      firstOp.p <= secondOp.p && secondOp.p <= firstOp.p + firstOp.i.length
+    ) {
+      offset = secondOp.p - firstOp.p
+      const insertedText = firstOp.i.slice(offset, offset + secondOp.d.length)
+      // Only trim the insert when the delete is fully contained within in it
+      if (insertedText === secondOp.d) {
+        const insert = strRemove(firstOp.i, offset, secondOp.d.length)
+        return [
+          {
+            meta: {
+              start_ts: firstUpdate.meta.start_ts,
+              end_ts: secondUpdate.meta.end_ts,
+              user_id: firstUpdate.meta.user_id
+            },
+            op: {
+              p: firstOp.p,
+              i: insert
+            },
+            v: secondUpdate.v
+          }
+        ]
+      } else {
+        // This will only happen if the delete extends outside the insert
+        return [firstUpdate, secondUpdate]
+      }
+
+      // A delete then an insert at the same place, likely a copy-paste of a chunk of content
+    } else if (
+      firstOp.d != null &&
+      secondOp.i != null &&
+      firstOp.p === secondOp.p
+    ) {
+      offset = firstOp.p
+      const diff_ops = this.diffAsShareJsOps(firstOp.d, secondOp.i)
+      if (diff_ops.length === 0) {
+        return [
+          {
+            // Noop
+            meta: {
+              start_ts: firstUpdate.meta.start_ts,
+              end_ts: secondUpdate.meta.end_ts,
+              user_id: firstUpdate.meta.user_id
+            },
+            op: {
+              p: firstOp.p,
+              i: ''
+            },
+            v: secondUpdate.v
+          }
+        ]
+      } else {
+        return diff_ops.map(function(op) {
+          op.p += offset
+          return {
+            meta: {
+              start_ts: firstUpdate.meta.start_ts,
+              end_ts: secondUpdate.meta.end_ts,
+              user_id: firstUpdate.meta.user_id
+            },
+            op,
+            v: secondUpdate.v
+          }
+        })
+      }
+    } else {
+      return [firstUpdate, secondUpdate]
+    }
+  },
+
+  ADDED: 1,
+  REMOVED: -1,
+  UNCHANGED: 0,
+  diffAsShareJsOps(before, after, callback) {
+    if (callback == null) {
+      callback = function(error, ops) {}
+    }
+    const diffs = dmp.diff_main(before, after)
+    dmp.diff_cleanupSemantic(diffs)
+
+    const ops = []
+    let position = 0
+    for (const diff of Array.from(diffs)) {
+      const type = diff[0]
+      const content = diff[1]
+      if (type === this.ADDED) {
+        ops.push({
+          i: content,
+          p: position
+        })
+        position += content.length
+      } else if (type === this.REMOVED) {
+        ops.push({
+          d: content,
+          p: position
+        })
+      } else if (type === this.UNCHANGED) {
+        position += content.length
+      } else {
+        throw 'Unknown type'
+      }
+    }
+    return ops
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/app/js/UpdateTrimmer.js

@@ -0,0 +1,77 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let UpdateTrimmer
+const MongoManager = require('./MongoManager')
+const WebApiManager = require('./WebApiManager')
+const logger = require('logger-sharelatex')
+
+module.exports = UpdateTrimmer = {
+  shouldTrimUpdates(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, shouldTrim) {}
+    }
+    return MongoManager.getProjectMetaData(project_id, function(
+      error,
+      metadata
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (metadata != null ? metadata.preserveHistory : undefined) {
+        return callback(null, false)
+      } else {
+        return WebApiManager.getProjectDetails(project_id, function(
+          error,
+          details
+        ) {
+          if (error != null) {
+            return callback(error)
+          }
+          logger.log({ project_id, details }, 'got details')
+          if (
+            __guard__(
+              details != null ? details.features : undefined,
+              x => x.versioning
+            )
+          ) {
+            return MongoManager.setProjectMetaData(
+              project_id,
+              { preserveHistory: true },
+              function(error) {
+                if (error != null) {
+                  return callback(error)
+                }
+                return MongoManager.upgradeHistory(project_id, function(error) {
+                  if (error != null) {
+                    return callback(error)
+                  }
+                  return callback(null, false)
+                })
+              }
+            )
+          } else {
+            return callback(null, true)
+          }
+        })
+      }
+    })
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/app/js/UpdatesManager.js

@@ -0,0 +1,832 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let fiveMinutes, UpdatesManager
+const MongoManager = require('./MongoManager')
+const PackManager = require('./PackManager')
+const RedisManager = require('./RedisManager')
+const UpdateCompressor = require('./UpdateCompressor')
+const LockManager = require('./LockManager')
+const WebApiManager = require('./WebApiManager')
+const UpdateTrimmer = require('./UpdateTrimmer')
+const logger = require('logger-sharelatex')
+const async = require('async')
+const _ = require('underscore')
+const Settings = require('settings-sharelatex')
+const keys = Settings.redis.lock.key_schema
+
+module.exports = UpdatesManager = {
+  compressAndSaveRawUpdates(
+    project_id,
+    doc_id,
+    rawUpdates,
+    temporary,
+    callback
+  ) {
+    let i
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const { length } = rawUpdates
+    if (length === 0) {
+      return callback()
+    }
+
+    // check that ops are in the correct order
+    for (i = 0; i < rawUpdates.length; i++) {
+      const op = rawUpdates[i]
+      if (i > 0) {
+        const thisVersion = op != null ? op.v : undefined
+        const prevVersion = __guard__(rawUpdates[i - 1], x => x.v)
+        if (!(prevVersion < thisVersion)) {
+          logger.error(
+            {
+              project_id,
+              doc_id,
+              rawUpdates,
+              temporary,
+              thisVersion,
+              prevVersion
+            },
+            'op versions out of order'
+          )
+        }
+      }
+    }
+
+    // FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
+    // CORRECTION:  we do use it to log the time in case of error
+    return MongoManager.peekLastCompressedUpdate(doc_id, function(
+      error,
+      lastCompressedUpdate,
+      lastVersion
+    ) {
+      // lastCompressedUpdate is the most recent update in Mongo, and
+      // lastVersion is its sharejs version number.
+      //
+      // The peekLastCompressedUpdate method may pass the update back
+      // as 'null' (for example if the previous compressed update has
+      // been archived).  In this case it can still pass back the
+      // lastVersion from the update to allow us to check consistency.
+      let op
+      if (error != null) {
+        return callback(error)
+      }
+
+      // Ensure that raw updates start where lastVersion left off
+      if (lastVersion != null) {
+        const discardedUpdates = []
+        rawUpdates = rawUpdates.slice(0)
+        while (rawUpdates[0] != null && rawUpdates[0].v <= lastVersion) {
+          discardedUpdates.push(rawUpdates.shift())
+        }
+        if (discardedUpdates.length) {
+          logger.error(
+            { project_id, doc_id, discardedUpdates, temporary, lastVersion },
+            'discarded updates already present'
+          )
+        }
+
+        if (rawUpdates[0] != null && rawUpdates[0].v !== lastVersion + 1) {
+          const ts = __guard__(
+            lastCompressedUpdate != null
+              ? lastCompressedUpdate.meta
+              : undefined,
+            x1 => x1.end_ts
+          )
+          const last_timestamp = ts != null ? new Date(ts) : 'unknown time'
+          error = new Error(
+            `Tried to apply raw op at version ${rawUpdates[0].v} to last compressed update with version ${lastVersion} from ${last_timestamp}`
+          )
+          logger.error(
+            {
+              err: error,
+              doc_id,
+              project_id,
+              prev_end_ts: ts,
+              temporary,
+              lastCompressedUpdate
+            },
+            'inconsistent doc versions'
+          )
+          if (
+            (Settings.trackchanges != null
+              ? Settings.trackchanges.continueOnError
+              : undefined) &&
+            rawUpdates[0].v > lastVersion + 1
+          ) {
+            // we have lost some ops - continue to write into the database, we can't recover at this point
+            lastCompressedUpdate = null
+          } else {
+            return callback(error)
+          }
+        }
+      }
+
+      if (rawUpdates.length === 0) {
+        return callback()
+      }
+
+      // some old large ops in redis need to be rejected, they predate
+      // the size limit that now prevents them going through the system
+      const REJECT_LARGE_OP_SIZE = 4 * 1024 * 1024
+      for (var rawUpdate of Array.from(rawUpdates)) {
+        const opSizes = (() => {
+          const result = []
+          for (op of Array.from(
+            (rawUpdate != null ? rawUpdate.op : undefined) || []
+          )) {
+            result.push(
+              (op.i != null ? op.i.length : undefined) ||
+                (op.d != null ? op.d.length : undefined)
+            )
+          }
+          return result
+        })()
+        const size = _.max(opSizes)
+        if (size > REJECT_LARGE_OP_SIZE) {
+          error = new Error(
+            `dropped op exceeding maximum allowed size of ${REJECT_LARGE_OP_SIZE}`
+          )
+          logger.error(
+            { err: error, doc_id, project_id, size, rawUpdate },
+            'dropped op - too big'
+          )
+          rawUpdate.op = []
+        }
+      }
+
+      const compressedUpdates = UpdateCompressor.compressRawUpdates(
+        null,
+        rawUpdates
+      )
+      return PackManager.insertCompressedUpdates(
+        project_id,
+        doc_id,
+        lastCompressedUpdate,
+        compressedUpdates,
+        temporary,
+        function(error, result) {
+          if (error != null) {
+            return callback(error)
+          }
+          if (result != null) {
+            logger.log(
+              {
+                project_id,
+                doc_id,
+                orig_v:
+                  lastCompressedUpdate != null
+                    ? lastCompressedUpdate.v
+                    : undefined,
+                new_v: result.v
+              },
+              'inserted updates into pack'
+            )
+          }
+          return callback()
+        }
+      )
+    })
+  },
+
+  // Check whether the updates are temporary (per-project property)
+  _prepareProjectForUpdates(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, temporary) {}
+    }
+    return UpdateTrimmer.shouldTrimUpdates(project_id, function(
+      error,
+      temporary
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return callback(null, temporary)
+    })
+  },
+
+  // Check for project id on document history (per-document property)
+  _prepareDocForUpdates(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return MongoManager.backportProjectId(project_id, doc_id, function(error) {
+      if (error != null) {
+        return callback(error)
+      }
+      return callback(null)
+    })
+  },
+
+  // Apply updates for specific project/doc after preparing at project and doc level
+  REDIS_READ_BATCH_SIZE: 100,
+  processUncompressedUpdates(project_id, doc_id, temporary, callback) {
+    // get the updates as strings from redis (so we can delete them after they are applied)
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return RedisManager.getOldestDocUpdates(
+      doc_id,
+      UpdatesManager.REDIS_READ_BATCH_SIZE,
+      function(error, docUpdates) {
+        if (error != null) {
+          return callback(error)
+        }
+        const { length } = docUpdates
+        // parse the redis strings into ShareJs updates
+        return RedisManager.expandDocUpdates(docUpdates, function(
+          error,
+          rawUpdates
+        ) {
+          if (error != null) {
+            logger.err(
+              { project_id, doc_id, docUpdates },
+              'failed to parse docUpdates'
+            )
+            return callback(error)
+          }
+          logger.log(
+            { project_id, doc_id, rawUpdates },
+            'retrieved raw updates from redis'
+          )
+          return UpdatesManager.compressAndSaveRawUpdates(
+            project_id,
+            doc_id,
+            rawUpdates,
+            temporary,
+            function(error) {
+              if (error != null) {
+                return callback(error)
+              }
+              logger.log(
+                { project_id, doc_id },
+                'compressed and saved doc updates'
+              )
+              // delete the applied updates from redis
+              return RedisManager.deleteAppliedDocUpdates(
+                project_id,
+                doc_id,
+                docUpdates,
+                function(error) {
+                  if (error != null) {
+                    return callback(error)
+                  }
+                  if (length === UpdatesManager.REDIS_READ_BATCH_SIZE) {
+                    // There might be more updates
+                    logger.log(
+                      { project_id, doc_id },
+                      'continuing processing updates'
+                    )
+                    return setTimeout(
+                      () =>
+                        UpdatesManager.processUncompressedUpdates(
+                          project_id,
+                          doc_id,
+                          temporary,
+                          callback
+                        ),
+                      0
+                    )
+                  } else {
+                    logger.log(
+                      { project_id, doc_id },
+                      'all raw updates processed'
+                    )
+                    return callback()
+                  }
+                }
+              )
+            }
+          )
+        })
+      }
+    )
+  },
+
+  // Process updates for a doc when we flush it individually
+  processUncompressedUpdatesWithLock(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return UpdatesManager._prepareProjectForUpdates(project_id, function(
+      error,
+      temporary
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return UpdatesManager._processUncompressedUpdatesForDocWithLock(
+        project_id,
+        doc_id,
+        temporary,
+        callback
+      )
+    })
+  },
+
+  // Process updates for a doc when the whole project is flushed (internal method)
+  _processUncompressedUpdatesForDocWithLock(
+    project_id,
+    doc_id,
+    temporary,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return UpdatesManager._prepareDocForUpdates(project_id, doc_id, function(
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return LockManager.runWithLock(
+        keys.historyLock({ doc_id }),
+        releaseLock =>
+          UpdatesManager.processUncompressedUpdates(
+            project_id,
+            doc_id,
+            temporary,
+            releaseLock
+          ),
+        callback
+      )
+    })
+  },
+
+  // Process all updates for a project, only check project-level information once
+  processUncompressedUpdatesForProject(project_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return RedisManager.getDocIdsWithHistoryOps(project_id, function(
+      error,
+      doc_ids
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return UpdatesManager._prepareProjectForUpdates(project_id, function(
+        error,
+        temporary
+      ) {
+        const jobs = []
+        for (const doc_id of Array.from(doc_ids)) {
+          ;(doc_id =>
+            jobs.push(cb =>
+              UpdatesManager._processUncompressedUpdatesForDocWithLock(
+                project_id,
+                doc_id,
+                temporary,
+                cb
+              )
+            ))(doc_id)
+        }
+        return async.parallelLimit(jobs, 5, callback)
+      })
+    })
+  },
+
+  // flush all outstanding changes
+  flushAll(limit, callback) {
+    if (callback == null) {
+      callback = function(error, result) {}
+    }
+    return RedisManager.getProjectIdsWithHistoryOps(function(
+      error,
+      project_ids
+    ) {
+      let project_id
+      if (error != null) {
+        return callback(error)
+      }
+      logger.log(
+        {
+          count: project_ids != null ? project_ids.length : undefined,
+          project_ids
+        },
+        'found projects'
+      )
+      const jobs = []
+      project_ids = _.shuffle(project_ids) // randomise to avoid hitting same projects each time
+      const selectedProjects =
+        limit < 0 ? project_ids : project_ids.slice(0, limit)
+      for (project_id of Array.from(selectedProjects)) {
+        ;(project_id =>
+          jobs.push(cb =>
+            UpdatesManager.processUncompressedUpdatesForProject(
+              project_id,
+              err => cb(null, { failed: err != null, project_id })
+            )
+          ))(project_id)
+      }
+      return async.series(jobs, function(error, result) {
+        let x
+        if (error != null) {
+          return callback(error)
+        }
+        const failedProjects = (() => {
+          const result1 = []
+          for (x of Array.from(result)) {
+            if (x.failed) {
+              result1.push(x.project_id)
+            }
+          }
+          return result1
+        })()
+        const succeededProjects = (() => {
+          const result2 = []
+          for (x of Array.from(result)) {
+            if (!x.failed) {
+              result2.push(x.project_id)
+            }
+          }
+          return result2
+        })()
+        return callback(null, {
+          failed: failedProjects,
+          succeeded: succeededProjects,
+          all: project_ids
+        })
+      })
+    })
+  },
+
+  getDanglingUpdates(callback) {
+    if (callback == null) {
+      callback = function(error, doc_ids) {}
+    }
+    return RedisManager.getAllDocIdsWithHistoryOps(function(
+      error,
+      all_doc_ids
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return RedisManager.getProjectIdsWithHistoryOps(function(
+        error,
+        all_project_ids
+      ) {
+        if (error != null) {
+          return callback(error)
+        }
+        // function to get doc_ids for each project
+        const task = cb =>
+          async.concatSeries(
+            all_project_ids,
+            RedisManager.getDocIdsWithHistoryOps,
+            cb
+          )
+        // find the dangling doc ids
+        return task(function(error, project_doc_ids) {
+          const dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids)
+          logger.log(
+            { all_doc_ids, all_project_ids, project_doc_ids, dangling_doc_ids },
+            'checking for dangling doc ids'
+          )
+          return callback(null, dangling_doc_ids)
+        })
+      })
+    })
+  },
+
+  getDocUpdates(project_id, doc_id, options, callback) {
+    if (options == null) {
+      options = {}
+    }
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return UpdatesManager.processUncompressedUpdatesWithLock(
+      project_id,
+      doc_id,
+      function(error) {
+        if (error != null) {
+          return callback(error)
+        }
+        // console.log "options", options
+        return PackManager.getOpsByVersionRange(
+          project_id,
+          doc_id,
+          options.from,
+          options.to,
+          function(error, updates) {
+            if (error != null) {
+              return callback(error)
+            }
+            return callback(null, updates)
+          }
+        )
+      }
+    )
+  },
+
+  getDocUpdatesWithUserInfo(project_id, doc_id, options, callback) {
+    if (options == null) {
+      options = {}
+    }
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return UpdatesManager.getDocUpdates(project_id, doc_id, options, function(
+      error,
+      updates
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return UpdatesManager.fillUserInfo(updates, function(error, updates) {
+        if (error != null) {
+          return callback(error)
+        }
+        return callback(null, updates)
+      })
+    })
+  },
+
+  getSummarizedProjectUpdates(project_id, options, callback) {
+    if (options == null) {
+      options = {}
+    }
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    if (!options.min_count) {
+      options.min_count = 25
+    }
+    let summarizedUpdates = []
+    const { before } = options
+    let nextBeforeTimestamp = null
+    return UpdatesManager.processUncompressedUpdatesForProject(
+      project_id,
+      function(error) {
+        if (error != null) {
+          return callback(error)
+        }
+        return PackManager.makeProjectIterator(project_id, before, function(
+          err,
+          iterator
+        ) {
+          if (err != null) {
+            return callback(err)
+          }
+          // repeatedly get updates and pass them through the summariser to get an final output with user info
+          return async.whilst(
+            () =>
+              // console.log "checking iterator.done", iterator.done()
+              summarizedUpdates.length < options.min_count && !iterator.done(),
+
+            cb =>
+              iterator.next(function(err, partialUpdates) {
+                if (err != null) {
+                  return callback(err)
+                }
+                // logger.log {partialUpdates}, 'got partialUpdates'
+                if (partialUpdates.length === 0) {
+                  return cb()
+                } // # FIXME should try to avoid this happening
+                nextBeforeTimestamp =
+                  partialUpdates[partialUpdates.length - 1].meta.end_ts
+                // add the updates to the summary list
+                summarizedUpdates = UpdatesManager._summarizeUpdates(
+                  partialUpdates,
+                  summarizedUpdates
+                )
+                return cb()
+              }),
+
+            () =>
+              // finally done all updates
+              // console.log 'summarized Updates', summarizedUpdates
+              UpdatesManager.fillSummarizedUserInfo(summarizedUpdates, function(
+                err,
+                results
+              ) {
+                if (err != null) {
+                  return callback(err)
+                }
+                return callback(
+                  null,
+                  results,
+                  !iterator.done() ? nextBeforeTimestamp : undefined
+                )
+              })
+          )
+        })
+      }
+    )
+  },
+
+  fetchUserInfo(users, callback) {
+    if (callback == null) {
+      callback = function(error, fetchedUserInfo) {}
+    }
+    const jobs = []
+    const fetchedUserInfo = {}
+    for (const user_id in users) {
+      ;(user_id =>
+        jobs.push(callback =>
+          WebApiManager.getUserInfo(user_id, function(error, userInfo) {
+            if (error != null) {
+              return callback(error)
+            }
+            fetchedUserInfo[user_id] = userInfo
+            return callback()
+          })
+        ))(user_id)
+    }
+
+    return async.series(jobs, function(err) {
+      if (err != null) {
+        return callback(err)
+      }
+      return callback(null, fetchedUserInfo)
+    })
+  },
+
+  fillUserInfo(updates, callback) {
+    let update, user_id
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    const users = {}
+    for (update of Array.from(updates)) {
+      ;({ user_id } = update.meta)
+      if (UpdatesManager._validUserId(user_id)) {
+        users[user_id] = true
+      }
+    }
+
+    return UpdatesManager.fetchUserInfo(users, function(
+      error,
+      fetchedUserInfo
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      for (update of Array.from(updates)) {
+        ;({ user_id } = update.meta)
+        delete update.meta.user_id
+        if (UpdatesManager._validUserId(user_id)) {
+          update.meta.user = fetchedUserInfo[user_id]
+        }
+      }
+      return callback(null, updates)
+    })
+  },
+
+  fillSummarizedUserInfo(updates, callback) {
+    let update, user_id, user_ids
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    const users = {}
+    for (update of Array.from(updates)) {
+      user_ids = update.meta.user_ids || []
+      for (user_id of Array.from(user_ids)) {
+        if (UpdatesManager._validUserId(user_id)) {
+          users[user_id] = true
+        }
+      }
+    }
+
+    return UpdatesManager.fetchUserInfo(users, function(
+      error,
+      fetchedUserInfo
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      for (update of Array.from(updates)) {
+        user_ids = update.meta.user_ids || []
+        update.meta.users = []
+        delete update.meta.user_ids
+        for (user_id of Array.from(user_ids)) {
+          if (UpdatesManager._validUserId(user_id)) {
+            update.meta.users.push(fetchedUserInfo[user_id])
+          } else {
+            update.meta.users.push(null)
+          }
+        }
+      }
+      return callback(null, updates)
+    })
+  },
+
+  _validUserId(user_id) {
+    if (user_id == null) {
+      return false
+    } else {
+      return !!user_id.match(/^[a-f0-9]{24}$/)
+    }
+  },
+
+  TIME_BETWEEN_DISTINCT_UPDATES: (fiveMinutes = 5 * 60 * 1000),
+  SPLIT_ON_DELETE_SIZE: 16, // characters
+  _summarizeUpdates(updates, existingSummarizedUpdates) {
+    if (existingSummarizedUpdates == null) {
+      existingSummarizedUpdates = []
+    }
+    const summarizedUpdates = existingSummarizedUpdates.slice()
+    let previousUpdateWasBigDelete = false
+    for (const update of Array.from(updates)) {
+      var doc_id
+      const earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
+      let shouldConcat = false
+
+      // If a user inserts some text, then deletes a big chunk including that text,
+      // the update we show might concat the insert and delete, and there will be no sign
+      // of that insert having happened, or be able to restore to it (restoring after a big delete is common).
+      // So, we split the summary on 'big' deletes. However, we've stepping backwards in time with
+      // most recent changes considered first, so if this update is a big delete, we want to start
+      // a new summarized update next timge, hence we monitor the previous update.
+      if (previousUpdateWasBigDelete) {
+        shouldConcat = false
+      } else if (
+        earliestUpdate &&
+        earliestUpdate.meta.end_ts - update.meta.start_ts <
+          this.TIME_BETWEEN_DISTINCT_UPDATES
+      ) {
+        // We're going backwards in time through the updates, so only combine if this update starts less than 5 minutes before
+        // the end of current summarized block, so no block spans more than 5 minutes.
+        shouldConcat = true
+      }
+
+      let isBigDelete = false
+      for (const op of Array.from(update.op || [])) {
+        if (op.d != null && op.d.length > this.SPLIT_ON_DELETE_SIZE) {
+          isBigDelete = true
+        }
+      }
+
+      previousUpdateWasBigDelete = isBigDelete
+
+      if (shouldConcat) {
+        // check if the user in this update is already present in the earliest update,
+        // if not, add them to the users list of the earliest update
+        earliestUpdate.meta.user_ids = _.union(earliestUpdate.meta.user_ids, [
+          update.meta.user_id
+        ])
+
+        doc_id = update.doc_id.toString()
+        const doc = earliestUpdate.docs[doc_id]
+        if (doc != null) {
+          doc.fromV = Math.min(doc.fromV, update.v)
+          doc.toV = Math.max(doc.toV, update.v)
+        } else {
+          earliestUpdate.docs[doc_id] = {
+            fromV: update.v,
+            toV: update.v
+          }
+        }
+
+        earliestUpdate.meta.start_ts = Math.min(
+          earliestUpdate.meta.start_ts,
+          update.meta.start_ts
+        )
+        earliestUpdate.meta.end_ts = Math.max(
+          earliestUpdate.meta.end_ts,
+          update.meta.end_ts
+        )
+      } else {
+        const newUpdate = {
+          meta: {
+            user_ids: [],
+            start_ts: update.meta.start_ts,
+            end_ts: update.meta.end_ts
+          },
+          docs: {}
+        }
+
+        newUpdate.docs[update.doc_id.toString()] = {
+          fromV: update.v,
+          toV: update.v
+        }
+        newUpdate.meta.user_ids.push(update.meta.user_id)
+        summarizedUpdates.push(newUpdate)
+      }
+    }
+
+    return summarizedUpdates
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/app/js/WebApiManager.js

@@ -0,0 +1,116 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let WebApiManager
+const request = require('requestretry') // allow retry on error https://github.com/FGRibreau/node-request-retry
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+
+// Don't let HTTP calls hang for a long time
+const MAX_HTTP_REQUEST_LENGTH = 15000 // 15 seconds
+
+// DEPRECATED! This method of getting user details via track-changes is deprecated
+// in the way we lay out our services.
+// Instead, web should be responsible for collecting the raw data (user_ids) and
+// filling it out with calls to other services. All API calls should create a
+// tree-like structure as much as possible, with web as the root.
+module.exports = WebApiManager = {
+  sendRequest(url, callback) {
+    if (callback == null) {
+      callback = function(error, body) {}
+    }
+    return request.get(
+      {
+        url: `${Settings.apis.web.url}${url}`,
+        timeout: MAX_HTTP_REQUEST_LENGTH,
+        maxAttempts: 2, // for node-request-retry
+        auth: {
+          user: Settings.apis.web.user,
+          pass: Settings.apis.web.pass,
+          sendImmediately: true
+        }
+      },
+      function(error, res, body) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (res.statusCode === 404) {
+          logger.log({ url }, 'got 404 from web api')
+          return callback(null, null)
+        }
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          return callback(null, body)
+        } else {
+          error = new Error(
+            `web returned a non-success status code: ${res.statusCode} (attempts: ${res.attempts})`
+          )
+          return callback(error)
+        }
+      }
+    )
+  },
+
+  getUserInfo(user_id, callback) {
+    if (callback == null) {
+      callback = function(error, userInfo) {}
+    }
+    const url = `/user/${user_id}/personal_info`
+    logger.log({ user_id }, 'getting user info from web')
+    return WebApiManager.sendRequest(url, function(error, body) {
+      let user
+      if (error != null) {
+        logger.error({ err: error, user_id, url }, 'error accessing web')
+        return callback(error)
+      }
+
+      if (body === null) {
+        logger.error({ user_id, url }, 'no user found')
+        return callback(null, null)
+      }
+      try {
+        user = JSON.parse(body)
+      } catch (error1) {
+        error = error1
+        return callback(error)
+      }
+      return callback(null, {
+        id: user.id,
+        email: user.email,
+        first_name: user.first_name,
+        last_name: user.last_name
+      })
+    })
+  },
+
+  getProjectDetails(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, details) {}
+    }
+    const url = `/project/${project_id}/details`
+    logger.log({ project_id }, 'getting project details from web')
+    return WebApiManager.sendRequest(url, function(error, body) {
+      let project
+      if (error != null) {
+        logger.error({ err: error, project_id, url }, 'error accessing web')
+        return callback(error)
+      }
+
+      try {
+        project = JSON.parse(body)
+      } catch (error1) {
+        error = error1
+        return callback(error)
+      }
+      return callback(null, project)
+    })
+  }
+}

services/track-changes/app/js/mongojs.js

@@ -0,0 +1,15 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+const Settings = require('settings-sharelatex')
+const mongojs = require('mongojs')
+const bson = require('bson')
+const db = mongojs(Settings.mongo.url, [
+  'docHistory',
+  'projectHistoryMetaData',
+  'docHistoryIndex'
+])
+module.exports = {
+  db,
+  ObjectId: mongojs.ObjectId,
+  BSON: new bson.BSONPure()
+}

services/track-changes/buildscript.txt

@@ -1,6 +1,6 @@
 track-changes
 --public-repo=True
---language=coffeescript
+--language=es
 --env-add=AWS_BUCKET=bucket
 --node-version=10.19.0
 --acceptance-creds=None

services/track-changes/config/settings.defaults.coffee

@@ -1,51 +0,0 @@
-Path = require('path')
-TMP_DIR = process.env["TMP_PATH"] or Path.resolve(Path.join(__dirname, "../../", "tmp"))
-
-module.exports =
-	mongo:
-		url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
-
-	internal:
-		trackchanges:
-			port: 3015
-			host: process.env["LISTEN_ADDRESS"] or "localhost"
-	apis:
-		documentupdater:
-			url: "http://#{process.env["DOCUMENT_UPDATER_HOST"] or process.env["DOCUPDATER_HOST"] or "localhost"}:3003"
-		docstore:
-			url: "http://#{process.env["DOCSTORE_HOST"] or "localhost"}:3016"
-		web:
-			url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}"
-			user: process.env['WEB_API_USER'] or "sharelatex"
-			pass: process.env['WEB_API_PASSWORD'] or "password"
-	redis:
-		lock:
-			host: process.env["REDIS_HOST"] or "localhost"
-			port: process.env['REDIS_PORT'] or 6379
-			password: process.env["REDIS_PASSWORD"] or ""
-			key_schema:
-				historyLock: ({doc_id}) -> "HistoryLock:{#{doc_id}}"
-				historyIndexLock: ({project_id}) -> "HistoryIndexLock:{#{project_id}}"
-		history:
-			host: process.env["REDIS_HOST"] or "localhost"
-			port: process.env['REDIS_PORT'] or 6379
-			password: process.env["REDIS_PASSWORD"] or ""
-			key_schema:
-				uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:{#{doc_id}}"
-				docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}"
-
-	trackchanges:
-		s3:
-			key: process.env['AWS_ACCESS_KEY_ID']
-			secret: process.env['AWS_SECRET_ACCESS_KEY']
-			endpoint: process.env['AWS_S3_ENDPOINT']
-			pathStyle: process.env['AWS_S3_PATH_STYLE'] == 'true'
-		stores:
-			doc_history: process.env['AWS_BUCKET']
-		continueOnError: process.env['TRACK_CHANGES_CONTINUE_ON_ERROR'] or false
-			
-	path:
-		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")
-
-	sentry:
-		dsn: process.env.SENTRY_DSN

services/track-changes/config/settings.defaults.js

@@ -0,0 +1,86 @@
+const Path = require('path')
+const TMP_DIR =
+  process.env.TMP_PATH || Path.resolve(Path.join(__dirname, '../../', 'tmp'))
+
+module.exports = {
+  mongo: {
+    url:
+      process.env.MONGO_CONNECTION_STRING ||
+      `mongodb://${process.env.MONGO_HOST || 'localhost'}/sharelatex`
+  },
+
+  internal: {
+    trackchanges: {
+      port: 3015,
+      host: process.env.LISTEN_ADDRESS || 'localhost'
+    }
+  },
+  apis: {
+    documentupdater: {
+      url: `http://${process.env.DOCUMENT_UPDATER_HOST ||
+        process.env.DOCUPDATER_HOST ||
+        'localhost'}:3003`
+    },
+    docstore: {
+      url: `http://${process.env.DOCSTORE_HOST || 'localhost'}:3016`
+    },
+    web: {
+      url: `http://${process.env.WEB_API_HOST ||
+        process.env.WEB_HOST ||
+        'localhost'}:${process.env.WEB_API_PORT ||
+        process.env.WEB_PORT ||
+        3000}`,
+      user: process.env.WEB_API_USER || 'sharelatex',
+      pass: process.env.WEB_API_PASSWORD || 'password'
+    }
+  },
+  redis: {
+    lock: {
+      host: process.env.REDIS_HOST || 'localhost',
+      port: process.env.REDIS_PORT || 6379,
+      password: process.env.REDIS_PASSWORD || '',
+      key_schema: {
+        historyLock({ doc_id }) {
+          return `HistoryLock:{${doc_id}}`
+        },
+        historyIndexLock({ project_id }) {
+          return `HistoryIndexLock:{${project_id}}`
+        }
+      }
+    },
+    history: {
+      host: process.env.REDIS_HOST || 'localhost',
+      port: process.env.REDIS_PORT || 6379,
+      password: process.env.REDIS_PASSWORD || '',
+      key_schema: {
+        uncompressedHistoryOps({ doc_id }) {
+          return `UncompressedHistoryOps:{${doc_id}}`
+        },
+        docsWithHistoryOps({ project_id }) {
+          return `DocsWithHistoryOps:{${project_id}}`
+        }
+      }
+    }
+  },
+
+  trackchanges: {
+    s3: {
+      key: process.env.AWS_ACCESS_KEY_ID,
+      secret: process.env.AWS_SECRET_ACCESS_KEY,
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      pathStyle: process.env.AWS_S3_PATH_STYLE === 'true'
+    },
+    stores: {
+      doc_history: process.env.AWS_BUCKET
+    },
+    continueOnError: process.env.TRACK_CHANGES_CONTINUE_ON_ERROR || false
+  },
+
+  path: {
+    dumpFolder: Path.join(TMP_DIR, 'dumpFolder')
+  },
+
+  sentry: {
+    dsn: process.env.SENTRY_DSN
+  }
+}

services/track-changes/nodemon.json

@@ -10,10 +10,9 @@
   },
 
   "watch": [
-    "app/coffee/",
-    "app.coffee",
+    "app/js/",
+    "app.js",
     "config/"
   ],
-  "ext": "coffee"
-
+  "ext": "js"
 }

services/track-changes/package.json

@@ -7,17 +7,15 @@
     "url": "https://github.com/sharelatex/track-changes-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
-    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
-    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
-    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
-    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "start": "node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
+    "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "nodemon": "nodemon --config nodemon.json",
-    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+    "lint": "node_modules/.bin/eslint .",
+    "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
+    "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write"
   },
   "dependencies": {
     "JSONStream": "^1.0.4",
@@ -42,11 +40,28 @@
     "underscore": "~1.7.0"
   },
   "devDependencies": {
+    "babel-eslint": "^10.0.3",
     "bunyan": "~2.0.2",
     "chai": "~4.1.1",
+    "eslint": "^6.6.0",
+    "eslint-config-prettier": "^6.10.0",
+    "eslint-config-standard": "^14.1.0",
+    "eslint-config-standard-jsx": "^8.1.0",
+    "eslint-config-standard-react": "^9.2.0",
+    "eslint-plugin-chai-expect": "^2.1.0",
+    "eslint-plugin-chai-friendly": "^0.5.0",
+    "eslint-plugin-import": "^2.20.1",
+    "eslint-plugin-jsx-a11y": "^6.2.3",
+    "eslint-plugin-mocha": "^6.2.2",
+    "eslint-plugin-node": "^11.0.0",
+    "eslint-plugin-prettier": "^3.1.2",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-react": "^7.18.3",
+    "eslint-plugin-standard": "^4.0.1",
     "memorystream": "0.3.1",
     "mocha": "^4.0.1",
-    "coffee-script": "^1.7.1",
+    "prettier": "^1.19.1",
+    "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "~0.3.0",
     "sinon": "~3.2.1",
     "timekeeper": "0.0.4"

services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee

@@ -1,311 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-request = require "request"
-rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Appending doc ops to the history", ->
-	before (done)->
-		TrackChangesApp.ensureRunning done
-
-	describe "when the history does not exist yet", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}, {
-				op: [{ i: "o", p: 4 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 4
-			}, {
-				op: [{ i: "o", p: 5 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 5
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should insert the compressed op into mongo", ->
-			expect(@updates[0].pack[0].op).to.deep.equal [{
-				p: 3, i: "foo"
-			}]
-
-		it "should insert the correct version number into mongo", ->
-			expect(@updates[0].v).to.equal 5
-
-		it "should store the doc id", ->
-			expect(@updates[0].doc_id.toString()).to.equal @doc_id
-
-		it "should store the project id", ->
-			expect(@updates[0].project_id.toString()).to.equal @project_id
-
-		it "should clear the doc from the DocsWithHistoryOps set", (done) ->
-			rclient.sismember "DocsWithHistoryOps:#{@project_id}", @doc_id, (error, member) ->
-				member.should.equal 0
-				done()
-			return null
-
-	describe "when the history has already been started", ->
-		beforeEach (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}, {
-				op: [{ i: "o", p: 4 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 4
-			}, {
-				op: [{ i: "o", p: 5 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 5
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, updates) =>
-					throw error if error?
-					done()
-			return null
-
-		describe "when the updates are recent and from the same user", ->
-			beforeEach (done) ->
-				TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-					op: [{ i: "b", p: 6 }]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: 6
-				}, {
-					op: [{ i: "a", p: 7 }]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: 7
-				}, {
-					op: [{ i: "r", p: 8 }]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: 8
-				}], (error) =>
-					throw error if error?
-					TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-						throw error if error?
-						done()
-				return null
-
-			it "should combine all the updates into one pack", ->
-				expect(@updates[0].pack[1].op).to.deep.equal [{
-					p: 6, i: "bar"
-				}]
-
-			it "should insert the correct version number into mongo", ->
-				expect(@updates[0].v_end).to.equal 8
-
-
-		describe "when the updates are far apart", ->
-			beforeEach (done) ->
-				oneDay = 24 * 60 * 60 * 1000
-				TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-					op: [{ i: "b", p: 6 }]
-					meta: { ts: Date.now() + oneDay, user_id: @user_id }
-					v: 6
-				}, {
-					op: [{ i: "a", p: 7 }]
-					meta: { ts: Date.now() + oneDay, user_id: @user_id }
-					v: 7
-				}, {
-					op: [{ i: "r", p: 8 }]
-					meta: { ts: Date.now() + oneDay, user_id: @user_id }
-					v: 8
-				}], (error) =>
-					throw error if error?
-					TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-						throw error if error?
-						done()
-				return null
-
-			it "should combine the updates into one pack", ->
-				expect(@updates[0].pack[0].op).to.deep.equal [{
-					p: 3, i: "foo"
-				}]
-				expect(@updates[0].pack[1].op).to.deep.equal [{
-					p: 6, i: "bar"
-				}]
-
-	describe "when the updates need processing in batches", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			updates = []
-			@expectedOp = [{ p:0, i: "" }]
-			for i in [0..250]
-				updates.push {
-					op: [{i: "a", p: 0}]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: i
-				}
-				@expectedOp[0].i = "a" + @expectedOp[0].i
-
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, updates, (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should concat the compressed op into mongo", ->
-			expect(@updates[0].pack.length).to.deep.equal 3  # batch size is 100
-
-		it "should insert the correct version number into mongo", ->
-			expect(@updates[0].v_end).to.equal 250
-
-
-	describe "when there are multiple ops in each update", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			oneDay = 24 * 60 * 60 * 1000
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }, { i: "o", p: 4 }, { i: "o", p: 5 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}, {
-				op: [{ i: "b", p: 6 }, { i: "a", p: 7 }, { i: "r", p: 8 }]
-				meta: { ts: Date.now() + oneDay, user_id: @user_id }
-				v: 4
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should insert the compressed ops into mongo", ->
-			expect(@updates[0].pack[0].op).to.deep.equal [{
-				p: 3, i: "foo"
-			}]
-			expect(@updates[0].pack[1].op).to.deep.equal [{
-				p: 6, i: "bar"
-			}]
-
-		it "should insert the correct version numbers into mongo", ->
-			expect(@updates[0].pack[0].v).to.equal 3
-			expect(@updates[0].pack[1].v).to.equal 4
-
-	describe "when there is a no-op update", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			oneDay = 24 * 60 * 60 * 1000
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: []
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}, {
-				op: [{ i: "foo", p: 3 }]
-				meta: { ts: Date.now() + oneDay, user_id: @user_id }
-				v: 4
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should insert the compressed no-op into mongo", ->
-			expect(@updates[0].pack[0].op).to.deep.equal []
-
-
-		it "should insert the compressed next update into mongo", ->
-			expect(@updates[0].pack[1].op).to.deep.equal [{
-				p: 3, i: "foo"
-			}]
-
-		it "should insert the correct version numbers into mongo", ->
-			expect(@updates[0].pack[0].v).to.equal 3
-			expect(@updates[0].pack[1].v).to.equal 4
-
-	describe "when there is a comment update", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ c: "foo", p: 3 }, {d: "bar", p: 6}]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should ignore the comment op", ->
-			expect(@updates[0].pack[0].op).to.deep.equal [{d: "bar", p: 6}]
-
-		it "should insert the correct version numbers into mongo", ->
-			expect(@updates[0].pack[0].v).to.equal 3
-
-	describe "when the project has versioning enabled", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: true
-
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should not add a expiresAt entry in the update in mongo", ->
-			expect(@updates[0].expiresAt).to.be.undefined
-
-	describe "when the project does not have versioning enabled", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: false
-
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
-					throw error if error?
-					done()
-			return null
-
-		it "should add a expiresAt entry in the update in mongo", ->
-			expect(@updates[0].expiresAt).to.exist

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -1,134 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-db = mongojs.db
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-request = require "request"
-rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockDocStoreApi = require "./helpers/MockDocStoreApi"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Archiving updates", ->
-	before (done) ->
-		if Settings?.trackchanges?.s3?.key.length < 1
-			message = new Error("s3 keys not setup, this test setup will fail")
-			return done(message)
-
-		TrackChangesClient.waitForS3 done
-
-	before (done) ->
-		@now = Date.now()
-		@to = @now
-		@user_id = ObjectId().toString()
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-
-		@minutes = 60 * 1000
-		@hours = 60 * @minutes
-
-		MockWebApi.projects[@project_id] =
-			features:
-				versioning: true
-		sinon.spy MockWebApi, "getProjectDetails"
-
-		MockWebApi.users[@user_id] = @user =
-			email: "user@sharelatex.com"
-			first_name: "Leo"
-			last_name: "Lion"
-			id: @user_id
-		sinon.spy MockWebApi, "getUserInfo"
-
-		MockDocStoreApi.docs[@doc_id] = @doc = 
-			_id: @doc_id
-			project_id: @project_id
-		sinon.spy MockDocStoreApi, "getAllDoc"
-
-		@updates = []
-		for i in [0..512+10]
-			@updates.push {
-				op: [{ i: "a", p: 0 }]
-				meta: { ts: @now + (i-2048) * @hours, user_id: @user_id }
-				v: 2 * i + 1
-			}
-			@updates.push {
-				op: [{ i: "b", p: 0 }]
-				meta: { ts: @now + (i-2048) * @hours + 10*@minutes, user_id: @user_id }
-				v: 2 * i + 2
-			}
-		TrackChangesApp.ensureRunning =>
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-				throw error if error?
-				TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
-					throw error if error?
-					done()
-		return null
-
-	after (done) ->
-		MockWebApi.getUserInfo.restore()
-		db.docHistory.remove {project_id: ObjectId(@project_id)}, () =>
-			db.docHistoryIndex.remove {project_id: ObjectId(@project_id)}, () =>
-				TrackChangesClient.removeS3Doc @project_id, @doc_id, done
-
-	describe "archiving a doc's updates", ->
-		before (done) ->
-			TrackChangesClient.pushDocHistory @project_id, @doc_id, (error) ->
-				throw error if error?
-				done()
-			return null
-
-		it "should have one cached pack", (done) ->
-			db.docHistory.count { doc_id: ObjectId(@doc_id), expiresAt:{$exists:true}}, (error, count) ->
-				throw error if error?
-				count.should.equal 1
-				done()
-
-		it "should have one remaining pack after cache is expired", (done) ->
-			db.docHistory.remove {
-				doc_id: ObjectId(@doc_id),
-				expiresAt:{$exists:true}
-			}, (err, result) =>
-				throw error if error?
-				db.docHistory.count { doc_id: ObjectId(@doc_id)}, (error, count) ->
-					throw error if error?
-					count.should.equal 1
-					done()
-
-		it "should have a docHistoryIndex entry marked as inS3", (done) ->
-			db.docHistoryIndex.findOne { _id: ObjectId(@doc_id) }, (error, index) ->
-				throw error if error?
-				index.packs[0].inS3.should.equal true
-				done()
-
-		it "should have a docHistoryIndex entry with the last version", (done) ->
-			db.docHistoryIndex.findOne { _id: ObjectId(@doc_id) }, (error, index) ->
-				throw error if error?
-				index.packs[0].v_end.should.equal 1024
-				done()
-
-		it "should store 1024 doc changes in S3 in one pack", (done) ->
-			db.docHistoryIndex.findOne { _id: ObjectId(@doc_id) }, (error, index) =>
-				throw error if error?
-				pack_id = index.packs[0]._id
-				TrackChangesClient.getS3Doc @project_id, @doc_id, pack_id, (error, doc) =>
-					doc.n.should.equal 1024
-					doc.pack.length.should.equal 1024
-					done()
-
-	describe "unarchiving a doc's updates", ->
-		before (done) ->
-			TrackChangesClient.pullDocHistory @project_id, @doc_id, (error) ->
-				throw error if error?
-				done()
-			return null
-
-		it "should restore both packs", (done) ->
-			db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
-				throw error if error?
-				count.should.equal 2
-				done()

services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee

@@ -1,151 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-request = require "request"
-rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Flushing updates", ->
-	before (done)->
-		TrackChangesApp.ensureRunning done
-
-	describe "flushing a doc's updates", ->
-		before (done) ->
-			@project_id = ObjectId().toString()
-			@doc_id = ObjectId().toString()
-			@user_id = ObjectId().toString()
-			MockWebApi.projects[@project_id] = features: versioning: true
-
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-				op: [{ i: "f", p: 3 }]
-				meta: { ts: Date.now(), user_id: @user_id }
-				v: 3
-			}], (error) =>
-				throw error if error?
-				TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
-					throw error if error?
-					done()
-			return null
-
-		it "should flush the op into mongo", (done) ->
-			TrackChangesClient.getCompressedUpdates @doc_id, (error, updates) ->
-				expect(updates[0].pack[0].op).to.deep.equal [{
-					p: 3, i: "f"
-				}]
-				done()
-			return null
-
-	describe "flushing a project's updates", ->
-		describe "with versioning enabled", ->
-			before (done) ->
-				@project_id = ObjectId().toString()
-				@doc_id = ObjectId().toString()
-				@user_id = ObjectId().toString()
-
-				@weeks = 7 * 24 * 60 * 60 * 1000
-
-				MockWebApi.projects[@project_id] =
-					features:
-						versioning: true
-
-				TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-					op: [{ i: "g", p: 2 }]
-					meta: { ts: Date.now() - 2 * @weeks, user_id: @user_id }
-					v: 2
-				}, {
-					op: [{ i: "f", p: 3 }]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: 3
-				}], (error) =>
-					throw error if error?
-					TrackChangesClient.flushProject @project_id, (error) ->
-						throw error if error?
-						done()
-				return null
-
-			it "should not mark the updates for deletion", (done) ->
-				TrackChangesClient.getCompressedUpdates @doc_id, (error, updates) ->
-					expect(updates[0].expiresAt).to.not.exist
-					done()
-				return null
-
-			it "should preserve history forever", (done) ->
-				TrackChangesClient.getProjectMetaData @project_id, (error, project) ->
-					expect(project.preserveHistory).to.equal true
-					done()
-				return null
-
-		describe "without versioning enabled", ->
-			before (done) ->
-				@project_id = ObjectId().toString()
-				@doc_id = ObjectId().toString()
-				@user_id = ObjectId().toString()
-
-				@weeks = 7 * 24 * 60 * 60 * 1000
-
-				MockWebApi.projects[@project_id] =
-					features:
-						versioning: false
-
-				TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-					op: [{ i: "g", p: 2 }]
-					meta: { ts: Date.now() - 2 * @weeks, user_id: @user_id }
-					v: 2
-				}, {
-					op: [{ i: "f", p: 3 }]
-					meta: { ts: Date.now(), user_id: @user_id }
-					v: 3
-				}], (error) =>
-					throw error if error?
-					TrackChangesClient.flushProject @project_id, (error) ->
-						throw error if error?
-						done()
-				return null
-
-			it "should mark the updates for deletion", (done) ->
-				TrackChangesClient.getCompressedUpdates @doc_id, (error, updates) ->
-					expect(updates[0].expiresAt).to.exist
-					done()
-				return null
-
-		describe "without versioning enabled but with preserveHistory set to true", ->
-			before (done) ->
-				@project_id = ObjectId().toString()
-				@doc_id = ObjectId().toString()
-				@user_id = ObjectId().toString()
-
-				@weeks = 7 * 24 * 60 * 60 * 1000
-
-				MockWebApi.projects[@project_id] =
-					features:
-						versioning: false
-
-				TrackChangesClient.setPreserveHistoryForProject @project_id, (error) =>
-					throw error if error?
-					TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
-						op: [{ i: "g", p: 2 }]
-						meta: { ts: Date.now() - 2 * @weeks, user_id: @user_id }
-						v: 2
-					}, {
-						op: [{ i: "f", p: 3 }]
-						meta: { ts: Date.now(), user_id: @user_id }
-						v: 3
-					}], (error) =>
-						throw error if error?
-						TrackChangesClient.flushProject @project_id, (error) ->
-							throw error if error?
-							done()
-				return null
-
-			it "should not mark the updates for deletion", (done) ->
-				TrackChangesClient.getCompressedUpdates @doc_id, (error, updates) ->
-					expect(updates[0].expiresAt).to.not.exist
-					done()
-				return null

services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee

@@ -1,84 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-db = mongojs.db
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Getting a diff", ->
-
-	beforeEach (done) ->
-		sinon.spy MockDocUpdaterApi, "getDoc"
-
-		@now = Date.now()
-		@from = @now - 100000000
-		@to = @now
-		@user_id = ObjectId().toString()
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-		MockWebApi.projects[@project_id] = features: versioning: true
-
-		MockWebApi.users[@user_id] = @user =
-			email: "user@sharelatex.com"
-			first_name: "Leo"
-			last_name: "Lion"
-			id: @user_id
-		sinon.spy MockWebApi, "getUserInfo"
-
-		twoMinutes = 2 * 60 * 1000
-
-		@updates = [{
-			op: [{ i: "one ", p: 0 }]
-			meta: { ts: @from - twoMinutes, user_id: @user_id }
-			v: 3
-		}, {
-			op: [{ i: "two ", p: 4 }]
-			meta: { ts: @from + twoMinutes, user_id: @user_id }
-			v: @fromVersion = 4
-		}, {
-			op: [{ i: "three ", p: 8 }]
-			meta: { ts: @to - twoMinutes, user_id: @user_id }
-			v: @toVersion = 5
-		}, {
-			op: [{ i: "four", p: 14 }]
-			meta: { ts: @to + twoMinutes, user_id: @user_id }
-			v: 6
-		}]
-		@lines = ["one two three four"]
-		@expected_diff = [
-			{ u: "one " }
-			{ i: "two three ", meta: { start_ts: @from + twoMinutes, end_ts: @to - twoMinutes, user: @user } }
-		]
-
-		MockDocUpdaterApi.docs[@doc_id] =
-			lines: @lines
-			version: 7
-		TrackChangesApp.ensureRunning =>
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-				throw error if error?
-				TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) =>
-					throw error if error?
-					@diff = diff.diff
-					done()
-		return null
-
-	afterEach () ->
-		MockDocUpdaterApi.getDoc.restore()
-		MockWebApi.getUserInfo.restore()
-		return null
-
-	it "should return the diff", ->
-		expect(@diff).to.deep.equal @expected_diff
-
-	it "should get the doc from the doc updater", ->
-		MockDocUpdaterApi.getDoc
-			.calledWith(@project_id, @doc_id)
-			.should.equal true
-		return null

services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee

@@ -1,126 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-db = mongojs.db
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Getting updates", ->
-	before (done) ->
-		@now = Date.now()
-		@to = @now
-		@user_id = ObjectId().toString()
-		@deleted_user_id = 'deleted_user'
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-
-		@minutes = 60 * 1000
-		@hours = 60 * @minutes
-
-		MockWebApi.projects[@project_id] =
-			features:
-				versioning: true
-
-		MockWebApi.users[@user_id] = @user =
-			email: "user@sharelatex.com"
-			first_name: "Leo"
-			last_name: "Lion"
-			id: @user_id
-		sinon.spy MockWebApi, "getUserInfo"
-
-		@updates = []
-		for i in [0..9]
-			@updates.push {
-				op: [{ i: "a", p: 0 }]
-				meta: { ts: @now - (9 - i) * @hours - 2 * @minutes, user_id: @user_id }
-				v: 2 * i + 1
-			}
-			@updates.push {
-				op: [{ i: "b", p: 0 }]
-				meta: { ts: @now - (9 - i) * @hours, user_id: @user_id }
-				v: 2 * i + 2
-			}
-		@updates[0].meta.user_id = @deleted_user_id
-
-		TrackChangesApp.ensureRunning =>
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-				throw error if error?
-				done()
-		return null
-
-	after: () ->
-		MockWebApi.getUserInfo.restore()
-		return null
-
-	describe "getting updates up to the limit", ->
-		before (done) ->
-			TrackChangesClient.getUpdates @project_id, { before: @to + 1, min_count: 3 }, (error, body) =>
-				throw error if error?
-				@updates = body.updates
-				done()
-			return null
-
-		it "should fetch the user details from the web api", ->
-			MockWebApi.getUserInfo
-				.calledWith(@user_id)
-				.should.equal true
-
-		it "should return at least the min_count number of summarized updates", ->
-			docs1 = {}
-			docs1[@doc_id] = toV: 20, fromV: 19
-			docs2 = {}
-			docs2[@doc_id] = toV: 18, fromV: 17
-			docs3 = {}
-			docs3[@doc_id] = toV: 16, fromV: 15
-			expect(@updates.slice(0,3)).to.deep.equal [{
-				docs: docs1
-				meta:
-					start_ts: @to - 2 * @minutes
-					end_ts: @to
-					users: [@user]
-			}, {
-				docs: docs2
-				meta:
-					start_ts: @to - 1 * @hours - 2 * @minutes
-					end_ts: @to - 1 * @hours
-					users: [@user]
-			}, {
-				docs: docs3
-				meta:
-					start_ts: @to - 2 * @hours - 2 * @minutes
-					end_ts: @to - 2 * @hours
-					users: [@user]
-			}]
-
-	describe "getting updates beyond the end of the database", ->
-		before (done) ->
-			TrackChangesClient.getUpdates @project_id, { before: @to - 8 * @hours + 1, min_count: 30 }, (error, body) =>
-				throw error if error?
-				@updates = body.updates
-				done()
-			return null
-
-		it "should return as many updates as it can", ->
-			docs1 = {}
-			docs1[@doc_id] = toV: 4, fromV: 3
-			docs2 = {}
-			docs2[@doc_id] = toV: 2, fromV: 1
-			expect(@updates).to.deep.equal [{
-				docs: docs1
-				meta:
-					start_ts: @to - 8 * @hours - 2 * @minutes
-					end_ts: @to - 8 * @hours
-					users: [@user]
-			}, {
-				docs: docs2
-				meta:
-					start_ts: @to - 9 * @hours - 2 * @minutes
-					end_ts: @to - 9 * @hours
-					users: [@user, null]
-			}]

services/track-changes/test/acceptance/coffee/LockManagerTests.coffee

@@ -1,39 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-LockManager = require "../../../app/js/LockManager"
-rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
-TrackChangesApp = require "./helpers/TrackChangesApp"
-
-describe "Locking document", ->
-
-	before (done)->
-		TrackChangesApp.ensureRunning done
-		return null
-		
-	describe "when the lock has expired in redis", ->
-		before (done) ->
-			LockManager.LOCK_TTL = 1 # second
-			LockManager.runWithLock "doc123", (releaseA) =>
-				# we create a lock A and allow it to expire in redis
-				setTimeout () ->
-					# now we create a new lock B and try to release A
-					LockManager.runWithLock "doc123", (releaseB) =>
-						releaseA()  # try to release lock A to see if it wipes out lock B
-					, (error) ->
-						# we never release lock B so nothing should happen here
-				, 1500 # enough time to wait until the lock has expired
-			, (error) ->
-				# we get here after trying to release lock A
-				done()
-			return null
-
-		it "the new lock should not be removed by the expired locker", (done) ->
-			LockManager.checkLock "doc123", (err, isFree) ->
-					expect(isFree).to.equal false
-					done()
-			return null

services/track-changes/test/acceptance/coffee/RestoringVersions.coffee

@@ -1,74 +0,0 @@
-sinon = require "sinon"
-chai = require("chai")
-chai.should()
-expect = chai.expect
-mongojs = require "../../../app/js/mongojs"
-db = mongojs.db
-ObjectId = mongojs.ObjectId
-Settings = require "settings-sharelatex"
-
-TrackChangesApp = require "./helpers/TrackChangesApp"
-TrackChangesClient = require "./helpers/TrackChangesClient"
-MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
-MockWebApi = require "./helpers/MockWebApi"
-
-describe "Restoring a version", ->
-	before (done) ->
-		sinon.spy MockDocUpdaterApi, "setDoc"
-
-		@now = Date.now()
-		@user_id = ObjectId().toString()
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-		MockWebApi.projects[@project_id] = features: versioning: true
-		
-		minutes = 60 * 1000
-
-		@updates = [{
-			op: [{ i: "one ", p: 0 }]
-			meta: { ts: @now - 6 * minutes, user_id: @user_id }
-			v: 3
-		}, {
-			op: [{ i: "two ", p: 4 }]
-			meta: { ts: @now - 4 * minutes, user_id: @user_id }
-			v: 4
-		}, {
-			op: [{ i: "three ", p: 8 }]
-			meta: { ts: @now - 2 * minutes, user_id: @user_id }
-			v: 5
-		}, {
-			op: [{ i: "four", p: 14 }]
-			meta: { ts: @now, user_id: @user_id }
-			v: 6
-		}]
-		@lines = ["one two three four"]
-		@restored_lines = ["one two "]
-		@beforeVersion = 5
-
-		MockWebApi.users[@user_id] = @user =
-			email: "user@sharelatex.com"
-			first_name: "Leo"
-			last_name: "Lion"
-			id: @user_id
-
-		MockDocUpdaterApi.docs[@doc_id] =
-			lines: @lines
-			version: 7
-
-		TrackChangesApp.ensureRunning =>
-			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-				throw error if error?
-				TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) =>
-					throw error if error?
-					done()
-		return null
-
-	after () ->
-		MockDocUpdaterApi.setDoc.restore()
-		return null
-
-	it "should set the doc in the doc updater", ->
-		MockDocUpdaterApi.setDoc
-			.calledWith(@project_id, @doc_id, @restored_lines, @user_id, true)
-			.should.equal true
-		return null

services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee

@@ -1,27 +0,0 @@
-express = require("express")
-app = express()
-
-module.exports = MockDocUpdaterApi =
-	docs: {}
-
-	getAllDoc: (project_id, callback = (error) ->) ->
-		callback null, @docs
-
-	run: () ->
-		app.get "/project/:project_id/doc", (req, res, next) =>
-			@getAllDoc req.params.project_id, (error, docs) ->
-				if error?
-					res.send 500
-				if !docs?
-					res.send 404
-				else
-					res.send JSON.stringify docs
-
-		app.listen 3016, (error) ->
-			throw error if error?
-		.on "error", (error) ->
-			console.error "error starting MockDocStoreApi:", error.message
-			process.exit(1)
-
-MockDocUpdaterApi.run()
-

services/track-changes/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee

@@ -1,40 +0,0 @@
-express = require("express")
-app = express()
-
-module.exports = MockDocUpdaterApi =
-	docs: {}
-
-	getDoc: (project_id, doc_id, callback = (error) ->) ->
-		callback null, @docs[doc_id]
-
-	setDoc: (project_id, doc_id, lines, user_id, undoing, callback = (error) ->) ->
-		@docs[doc_id] ||= {}
-		@docs[doc_id].lines = lines
-		callback()
-
-	run: () ->
-		app.get "/project/:project_id/doc/:doc_id", (req, res, next) =>
-			@getDoc req.params.project_id, req.params.doc_id, (error, doc) ->
-				if error?
-					res.send 500
-				if !doc?
-					res.send 404
-				else
-					res.send JSON.stringify doc
-
-		app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) =>
-			@setDoc req.params.project_id, req.params.doc_id, req.body.lines, req.body.user_id, req.body.undoing, (errr, doc) ->
-				if error?
-					res.send 500
-				else
-					res.send 204
-
-		app.listen 3003, (error) ->
-			throw error if error?
-		.on "error", (error) ->
-			console.error "error starting MockDocUpdaterApi:", error.message
-			process.exit(1)
-
-
-MockDocUpdaterApi.run()
-

services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee

@@ -1,41 +0,0 @@
-express = require("express")
-app = express()
-
-module.exports = MockWebApi =
-	users: {}
-
-	projects: {}
-
-	getUserInfo: (user_id, callback = (error) ->) ->
-		callback null, @users[user_id] or null
-
-	getProjectDetails: (project_id, callback = (error, project) ->) ->
-		callback null, @projects[project_id]
-
-	run: () ->
-		app.get "/user/:user_id/personal_info", (req, res, next) =>
-			@getUserInfo req.params.user_id, (error, user) ->
-				if error?
-					res.send 500
-				if !user?
-					res.send 404
-				else
-					res.send JSON.stringify user
-
-		app.get "/project/:project_id/details", (req, res, next) =>
-			@getProjectDetails req.params.project_id, (error, project) ->
-				if error?
-					res.send 500
-				if !project?
-					res.send 404
-				else
-					res.send JSON.stringify project
-
-		app.listen 3000, (error) ->
-			throw error if error?
-		.on "error", (error) ->
-			console.error "error starting MockWebApiServer:", error.message
-			process.exit(1)
-
-MockWebApi.run()
-

services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee

@@ -1,24 +0,0 @@
-app = require('../../../../app')
-require("logger-sharelatex")
-logger = require("logger-sharelatex")
-Settings = require("settings-sharelatex")
-
-module.exports =
-	running: false
-	initing: false
-	callbacks: []
-	ensureRunning: (callback = (error) ->) ->
-		if @running
-			return callback()
-		else if @initing
-			@callbacks.push callback
-		else
-			@initing = true
-			@callbacks.push callback
-			app.listen Settings.internal?.trackchanges?.port, "localhost", (error) => 
-				throw error if error?
-				@running = true
-				logger.log("track changes running in dev mode")
-
-				for callback in @callbacks
-					callback()

services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee

@@ -1,144 +0,0 @@
-async = require 'async'
-zlib = require 'zlib'
-request = require "request"
-Settings = require "settings-sharelatex"
-rclient = require("redis-sharelatex").createClient(Settings.redis.history) # Only works locally for now
-Keys = Settings.redis.history.key_schema
-{db, ObjectId} = require "../../../../app/js/mongojs"
-
-aws = require "aws-sdk"
-s3 = new aws.S3(
-	accessKeyId: Settings.trackchanges.s3.key
-	secretAccessKey: Settings.trackchanges.s3.secret
-	endpoint: Settings.trackchanges.s3.endpoint
-	s3ForcePathStyle: Settings.trackchanges.s3.pathStyle
-)
-S3_BUCKET = Settings.trackchanges.stores.doc_history
-
-module.exports = TrackChangesClient =
-	flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) ->
-		TrackChangesClient.flushDoc project_id, doc_id, (error) ->
-			return callback(error) if error?
-			TrackChangesClient.getCompressedUpdates doc_id, callback
-
-	flushDoc: (project_id, doc_id, callback = (error) ->) ->
-		request.post {
-			url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/flush"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 204
-			callback(error)
-
-	flushProject: (project_id, callback = (error) ->) ->
-		request.post {
-			url: "http://localhost:3015/project/#{project_id}/flush"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 204
-			callback(error)
-
-	getCompressedUpdates: (doc_id, callback = (error, updates) ->) ->
-		db.docHistory
-			.find(doc_id: ObjectId(doc_id))
-			.sort("meta.end_ts": 1)
-			.toArray callback
-
-	getProjectMetaData: (project_id, callback = (error, updates) ->) ->
-		db.projectHistoryMetaData
-			.find {
-				project_id: ObjectId(project_id)
-			},
-			(error, projects) ->
-				callback error, projects[0]
-
-	setPreserveHistoryForProject: (project_id, callback = (error) ->) ->
-		db.projectHistoryMetaData.update {
-			project_id: ObjectId(project_id)
-			}, {
-				$set: { preserveHistory: true }
-			}, {
-				upsert: true
-			}, callback
-
-	pushRawUpdates: (project_id, doc_id, updates, callback = (error) ->) ->
-		rclient.sadd Keys.docsWithHistoryOps({project_id}), doc_id, (error) ->
-			return callback(error) if error?
-			rclient.rpush Keys.uncompressedHistoryOps({doc_id}), (JSON.stringify(u) for u in updates)..., callback
-
-	getDiff: (project_id, doc_id, from, to, callback = (error, diff) ->) ->
-		request.get {
-			url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/diff?from=#{from}&to=#{to}"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 200
-			callback null, JSON.parse(body)
-
-	getUpdates: (project_id, options, callback = (error, body) ->) ->
-		request.get {
-			url: "http://localhost:3015/project/#{project_id}/updates?before=#{options.before}&min_count=#{options.min_count}"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 200
-			callback null, JSON.parse(body)
-
-	restoreDoc: (project_id, doc_id, version, user_id, callback = (error) ->) ->
-		request.post {
-			url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/version/#{version}/restore"
-			headers:
-				"X-User-Id": user_id
-		}, (error, response, body) =>
-			response.statusCode.should.equal 204
-			callback null
-
-	pushDocHistory: (project_id, doc_id, callback = (error) ->) ->
-		request.post {
-			url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/push"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 204
-			callback(error)
-
-	pullDocHistory: (project_id, doc_id, callback = (error) ->) ->
-		request.post {
-			url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/pull"
-		}, (error, response, body) =>
-			response.statusCode.should.equal 204
-			callback(error)
-
-	waitForS3: (done, retries=42) ->
-		if !Settings.trackchanges.s3.endpoint
-			return done()
-
-		request.get "#{Settings.trackchanges.s3.endpoint}/", (err, res) ->
-			if res && res.statusCode < 500
-				return done()
-
-			if retries == 0
-				return done(err or new Error("s3 returned #{res.statusCode}"))
-
-			setTimeout () ->
-				TrackChangesClient.waitForS3(done, --retries)
-			, 1000
-
-	getS3Doc: (project_id, doc_id, pack_id, callback = (error, body) ->) ->
-		params =
-			Bucket: S3_BUCKET
-			Key: "#{project_id}/changes-#{doc_id}/pack-#{pack_id}"
-
-		s3.getObject params, (error, data) ->
-			return callback(error) if error?
-			body = data.Body
-			return callback(new Error("empty response from s3")) if not body?
-			zlib.gunzip body, (err, result) ->
-				return callback(err) if err?
-				callback(null, JSON.parse(result.toString()))
-
-	removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
-		params =
-			Bucket: S3_BUCKET
-			Prefix: "#{project_id}/changes-#{doc_id}"
-
-		s3.listObjects params, (error, data) ->
-			return callback(error) if error?
-
-			params =
-				Bucket: S3_BUCKET
-				Delete:
-					Objects: data.Contents.map((s3object) -> {Key: s3object.Key})
-
-			s3.deleteObjects params, callback

services/track-changes/test/acceptance/js/AppendingUpdatesTests.js

@@ -0,0 +1,590 @@
+/* eslint-disable
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+const request = require('request')
+const rclient = require('redis').createClient(Settings.redis.history) // Only works locally for now
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Appending doc ops to the history', function() {
+  before(function(done) {
+    return TrackChangesApp.ensureRunning(done)
+  })
+
+  describe('when the history does not exist yet', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [{ i: 'f', p: 3 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          },
+          {
+            op: [{ i: 'o', p: 4 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 4
+          },
+          {
+            op: [{ i: 'o', p: 5 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 5
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    it('should insert the compressed op into mongo', function() {
+      return expect(this.updates[0].pack[0].op).to.deep.equal([
+        {
+          p: 3,
+          i: 'foo'
+        }
+      ])
+    })
+
+    it('should insert the correct version number into mongo', function() {
+      return expect(this.updates[0].v).to.equal(5)
+    })
+
+    it('should store the doc id', function() {
+      return expect(this.updates[0].doc_id.toString()).to.equal(this.doc_id)
+    })
+
+    it('should store the project id', function() {
+      return expect(this.updates[0].project_id.toString()).to.equal(
+        this.project_id
+      )
+    })
+
+    return it('should clear the doc from the DocsWithHistoryOps set', function(done) {
+      rclient.sismember(
+        `DocsWithHistoryOps:${this.project_id}`,
+        this.doc_id,
+        (error, member) => {
+          member.should.equal(0)
+          return done()
+        }
+      )
+      return null
+    })
+  })
+
+  describe('when the history has already been started', function() {
+    beforeEach(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [{ i: 'f', p: 3 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          },
+          {
+            op: [{ i: 'o', p: 4 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 4
+          },
+          {
+            op: [{ i: 'o', p: 5 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 5
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    describe('when the updates are recent and from the same user', function() {
+      beforeEach(function(done) {
+        TrackChangesClient.pushRawUpdates(
+          this.project_id,
+          this.doc_id,
+          [
+            {
+              op: [{ i: 'b', p: 6 }],
+              meta: { ts: Date.now(), user_id: this.user_id },
+              v: 6
+            },
+            {
+              op: [{ i: 'a', p: 7 }],
+              meta: { ts: Date.now(), user_id: this.user_id },
+              v: 7
+            },
+            {
+              op: [{ i: 'r', p: 8 }],
+              meta: { ts: Date.now(), user_id: this.user_id },
+              v: 8
+            }
+          ],
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return TrackChangesClient.flushAndGetCompressedUpdates(
+              this.project_id,
+              this.doc_id,
+              (error, updates) => {
+                this.updates = updates
+                if (error != null) {
+                  throw error
+                }
+                return done()
+              }
+            )
+          }
+        )
+        return null
+      })
+
+      it('should combine all the updates into one pack', function() {
+        return expect(this.updates[0].pack[1].op).to.deep.equal([
+          {
+            p: 6,
+            i: 'bar'
+          }
+        ])
+      })
+
+      return it('should insert the correct version number into mongo', function() {
+        return expect(this.updates[0].v_end).to.equal(8)
+      })
+    })
+
+    return describe('when the updates are far apart', function() {
+      beforeEach(function(done) {
+        const oneDay = 24 * 60 * 60 * 1000
+        TrackChangesClient.pushRawUpdates(
+          this.project_id,
+          this.doc_id,
+          [
+            {
+              op: [{ i: 'b', p: 6 }],
+              meta: { ts: Date.now() + oneDay, user_id: this.user_id },
+              v: 6
+            },
+            {
+              op: [{ i: 'a', p: 7 }],
+              meta: { ts: Date.now() + oneDay, user_id: this.user_id },
+              v: 7
+            },
+            {
+              op: [{ i: 'r', p: 8 }],
+              meta: { ts: Date.now() + oneDay, user_id: this.user_id },
+              v: 8
+            }
+          ],
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return TrackChangesClient.flushAndGetCompressedUpdates(
+              this.project_id,
+              this.doc_id,
+              (error, updates) => {
+                this.updates = updates
+                if (error != null) {
+                  throw error
+                }
+                return done()
+              }
+            )
+          }
+        )
+        return null
+      })
+
+      return it('should combine the updates into one pack', function() {
+        expect(this.updates[0].pack[0].op).to.deep.equal([
+          {
+            p: 3,
+            i: 'foo'
+          }
+        ])
+        return expect(this.updates[0].pack[1].op).to.deep.equal([
+          {
+            p: 6,
+            i: 'bar'
+          }
+        ])
+      })
+    })
+  })
+
+  describe('when the updates need processing in batches', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      const updates = []
+      this.expectedOp = [{ p: 0, i: '' }]
+      for (let i = 0; i <= 250; i++) {
+        updates.push({
+          op: [{ i: 'a', p: 0 }],
+          meta: { ts: Date.now(), user_id: this.user_id },
+          v: i
+        })
+        this.expectedOp[0].i = `a${this.expectedOp[0].i}`
+      }
+
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        updates,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates1) => {
+              this.updates = updates1
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    it('should concat the compressed op into mongo', function() {
+      return expect(this.updates[0].pack.length).to.deep.equal(3)
+    }) // batch size is 100
+
+    return it('should insert the correct version number into mongo', function() {
+      return expect(this.updates[0].v_end).to.equal(250)
+    })
+  })
+
+  describe('when there are multiple ops in each update', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      const oneDay = 24 * 60 * 60 * 1000
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [
+              { i: 'f', p: 3 },
+              { i: 'o', p: 4 },
+              { i: 'o', p: 5 }
+            ],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          },
+          {
+            op: [
+              { i: 'b', p: 6 },
+              { i: 'a', p: 7 },
+              { i: 'r', p: 8 }
+            ],
+            meta: { ts: Date.now() + oneDay, user_id: this.user_id },
+            v: 4
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    it('should insert the compressed ops into mongo', function() {
+      expect(this.updates[0].pack[0].op).to.deep.equal([
+        {
+          p: 3,
+          i: 'foo'
+        }
+      ])
+      return expect(this.updates[0].pack[1].op).to.deep.equal([
+        {
+          p: 6,
+          i: 'bar'
+        }
+      ])
+    })
+
+    return it('should insert the correct version numbers into mongo', function() {
+      expect(this.updates[0].pack[0].v).to.equal(3)
+      return expect(this.updates[0].pack[1].v).to.equal(4)
+    })
+  })
+
+  describe('when there is a no-op update', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      const oneDay = 24 * 60 * 60 * 1000
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          },
+          {
+            op: [{ i: 'foo', p: 3 }],
+            meta: { ts: Date.now() + oneDay, user_id: this.user_id },
+            v: 4
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    it('should insert the compressed no-op into mongo', function() {
+      return expect(this.updates[0].pack[0].op).to.deep.equal([])
+    })
+
+    it('should insert the compressed next update into mongo', function() {
+      return expect(this.updates[0].pack[1].op).to.deep.equal([
+        {
+          p: 3,
+          i: 'foo'
+        }
+      ])
+    })
+
+    return it('should insert the correct version numbers into mongo', function() {
+      expect(this.updates[0].pack[0].v).to.equal(3)
+      return expect(this.updates[0].pack[1].v).to.equal(4)
+    })
+  })
+
+  describe('when there is a comment update', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [
+              { c: 'foo', p: 3 },
+              { d: 'bar', p: 6 }
+            ],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    it('should ignore the comment op', function() {
+      return expect(this.updates[0].pack[0].op).to.deep.equal([
+        { d: 'bar', p: 6 }
+      ])
+    })
+
+    return it('should insert the correct version numbers into mongo', function() {
+      return expect(this.updates[0].pack[0].v).to.equal(3)
+    })
+  })
+
+  describe('when the project has versioning enabled', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: true } }
+
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [{ i: 'f', p: 3 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    return it('should not add a expiresAt entry in the update in mongo', function() {
+      return expect(this.updates[0].expiresAt).to.be.undefined
+    })
+  })
+
+  return describe('when the project does not have versioning enabled', function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: false } }
+
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [{ i: 'f', p: 3 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushAndGetCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            (error, updates) => {
+              this.updates = updates
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    return it('should add a expiresAt entry in the update in mongo', function() {
+      return expect(this.updates[0].expiresAt).to.exist
+    })
+  })
+})

services/track-changes/test/acceptance/js/ArchivingUpdatesTests.js

@@ -0,0 +1,273 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-undef,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS202: Simplify dynamic range loops
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { db } = mongojs
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+const request = require('request')
+const rclient = require('redis').createClient(Settings.redis.history) // Only works locally for now
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockDocStoreApi = require('./helpers/MockDocStoreApi')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Archiving updates', function() {
+  before(function(done) {
+    if (
+      __guard__(
+        __guard__(
+          Settings != null ? Settings.trackchanges : undefined,
+          x1 => x1.s3
+        ),
+        x => x.key.length
+      ) < 1
+    ) {
+      const message = new Error('s3 keys not setup, this test setup will fail')
+      return done(message)
+    }
+
+    return TrackChangesClient.waitForS3(done)
+  })
+
+  before(function(done) {
+    this.now = Date.now()
+    this.to = this.now
+    this.user_id = ObjectId().toString()
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+
+    this.minutes = 60 * 1000
+    this.hours = 60 * this.minutes
+
+    MockWebApi.projects[this.project_id] = {
+      features: {
+        versioning: true
+      }
+    }
+    sinon.spy(MockWebApi, 'getProjectDetails')
+
+    MockWebApi.users[this.user_id] = this.user = {
+      email: 'user@sharelatex.com',
+      first_name: 'Leo',
+      last_name: 'Lion',
+      id: this.user_id
+    }
+    sinon.spy(MockWebApi, 'getUserInfo')
+
+    MockDocStoreApi.docs[this.doc_id] = this.doc = {
+      _id: this.doc_id,
+      project_id: this.project_id
+    }
+    sinon.spy(MockDocStoreApi, 'getAllDoc')
+
+    this.updates = []
+    for (
+      let i = 0, end = 512 + 10, asc = end >= 0;
+      asc ? i <= end : i >= end;
+      asc ? i++ : i--
+    ) {
+      this.updates.push({
+        op: [{ i: 'a', p: 0 }],
+        meta: { ts: this.now + (i - 2048) * this.hours, user_id: this.user_id },
+        v: 2 * i + 1
+      })
+      this.updates.push({
+        op: [{ i: 'b', p: 0 }],
+        meta: {
+          ts: this.now + (i - 2048) * this.hours + 10 * this.minutes,
+          user_id: this.user_id
+        },
+        v: 2 * i + 2
+      })
+    }
+    TrackChangesApp.ensureRunning(() => {
+      return TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        this.updates,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushDoc(
+            this.project_id,
+            this.doc_id,
+            error => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+    })
+    return null
+  })
+
+  after(function(done) {
+    MockWebApi.getUserInfo.restore()
+    return db.docHistory.remove(
+      { project_id: ObjectId(this.project_id) },
+      () => {
+        return db.docHistoryIndex.remove(
+          { project_id: ObjectId(this.project_id) },
+          () => {
+            return TrackChangesClient.removeS3Doc(
+              this.project_id,
+              this.doc_id,
+              done
+            )
+          }
+        )
+      }
+    )
+  })
+
+  describe("archiving a doc's updates", function() {
+    before(function(done) {
+      TrackChangesClient.pushDocHistory(this.project_id, this.doc_id, error => {
+        if (error != null) {
+          throw error
+        }
+        return done()
+      })
+      return null
+    })
+
+    it('should have one cached pack', function(done) {
+      return db.docHistory.count(
+        { doc_id: ObjectId(this.doc_id), expiresAt: { $exists: true } },
+        (error, count) => {
+          if (error != null) {
+            throw error
+          }
+          count.should.equal(1)
+          return done()
+        }
+      )
+    })
+
+    it('should have one remaining pack after cache is expired', function(done) {
+      return db.docHistory.remove(
+        {
+          doc_id: ObjectId(this.doc_id),
+          expiresAt: { $exists: true }
+        },
+        (err, result) => {
+          if (typeof error !== 'undefined' && error !== null) {
+            throw error
+          }
+          return db.docHistory.count(
+            { doc_id: ObjectId(this.doc_id) },
+            (error, count) => {
+              if (error != null) {
+                throw error
+              }
+              count.should.equal(1)
+              return done()
+            }
+          )
+        }
+      )
+    })
+
+    it('should have a docHistoryIndex entry marked as inS3', function(done) {
+      return db.docHistoryIndex.findOne(
+        { _id: ObjectId(this.doc_id) },
+        (error, index) => {
+          if (error != null) {
+            throw error
+          }
+          index.packs[0].inS3.should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    it('should have a docHistoryIndex entry with the last version', function(done) {
+      return db.docHistoryIndex.findOne(
+        { _id: ObjectId(this.doc_id) },
+        (error, index) => {
+          if (error != null) {
+            throw error
+          }
+          index.packs[0].v_end.should.equal(1024)
+          return done()
+        }
+      )
+    })
+
+    return it('should store 1024 doc changes in S3 in one pack', function(done) {
+      return db.docHistoryIndex.findOne(
+        { _id: ObjectId(this.doc_id) },
+        (error, index) => {
+          if (error != null) {
+            throw error
+          }
+          const pack_id = index.packs[0]._id
+          return TrackChangesClient.getS3Doc(
+            this.project_id,
+            this.doc_id,
+            pack_id,
+            (error, doc) => {
+              doc.n.should.equal(1024)
+              doc.pack.length.should.equal(1024)
+              return done()
+            }
+          )
+        }
+      )
+    })
+  })
+
+  return describe("unarchiving a doc's updates", function() {
+    before(function(done) {
+      TrackChangesClient.pullDocHistory(this.project_id, this.doc_id, error => {
+        if (error != null) {
+          throw error
+        }
+        return done()
+      })
+      return null
+    })
+
+    return it('should restore both packs', function(done) {
+      return db.docHistory.count(
+        { doc_id: ObjectId(this.doc_id) },
+        (error, count) => {
+          if (error != null) {
+            throw error
+          }
+          count.should.equal(2)
+          return done()
+        }
+      )
+    })
+  })
+})
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/test/acceptance/js/FlushingUpdatesTests.js

@@ -0,0 +1,276 @@
+/* eslint-disable
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+const request = require('request')
+const rclient = require('redis').createClient(Settings.redis.history) // Only works locally for now
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Flushing updates', function() {
+  before(function(done) {
+    return TrackChangesApp.ensureRunning(done)
+  })
+
+  describe("flushing a doc's updates", function() {
+    before(function(done) {
+      this.project_id = ObjectId().toString()
+      this.doc_id = ObjectId().toString()
+      this.user_id = ObjectId().toString()
+      MockWebApi.projects[this.project_id] = { features: { versioning: true } }
+
+      TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        [
+          {
+            op: [{ i: 'f', p: 3 }],
+            meta: { ts: Date.now(), user_id: this.user_id },
+            v: 3
+          }
+        ],
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.flushDoc(
+            this.project_id,
+            this.doc_id,
+            error => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+      return null
+    })
+
+    return it('should flush the op into mongo', function(done) {
+      TrackChangesClient.getCompressedUpdates(this.doc_id, (error, updates) => {
+        expect(updates[0].pack[0].op).to.deep.equal([
+          {
+            p: 3,
+            i: 'f'
+          }
+        ])
+        return done()
+      })
+      return null
+    })
+  })
+
+  return describe("flushing a project's updates", function() {
+    describe('with versioning enabled', function() {
+      before(function(done) {
+        this.project_id = ObjectId().toString()
+        this.doc_id = ObjectId().toString()
+        this.user_id = ObjectId().toString()
+
+        this.weeks = 7 * 24 * 60 * 60 * 1000
+
+        MockWebApi.projects[this.project_id] = {
+          features: {
+            versioning: true
+          }
+        }
+
+        TrackChangesClient.pushRawUpdates(
+          this.project_id,
+          this.doc_id,
+          [
+            {
+              op: [{ i: 'g', p: 2 }],
+              meta: { ts: Date.now() - 2 * this.weeks, user_id: this.user_id },
+              v: 2
+            },
+            {
+              op: [{ i: 'f', p: 3 }],
+              meta: { ts: Date.now(), user_id: this.user_id },
+              v: 3
+            }
+          ],
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return TrackChangesClient.flushProject(this.project_id, error => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            })
+          }
+        )
+        return null
+      })
+
+      it('should not mark the updates for deletion', function(done) {
+        TrackChangesClient.getCompressedUpdates(
+          this.doc_id,
+          (error, updates) => {
+            expect(updates[0].expiresAt).to.not.exist
+            return done()
+          }
+        )
+        return null
+      })
+
+      return it('should preserve history forever', function(done) {
+        TrackChangesClient.getProjectMetaData(
+          this.project_id,
+          (error, project) => {
+            expect(project.preserveHistory).to.equal(true)
+            return done()
+          }
+        )
+        return null
+      })
+    })
+
+    describe('without versioning enabled', function() {
+      before(function(done) {
+        this.project_id = ObjectId().toString()
+        this.doc_id = ObjectId().toString()
+        this.user_id = ObjectId().toString()
+
+        this.weeks = 7 * 24 * 60 * 60 * 1000
+
+        MockWebApi.projects[this.project_id] = {
+          features: {
+            versioning: false
+          }
+        }
+
+        TrackChangesClient.pushRawUpdates(
+          this.project_id,
+          this.doc_id,
+          [
+            {
+              op: [{ i: 'g', p: 2 }],
+              meta: { ts: Date.now() - 2 * this.weeks, user_id: this.user_id },
+              v: 2
+            },
+            {
+              op: [{ i: 'f', p: 3 }],
+              meta: { ts: Date.now(), user_id: this.user_id },
+              v: 3
+            }
+          ],
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return TrackChangesClient.flushProject(this.project_id, error => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            })
+          }
+        )
+        return null
+      })
+
+      return it('should mark the updates for deletion', function(done) {
+        TrackChangesClient.getCompressedUpdates(
+          this.doc_id,
+          (error, updates) => {
+            expect(updates[0].expiresAt).to.exist
+            return done()
+          }
+        )
+        return null
+      })
+    })
+
+    return describe('without versioning enabled but with preserveHistory set to true', function() {
+      before(function(done) {
+        this.project_id = ObjectId().toString()
+        this.doc_id = ObjectId().toString()
+        this.user_id = ObjectId().toString()
+
+        this.weeks = 7 * 24 * 60 * 60 * 1000
+
+        MockWebApi.projects[this.project_id] = {
+          features: {
+            versioning: false
+          }
+        }
+
+        TrackChangesClient.setPreserveHistoryForProject(
+          this.project_id,
+          error => {
+            if (error != null) {
+              throw error
+            }
+            return TrackChangesClient.pushRawUpdates(
+              this.project_id,
+              this.doc_id,
+              [
+                {
+                  op: [{ i: 'g', p: 2 }],
+                  meta: {
+                    ts: Date.now() - 2 * this.weeks,
+                    user_id: this.user_id
+                  },
+                  v: 2
+                },
+                {
+                  op: [{ i: 'f', p: 3 }],
+                  meta: { ts: Date.now(), user_id: this.user_id },
+                  v: 3
+                }
+              ],
+              error => {
+                if (error != null) {
+                  throw error
+                }
+                return TrackChangesClient.flushProject(
+                  this.project_id,
+                  error => {
+                    if (error != null) {
+                      throw error
+                    }
+                    return done()
+                  }
+                )
+              }
+            )
+          }
+        )
+        return null
+      })
+
+      return it('should not mark the updates for deletion', function(done) {
+        TrackChangesClient.getCompressedUpdates(
+          this.doc_id,
+          (error, updates) => {
+            expect(updates[0].expiresAt).to.not.exist
+            return done()
+          }
+        )
+        return null
+      })
+    })
+  })
+})

services/track-changes/test/acceptance/js/GettingADiffTests.js

@@ -0,0 +1,131 @@
+/* eslint-disable
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { db } = mongojs
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockDocUpdaterApi = require('./helpers/MockDocUpdaterApi')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Getting a diff', function() {
+  beforeEach(function(done) {
+    sinon.spy(MockDocUpdaterApi, 'getDoc')
+
+    this.now = Date.now()
+    this.from = this.now - 100000000
+    this.to = this.now
+    this.user_id = ObjectId().toString()
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+    MockWebApi.projects[this.project_id] = { features: { versioning: true } }
+
+    MockWebApi.users[this.user_id] = this.user = {
+      email: 'user@sharelatex.com',
+      first_name: 'Leo',
+      last_name: 'Lion',
+      id: this.user_id
+    }
+    sinon.spy(MockWebApi, 'getUserInfo')
+
+    const twoMinutes = 2 * 60 * 1000
+
+    this.updates = [
+      {
+        op: [{ i: 'one ', p: 0 }],
+        meta: { ts: this.from - twoMinutes, user_id: this.user_id },
+        v: 3
+      },
+      {
+        op: [{ i: 'two ', p: 4 }],
+        meta: { ts: this.from + twoMinutes, user_id: this.user_id },
+        v: (this.fromVersion = 4)
+      },
+      {
+        op: [{ i: 'three ', p: 8 }],
+        meta: { ts: this.to - twoMinutes, user_id: this.user_id },
+        v: (this.toVersion = 5)
+      },
+      {
+        op: [{ i: 'four', p: 14 }],
+        meta: { ts: this.to + twoMinutes, user_id: this.user_id },
+        v: 6
+      }
+    ]
+    this.lines = ['one two three four']
+    this.expected_diff = [
+      { u: 'one ' },
+      {
+        i: 'two three ',
+        meta: {
+          start_ts: this.from + twoMinutes,
+          end_ts: this.to - twoMinutes,
+          user: this.user
+        }
+      }
+    ]
+
+    MockDocUpdaterApi.docs[this.doc_id] = {
+      lines: this.lines,
+      version: 7
+    }
+    TrackChangesApp.ensureRunning(() => {
+      return TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        this.updates,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.getDiff(
+            this.project_id,
+            this.doc_id,
+            this.fromVersion,
+            this.toVersion,
+            (error, diff) => {
+              if (error != null) {
+                throw error
+              }
+              this.diff = diff.diff
+              return done()
+            }
+          )
+        }
+      )
+    })
+    return null
+  })
+
+  afterEach(function() {
+    MockDocUpdaterApi.getDoc.restore()
+    MockWebApi.getUserInfo.restore()
+    return null
+  })
+
+  it('should return the diff', function() {
+    return expect(this.diff).to.deep.equal(this.expected_diff)
+  })
+
+  return it('should get the doc from the doc updater', function() {
+    MockDocUpdaterApi.getDoc
+      .calledWith(this.project_id, this.doc_id)
+      .should.equal(true)
+    return null
+  })
+})

services/track-changes/test/acceptance/js/GettingUpdatesTests.js

@@ -0,0 +1,190 @@
+/* eslint-disable
+    chai-friendly/no-unused-expressions,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { db } = mongojs
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Getting updates', function() {
+  before(function(done) {
+    this.now = Date.now()
+    this.to = this.now
+    this.user_id = ObjectId().toString()
+    this.deleted_user_id = 'deleted_user'
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+
+    this.minutes = 60 * 1000
+    this.hours = 60 * this.minutes
+
+    MockWebApi.projects[this.project_id] = {
+      features: {
+        versioning: true
+      }
+    }
+
+    MockWebApi.users[this.user_id] = this.user = {
+      email: 'user@sharelatex.com',
+      first_name: 'Leo',
+      last_name: 'Lion',
+      id: this.user_id
+    }
+    sinon.spy(MockWebApi, 'getUserInfo')
+
+    this.updates = []
+    for (let i = 0; i <= 9; i++) {
+      this.updates.push({
+        op: [{ i: 'a', p: 0 }],
+        meta: {
+          ts: this.now - (9 - i) * this.hours - 2 * this.minutes,
+          user_id: this.user_id
+        },
+        v: 2 * i + 1
+      })
+      this.updates.push({
+        op: [{ i: 'b', p: 0 }],
+        meta: { ts: this.now - (9 - i) * this.hours, user_id: this.user_id },
+        v: 2 * i + 2
+      })
+    }
+    this.updates[0].meta.user_id = this.deleted_user_id
+
+    TrackChangesApp.ensureRunning(() => {
+      return TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        this.updates,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        }
+      )
+    })
+    return null
+  })
+
+  ;({
+    after() {
+      MockWebApi.getUserInfo.restore()
+      return null
+    }
+  })
+
+  describe('getting updates up to the limit', function() {
+    before(function(done) {
+      TrackChangesClient.getUpdates(
+        this.project_id,
+        { before: this.to + 1, min_count: 3 },
+        (error, body) => {
+          if (error != null) {
+            throw error
+          }
+          this.updates = body.updates
+          return done()
+        }
+      )
+      return null
+    })
+
+    it('should fetch the user details from the web api', function() {
+      return MockWebApi.getUserInfo.calledWith(this.user_id).should.equal(true)
+    })
+
+    return it('should return at least the min_count number of summarized updates', function() {
+      const docs1 = {}
+      docs1[this.doc_id] = { toV: 20, fromV: 19 }
+      const docs2 = {}
+      docs2[this.doc_id] = { toV: 18, fromV: 17 }
+      const docs3 = {}
+      docs3[this.doc_id] = { toV: 16, fromV: 15 }
+      return expect(this.updates.slice(0, 3)).to.deep.equal([
+        {
+          docs: docs1,
+          meta: {
+            start_ts: this.to - 2 * this.minutes,
+            end_ts: this.to,
+            users: [this.user]
+          }
+        },
+        {
+          docs: docs2,
+          meta: {
+            start_ts: this.to - 1 * this.hours - 2 * this.minutes,
+            end_ts: this.to - 1 * this.hours,
+            users: [this.user]
+          }
+        },
+        {
+          docs: docs3,
+          meta: {
+            start_ts: this.to - 2 * this.hours - 2 * this.minutes,
+            end_ts: this.to - 2 * this.hours,
+            users: [this.user]
+          }
+        }
+      ])
+    })
+  })
+
+  return describe('getting updates beyond the end of the database', function() {
+    before(function(done) {
+      TrackChangesClient.getUpdates(
+        this.project_id,
+        { before: this.to - 8 * this.hours + 1, min_count: 30 },
+        (error, body) => {
+          if (error != null) {
+            throw error
+          }
+          this.updates = body.updates
+          return done()
+        }
+      )
+      return null
+    })
+
+    return it('should return as many updates as it can', function() {
+      const docs1 = {}
+      docs1[this.doc_id] = { toV: 4, fromV: 3 }
+      const docs2 = {}
+      docs2[this.doc_id] = { toV: 2, fromV: 1 }
+      return expect(this.updates).to.deep.equal([
+        {
+          docs: docs1,
+          meta: {
+            start_ts: this.to - 8 * this.hours - 2 * this.minutes,
+            end_ts: this.to - 8 * this.hours,
+            users: [this.user]
+          }
+        },
+        {
+          docs: docs2,
+          meta: {
+            start_ts: this.to - 9 * this.hours - 2 * this.minutes,
+            end_ts: this.to - 9 * this.hours,
+            users: [this.user, null]
+          }
+        }
+      ])
+    })
+  })
+})

services/track-changes/test/acceptance/js/LockManagerTests.js

@@ -0,0 +1,66 @@
+/* eslint-disable
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+const LockManager = require('../../../app/js/LockManager')
+const rclient = require('redis').createClient(Settings.redis.history) // Only works locally for now
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+
+describe('Locking document', function() {
+  before(function(done) {
+    TrackChangesApp.ensureRunning(done)
+    return null
+  })
+
+  return describe('when the lock has expired in redis', function() {
+    before(function(done) {
+      LockManager.LOCK_TTL = 1 // second
+      LockManager.runWithLock(
+        'doc123',
+        releaseA => {
+          // we create a lock A and allow it to expire in redis
+          return setTimeout(
+            () =>
+              // now we create a new lock B and try to release A
+              LockManager.runWithLock(
+                'doc123',
+                releaseB => {
+                  return releaseA()
+                }, // try to release lock A to see if it wipes out lock B
+                error => {}
+              ),
+
+            // we never release lock B so nothing should happen here
+            1500
+          )
+        }, // enough time to wait until the lock has expired
+        error =>
+          // we get here after trying to release lock A
+          done()
+      )
+      return null
+    })
+
+    return it('the new lock should not be removed by the expired locker', function(done) {
+      LockManager.checkLock('doc123', (err, isFree) => {
+        expect(isFree).to.equal(false)
+        return done()
+      })
+      return null
+    })
+  })
+})

services/track-changes/test/acceptance/js/RestoringVersions.js

@@ -0,0 +1,120 @@
+/* eslint-disable
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+chai.should()
+const { expect } = chai
+const mongojs = require('../../../app/js/mongojs')
+const { db } = mongojs
+const { ObjectId } = mongojs
+const Settings = require('settings-sharelatex')
+
+const TrackChangesApp = require('./helpers/TrackChangesApp')
+const TrackChangesClient = require('./helpers/TrackChangesClient')
+const MockDocUpdaterApi = require('./helpers/MockDocUpdaterApi')
+const MockWebApi = require('./helpers/MockWebApi')
+
+describe('Restoring a version', function() {
+  before(function(done) {
+    sinon.spy(MockDocUpdaterApi, 'setDoc')
+
+    this.now = Date.now()
+    this.user_id = ObjectId().toString()
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+    MockWebApi.projects[this.project_id] = { features: { versioning: true } }
+
+    const minutes = 60 * 1000
+
+    this.updates = [
+      {
+        op: [{ i: 'one ', p: 0 }],
+        meta: { ts: this.now - 6 * minutes, user_id: this.user_id },
+        v: 3
+      },
+      {
+        op: [{ i: 'two ', p: 4 }],
+        meta: { ts: this.now - 4 * minutes, user_id: this.user_id },
+        v: 4
+      },
+      {
+        op: [{ i: 'three ', p: 8 }],
+        meta: { ts: this.now - 2 * minutes, user_id: this.user_id },
+        v: 5
+      },
+      {
+        op: [{ i: 'four', p: 14 }],
+        meta: { ts: this.now, user_id: this.user_id },
+        v: 6
+      }
+    ]
+    this.lines = ['one two three four']
+    this.restored_lines = ['one two ']
+    this.beforeVersion = 5
+
+    MockWebApi.users[this.user_id] = this.user = {
+      email: 'user@sharelatex.com',
+      first_name: 'Leo',
+      last_name: 'Lion',
+      id: this.user_id
+    }
+
+    MockDocUpdaterApi.docs[this.doc_id] = {
+      lines: this.lines,
+      version: 7
+    }
+
+    TrackChangesApp.ensureRunning(() => {
+      return TrackChangesClient.pushRawUpdates(
+        this.project_id,
+        this.doc_id,
+        this.updates,
+        error => {
+          if (error != null) {
+            throw error
+          }
+          return TrackChangesClient.restoreDoc(
+            this.project_id,
+            this.doc_id,
+            this.beforeVersion,
+            this.user_id,
+            error => {
+              if (error != null) {
+                throw error
+              }
+              return done()
+            }
+          )
+        }
+      )
+    })
+    return null
+  })
+
+  after(function() {
+    MockDocUpdaterApi.setDoc.restore()
+    return null
+  })
+
+  return it('should set the doc in the doc updater', function() {
+    MockDocUpdaterApi.setDoc
+      .calledWith(
+        this.project_id,
+        this.doc_id,
+        this.restored_lines,
+        this.user_id,
+        true
+      )
+      .should.equal(true)
+    return null
+  })
+})

services/track-changes/test/acceptance/js/helpers/MockDocStoreApi.js

@@ -0,0 +1,54 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockDocUpdaterApi
+const express = require('express')
+const app = express()
+
+module.exports = MockDocUpdaterApi = {
+  docs: {},
+
+  getAllDoc(project_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return callback(null, this.docs)
+  },
+
+  run() {
+    app.get('/project/:project_id/doc', (req, res, next) => {
+      return this.getAllDoc(req.params.project_id, (error, docs) => {
+        if (error != null) {
+          res.send(500)
+        }
+        if (docs == null) {
+          return res.send(404)
+        } else {
+          return res.send(JSON.stringify(docs))
+        }
+      })
+    })
+
+    return app
+      .listen(3016, error => {
+        if (error != null) {
+          throw error
+        }
+      })
+      .on('error', error => {
+        console.error('error starting MockDocStoreApi:', error.message)
+        return process.exit(1)
+      })
+  }
+}
+
+MockDocUpdaterApi.run()

services/track-changes/test/acceptance/js/helpers/MockDocUpdaterApi.js

@@ -0,0 +1,91 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-undef,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockDocUpdaterApi
+const express = require('express')
+const app = express()
+
+module.exports = MockDocUpdaterApi = {
+  docs: {},
+
+  getDoc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return callback(null, this.docs[doc_id])
+  },
+
+  setDoc(project_id, doc_id, lines, user_id, undoing, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (!this.docs[doc_id]) {
+      this.docs[doc_id] = {}
+    }
+    this.docs[doc_id].lines = lines
+    return callback()
+  },
+
+  run() {
+    app.get('/project/:project_id/doc/:doc_id', (req, res, next) => {
+      return this.getDoc(
+        req.params.project_id,
+        req.params.doc_id,
+        (error, doc) => {
+          if (error != null) {
+            res.send(500)
+          }
+          if (doc == null) {
+            return res.send(404)
+          } else {
+            return res.send(JSON.stringify(doc))
+          }
+        }
+      )
+    })
+
+    app.post(
+      '/project/:project_id/doc/:doc_id',
+      express.bodyParser(),
+      (req, res, next) => {
+        return this.setDoc(
+          req.params.project_id,
+          req.params.doc_id,
+          req.body.lines,
+          req.body.user_id,
+          req.body.undoing,
+          (errr, doc) => {
+            if (typeof error !== 'undefined' && error !== null) {
+              return res.send(500)
+            } else {
+              return res.send(204)
+            }
+          }
+        )
+      }
+    )
+
+    return app
+      .listen(3003, error => {
+        if (error != null) {
+          throw error
+        }
+      })
+      .on('error', error => {
+        console.error('error starting MockDocUpdaterApi:', error.message)
+        return process.exit(1)
+      })
+  }
+}
+
+MockDocUpdaterApi.run()

services/track-changes/test/acceptance/js/helpers/MockWebApi.js

@@ -0,0 +1,76 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockWebApi
+const express = require('express')
+const app = express()
+
+module.exports = MockWebApi = {
+  users: {},
+
+  projects: {},
+
+  getUserInfo(user_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return callback(null, this.users[user_id] || null)
+  },
+
+  getProjectDetails(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, project) {}
+    }
+    return callback(null, this.projects[project_id])
+  },
+
+  run() {
+    app.get('/user/:user_id/personal_info', (req, res, next) => {
+      return this.getUserInfo(req.params.user_id, (error, user) => {
+        if (error != null) {
+          res.send(500)
+        }
+        if (user == null) {
+          return res.send(404)
+        } else {
+          return res.send(JSON.stringify(user))
+        }
+      })
+    })
+
+    app.get('/project/:project_id/details', (req, res, next) => {
+      return this.getProjectDetails(req.params.project_id, (error, project) => {
+        if (error != null) {
+          res.send(500)
+        }
+        if (project == null) {
+          return res.send(404)
+        } else {
+          return res.send(JSON.stringify(project))
+        }
+      })
+    })
+
+    return app
+      .listen(3000, error => {
+        if (error != null) {
+          throw error
+        }
+      })
+      .on('error', error => {
+        console.error('error starting MockWebApiServer:', error.message)
+        return process.exit(1)
+      })
+  }
+}
+
+MockWebApi.run()

services/track-changes/test/acceptance/js/helpers/TrackChangesApp.js

@@ -0,0 +1,66 @@
+/* eslint-disable
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const app = require('../../../../app')
+require('logger-sharelatex')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+
+module.exports = {
+  running: false,
+  initing: false,
+  callbacks: [],
+  ensureRunning(callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (this.running) {
+      return callback()
+    } else if (this.initing) {
+      return this.callbacks.push(callback)
+    } else {
+      this.initing = true
+      this.callbacks.push(callback)
+      return app.listen(
+        __guard__(
+          Settings.internal != null
+            ? Settings.internal.trackchanges
+            : undefined,
+          x => x.port
+        ),
+        'localhost',
+        error => {
+          if (error != null) {
+            throw error
+          }
+          this.running = true
+          logger.log('track changes running in dev mode')
+
+          return (() => {
+            const result = []
+            for (callback of Array.from(this.callbacks)) {
+              result.push(callback())
+            }
+            return result
+          })()
+        }
+      )
+    }
+  }
+}
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/track-changes/test/acceptance/js/helpers/TrackChangesClient.js

@@ -0,0 +1,288 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let TrackChangesClient
+const async = require('async')
+const zlib = require('zlib')
+const request = require('request')
+const Settings = require('settings-sharelatex')
+const rclient = require('redis-sharelatex').createClient(Settings.redis.history) // Only works locally for now
+const Keys = Settings.redis.history.key_schema
+const { db, ObjectId } = require('../../../../app/js/mongojs')
+
+const aws = require('aws-sdk')
+const s3 = new aws.S3({
+  accessKeyId: Settings.trackchanges.s3.key,
+  secretAccessKey: Settings.trackchanges.s3.secret,
+  endpoint: Settings.trackchanges.s3.endpoint,
+  s3ForcePathStyle: Settings.trackchanges.s3.pathStyle
+})
+const S3_BUCKET = Settings.trackchanges.stores.doc_history
+
+module.exports = TrackChangesClient = {
+  flushAndGetCompressedUpdates(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return TrackChangesClient.flushDoc(project_id, doc_id, error => {
+      if (error != null) {
+        return callback(error)
+      }
+      return TrackChangesClient.getCompressedUpdates(doc_id, callback)
+    })
+  },
+
+  flushDoc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/flush`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(204)
+        return callback(error)
+      }
+    )
+  },
+
+  flushProject(project_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:3015/project/${project_id}/flush`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(204)
+        return callback(error)
+      }
+    )
+  },
+
+  getCompressedUpdates(doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return db.docHistory
+      .find({ doc_id: ObjectId(doc_id) })
+      .sort({ 'meta.end_ts': 1 })
+      .toArray(callback)
+  },
+
+  getProjectMetaData(project_id, callback) {
+    if (callback == null) {
+      callback = function(error, updates) {}
+    }
+    return db.projectHistoryMetaData.find(
+      {
+        project_id: ObjectId(project_id)
+      },
+      (error, projects) => callback(error, projects[0])
+    )
+  },
+
+  setPreserveHistoryForProject(project_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return db.projectHistoryMetaData.update(
+      {
+        project_id: ObjectId(project_id)
+      },
+      {
+        $set: { preserveHistory: true }
+      },
+      {
+        upsert: true
+      },
+      callback
+    )
+  },
+
+  pushRawUpdates(project_id, doc_id, updates, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return rclient.sadd(
+      Keys.docsWithHistoryOps({ project_id }),
+      doc_id,
+      error => {
+        if (error != null) {
+          return callback(error)
+        }
+        return rclient.rpush(
+          Keys.uncompressedHistoryOps({ doc_id }),
+          ...Array.from(Array.from(updates).map(u => JSON.stringify(u))),
+          callback
+        )
+      }
+    )
+  },
+
+  getDiff(project_id, doc_id, from, to, callback) {
+    if (callback == null) {
+      callback = function(error, diff) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/diff?from=${from}&to=${to}`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(200)
+        return callback(null, JSON.parse(body))
+      }
+    )
+  },
+
+  getUpdates(project_id, options, callback) {
+    if (callback == null) {
+      callback = function(error, body) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:3015/project/${project_id}/updates?before=${options.before}&min_count=${options.min_count}`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(200)
+        return callback(null, JSON.parse(body))
+      }
+    )
+  },
+
+  restoreDoc(project_id, doc_id, version, user_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/version/${version}/restore`,
+        headers: {
+          'X-User-Id': user_id
+        }
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(204)
+        return callback(null)
+      }
+    )
+  },
+
+  pushDocHistory(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/push`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(204)
+        return callback(error)
+      }
+    )
+  },
+
+  pullDocHistory(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return request.post(
+      {
+        url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/pull`
+      },
+      (error, response, body) => {
+        response.statusCode.should.equal(204)
+        return callback(error)
+      }
+    )
+  },
+
+  waitForS3(done, retries) {
+    if (retries == null) {
+      retries = 42
+    }
+    if (!Settings.trackchanges.s3.endpoint) {
+      return done()
+    }
+
+    return request.get(`${Settings.trackchanges.s3.endpoint}/`, (err, res) => {
+      if (res && res.statusCode < 500) {
+        return done()
+      }
+
+      if (retries === 0) {
+        return done(err || new Error(`s3 returned ${res.statusCode}`))
+      }
+
+      return setTimeout(
+        () => TrackChangesClient.waitForS3(done, --retries),
+        1000
+      )
+    })
+  },
+
+  getS3Doc(project_id, doc_id, pack_id, callback) {
+    if (callback == null) {
+      callback = function(error, body) {}
+    }
+    const params = {
+      Bucket: S3_BUCKET,
+      Key: `${project_id}/changes-${doc_id}/pack-${pack_id}`
+    }
+
+    return s3.getObject(params, (error, data) => {
+      if (error != null) {
+        return callback(error)
+      }
+      const body = data.Body
+      if (body == null) {
+        return callback(new Error('empty response from s3'))
+      }
+      return zlib.gunzip(body, (err, result) => {
+        if (err != null) {
+          return callback(err)
+        }
+        return callback(null, JSON.parse(result.toString()))
+      })
+    })
+  },
+
+  removeS3Doc(project_id, doc_id, callback) {
+    if (callback == null) {
+      callback = function(error, res, body) {}
+    }
+    let params = {
+      Bucket: S3_BUCKET,
+      Prefix: `${project_id}/changes-${doc_id}`
+    }
+
+    return s3.listObjects(params, (error, data) => {
+      if (error != null) {
+        return callback(error)
+      }
+
+      params = {
+        Bucket: S3_BUCKET,
+        Delete: {
+          Objects: data.Contents.map(s3object => ({ Key: s3object.Key }))
+        }
+      }
+
+      return s3.deleteObjects(params, callback)
+    })
+  }
+}

services/track-changes/test/unit/coffee/DiffGenerator/DiffGeneratorTests.coffee

@@ -1,344 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/DiffGenerator.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "DiffGenerator", ->
-	beforeEach ->
-		@DiffGenerator = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": { warn: sinon.stub() }
-		@ts = Date.now()
-		@user_id = "mock-user-id"
-		@user_id_2 = "mock-user-id-2"
-		@meta = {
-			start_ts: @ts, end_ts: @ts, user_id: @user_id
-		}
-
-	describe "rewindOp", ->
-		describe "rewinding an insert", ->
-			it "should undo the insert", ->
-				content = "hello world"
-				rewoundContent = @DiffGenerator.rewindOp content, { p: 6, i: "wo" }
-				rewoundContent.should.equal "hello rld"
-
-		describe "rewinding a delete", ->
-			it "should undo the delete", ->
-				content = "hello rld"
-				rewoundContent = @DiffGenerator.rewindOp content, { p: 6, d: "wo" }
-				rewoundContent.should.equal "hello world"
-
-		describe "with an inconsistent update", ->
-			it "should throw an error", ->
-				content = "hello world"
-				expect( () =>
-					@DiffGenerator.rewindOp content, { p: 6, i: "foo" }
-				).to.throw(@DiffGenerator.ConsistencyError)
-		
-		describe "with an update which is beyond the length of the content", ->
-			it "should undo the insert as if it were at the end of the content", ->
-				content = "foobar"
-				rewoundContent = @DiffGenerator.rewindOp content, { p: 4, i: "bar" }
-				rewoundContent.should.equal "foo"
-
-	describe "rewindUpdate", ->
-		it "should rewind ops in reverse", ->
-			content = "aaabbbccc"
-			update =
-				op: [{ p: 3, i: "bbb" }, { p: 6, i: "ccc" }]
-			rewoundContent = @DiffGenerator.rewindUpdate content, update
-			rewoundContent.should.equal "aaa"
-
-	describe "rewindUpdates", ->
-		it "should rewind updates in reverse", ->
-			content = "aaabbbccc"
-			updates = [
-				{ op: [{ p: 3, i: "bbb" }] },
-				{ op: [{ p: 6, i: "ccc" }] }
-			]
-			rewoundContent = @DiffGenerator.rewindUpdates content, updates
-			rewoundContent.should.equal "aaa"
-
-	describe "buildDiff", ->
-		beforeEach ->
-			@diff = [ u: "mock-diff" ]
-			@content = "Hello world"
-			@updates = [
-				{ i: "mock-update-1" }
-				{ i: "mock-update-2" }
-				{ i: "mock-update-3" }
-			]
-			@DiffGenerator.applyUpdateToDiff = sinon.stub().returns(@diff)
-			@DiffGenerator.compressDiff = sinon.stub().returns(@diff)
-			@result = @DiffGenerator.buildDiff(@content, @updates)
-
-		it "should return the diff", ->
-			@result.should.deep.equal @diff
-
-		it "should build the content into an initial diff", ->
-			@DiffGenerator.applyUpdateToDiff
-				.calledWith([{
-					u: @content
-				}], @updates[0])
-				.should.equal true
-
-		it "should apply each update", ->
-			for update in @updates
-				@DiffGenerator.applyUpdateToDiff
-					.calledWith(sinon.match.any, update)
-					.should.equal true
-
-		it "should compress the diff", ->
-			@DiffGenerator.compressDiff
-				.calledWith(@diff)
-				.should.equal true
-
-	describe "compressDiff", ->
-		describe "with adjacent inserts with the same user_id", ->
-			it "should create one update with combined meta data and min/max timestamps", ->
-				diff = @DiffGenerator.compressDiff([
-					{ i: "foo", meta: { start_ts: 10, end_ts: 20, user: { id: @user_id } }}
-					{ i: "bar", meta: { start_ts: 5,  end_ts: 15, user: { id: @user_id } }}
-				])
-				expect(diff).to.deep.equal([
-					{ i: "foobar", meta: { start_ts: 5, end_ts: 20, user: { id: @user_id } }}
-				])
-
-		describe "with adjacent inserts with different user_ids", ->
-			it "should leave the inserts unchanged", ->
-				input = [
-					{ i: "foo", meta: { start_ts: 10, end_ts: 20, user: { id: @user_id } }}
-					{ i: "bar", meta: { start_ts: 5,  end_ts: 15, user: { id: @user_id_2 } }}
-				]
-				output = @DiffGenerator.compressDiff(input)
-				expect(output).to.deep.equal(input)
-
-		describe "with adjacent deletes with the same user_id", ->
-			it "should create one update with combined meta data and min/max timestamps", ->
-				diff = @DiffGenerator.compressDiff([
-					{ d: "foo", meta: { start_ts: 10, end_ts: 20, user: { id: @user_id } }}
-					{ d: "bar", meta: { start_ts: 5,  end_ts: 15, user: { id: @user_id } }}
-				])
-				expect(diff).to.deep.equal([
-					{ d: "foobar", meta: { start_ts: 5, end_ts: 20, user: { id: @user_id } }}
-				])
-
-		describe "with adjacent deletes with different user_ids", ->
-			it "should leave the deletes unchanged", ->
-				input = [
-					{ d: "foo", meta: { start_ts: 10, end_ts: 20, user: { id: @user_id } }}
-					{ d: "bar", meta: { start_ts: 5,  end_ts: 15, user: { id: @user_id_2 } }}
-				]
-				output = @DiffGenerator.compressDiff(input)
-				expect(output).to.deep.equal(input)
-
-	describe "applyUpdateToDiff", ->
-		describe "an insert", ->
-			it "should insert into the middle of (u)nchanged text", ->
-				diff = @DiffGenerator.applyUpdateToDiff(
-					[ { u: "foobar" } ],
-					{ op: [{ p: 3, i: "baz" }], meta: @meta }
-				)
-				expect(diff).to.deep.equal([
-					{ u: "foo" }
-					{ i: "baz", meta: @meta }
-					{ u: "bar" }
-				])
-
-			it "should insert into the start of (u)changed text", ->
-				diff = @DiffGenerator.applyUpdateToDiff(
-					[ { u: "foobar" } ],
-					{ op: [{ p: 0, i: "baz" }], meta: @meta }
-				)
-				expect(diff).to.deep.equal([
-					{ i: "baz", meta: @meta }
-					{ u: "foobar" }
-				])
-
-			it "should insert into the end of (u)changed text", ->
-				diff = @DiffGenerator.applyUpdateToDiff(
-					[ { u: "foobar" } ],
-					{ op: [{ p: 6, i: "baz" }], meta: @meta }
-				)
-				expect(diff).to.deep.equal([
-					{ u: "foobar" }
-					{ i: "baz", meta: @meta }
-				])
-
-			it "should insert into the middle of (i)inserted text", ->
-				diff = @DiffGenerator.applyUpdateToDiff(
-					[ { i: "foobar", meta: @meta } ],
-					{ op: [{ p: 3, i: "baz" }], meta: @meta }
-				)
-				expect(diff).to.deep.equal([
-					{ i: "foo", meta: @meta }
-					{ i: "baz", meta: @meta }
-					{ i: "bar", meta: @meta }
-				])
-
-			it "should not count deletes in the running length total", ->
-				diff = @DiffGenerator.applyUpdateToDiff(
-					[
-						{ d: "deleted", meta: @meta }
-						{ u: "foobar" }
-					],
-					{ op: [{ p: 3, i: "baz" }], meta: @meta }
-				)
-				expect(diff).to.deep.equal([
-					{ d: "deleted", meta: @meta }
-					{ u: "foo" }
-					{ i: "baz", meta: @meta }
-					{ u: "bar" }
-				])
-
-		describe "a delete", ->
-			describe "deleting unchanged text", ->
-				it "should delete from the middle of (u)nchanged text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foobazbar" } ],
-						{ op: [{ p: 3, d: "baz" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "foo" }
-						{ d: "baz", meta: @meta }
-						{ u: "bar" }
-					])
-
-				it "should delete from the start of (u)nchanged text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foobazbar" } ],
-						{ op: [{ p: 0, d: "foo" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ d: "foo", meta: @meta }
-						{ u: "bazbar" }
-					])
-
-				it "should delete from the end of (u)nchanged text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foobazbar" } ],
-						{ op: [{ p: 6, d: "bar" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "foobaz" }
-						{ d: "bar", meta: @meta }
-					])
-
-				it "should delete across multiple (u)changed text parts", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foo" }, { u: "baz" }, { u: "bar" } ],
-						{ op: [{ p: 2, d: "obazb" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "fo" }
-						{ d: "o", meta: @meta }
-						{ d: "baz", meta: @meta }
-						{ d: "b", meta: @meta }
-						{ u: "ar" }
-					])
-
-			describe "deleting inserts", ->
-				it "should delete from the middle of (i)nserted text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { i: "foobazbar", meta: @meta } ],
-						{ op: [{ p: 3, d: "baz" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ i: "foo", meta: @meta }
-						{ i: "bar", meta: @meta }
-					])
-
-				it "should delete from the start of (u)nchanged text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { i: "foobazbar", meta: @meta } ],
-						{ op: [{ p: 0, d: "foo" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ i: "bazbar", meta: @meta }
-					])
-
-				it "should delete from the end of (u)nchanged text", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { i: "foobazbar", meta: @meta } ],
-						{ op: [{ p: 6, d: "bar" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ i: "foobaz", meta: @meta }
-					])
-
-				it "should delete across multiple (u)changed and (i)nserted text parts", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foo" }, { i: "baz", meta: @meta }, { u: "bar" } ],
-						{ op: [{ p: 2, d: "obazb" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "fo" }
-						{ d: "o", meta: @meta }
-						{ d: "b", meta: @meta }
-						{ u: "ar" }
-					])
-
-			describe "deleting over existing deletes", ->
-				it "should delete across multiple (u)changed and (d)deleted text parts", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foo" }, { d: "baz", meta: @meta }, { u: "bar" } ],
-						{ op: [{ p: 2, d: "ob" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "fo" }
-						{ d: "o", meta: @meta }
-						{ d: "baz", meta: @meta }
-						{ d: "b", meta: @meta }
-						{ u: "ar" }
-					])
-
-			describe "deleting when the text doesn't match", ->
-				it "should throw an error when deleting from the middle of (u)nchanged text", ->
-					expect(
-						() => @DiffGenerator.applyUpdateToDiff(
-							[ { u: "foobazbar" } ],
-							{ op: [{ p: 3, d: "xxx" }], meta: @meta }
-						)
-					).to.throw(@DiffGenerator.ConsistencyError)
-
-				it "should throw an error when deleting from the start of (u)nchanged text", ->
-					expect(
-						() => @DiffGenerator.applyUpdateToDiff(
-							[ { u: "foobazbar" } ],
-							{ op: [{ p: 0, d: "xxx" }], meta: @meta }
-						)
-					).to.throw(@DiffGenerator.ConsistencyError)
-
-				it "should throw an error when deleting from the end of (u)nchanged text", ->
-					expect(
-						() => @DiffGenerator.applyUpdateToDiff(
-							[ { u: "foobazbar" } ],
-							{ op: [{ p: 6, d: "xxx" }] , meta: @meta }
-						)
-					).to.throw(@DiffGenerator.ConsistencyError)
-
-			describe "when the last update in the existing diff is a delete", ->
-				it "should insert the new update before the delete", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { u: "foo" }, { d: "bar", meta: @meta } ],
-						{ op: [{ p: 3, i: "baz" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ u: "foo" }
-						{ i: "baz", meta: @meta }
-						{ d: "bar", meta: @meta }
-					])
-
-			describe "when the only update in the existing diff is a delete", ->
-				it "should insert the new update after the delete", ->
-					diff = @DiffGenerator.applyUpdateToDiff(
-						[ { d: "bar", meta: @meta } ],
-						{ op: [{ p: 0, i: "baz" }], meta: @meta }
-					)
-					expect(diff).to.deep.equal([
-						{ d: "bar", meta: @meta }
-						{ i: "baz", meta: @meta }
-					])
-
-

services/track-changes/test/unit/coffee/DiffManager/DiffManagerTests.coffee

@@ -1,240 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/DiffManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "DiffManager", ->
-	beforeEach ->
-		@DiffManager = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
-			"./UpdatesManager": @UpdatesManager = {}
-			"./DocumentUpdaterManager": @DocumentUpdaterManager = {}
-			"./DiffGenerator": @DiffGenerator = {}
-		@callback = sinon.stub()
-		@from = new Date()
-		@to = new Date(Date.now() + 10000)
-		@project_id = "mock-project-id"
-		@doc_id = "mock-doc-id"
-
-	describe "getLatestDocAndUpdates", ->
-		beforeEach ->
-			@content = "hello world"
-			@version = 42
-			@updates = [ "mock-update-1", "mock-update-2" ]
-
-			@DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(2, null, @content, @version)
-			@UpdatesManager.getDocUpdatesWithUserInfo = sinon.stub().callsArgWith(3, null, @updates)
-
-		describe "with a fromVersion", ->
-			beforeEach ->
-				@DiffManager.getLatestDocAndUpdates @project_id, @doc_id, @from, @callback
-
-			it "should get the latest version of the doc", ->
-				@DocumentUpdaterManager.getDocument
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should get the latest updates", ->
-				@UpdatesManager.getDocUpdatesWithUserInfo
-					.calledWith(@project_id, @doc_id, from: @from)
-					.should.equal true
-
-			it "should call the callback with the content, version and updates", ->
-				@callback.calledWith(null, @content, @version, @updates).should.equal true
-
-		describe "with no fromVersion", ->
-			beforeEach ->
-				@DiffManager.getLatestDocAndUpdates @project_id, @doc_id, null, @callback
-
-			it "should get the latest version of the doc", ->
-				@DocumentUpdaterManager.getDocument
-					.calledWith(@project_id, @doc_id)
-					.should.equal true
-
-			it "should not get the latest updates", ->
-				@UpdatesManager.getDocUpdatesWithUserInfo
-					.called.should.equal false
-
-			it "should call the callback with the content, version and blank updates", ->
-				@callback.calledWith(null, @content, @version, []).should.equal true
-			
-
-	describe "getDiff", ->
-		beforeEach ->
-			@content = "hello world"
-			# Op versions are the version they were applied to, so doc is always one version
-			# ahead.s
-			@version = 43
-			@updates = [
-				{ op: "mock-4", v: 42, meta: { start_ts: new Date(@to.getTime() + 20)} }
-				{ op: "mock-3", v: 41, meta: { start_ts: new Date(@to.getTime() + 10)} }
-				{ op: "mock-2", v: 40, meta: { start_ts: new Date(@to.getTime() - 10)} }
-				{ op: "mock-1", v: 39, meta: { start_ts: new Date(@to.getTime() - 20)} }
-			]
-			@fromVersion = 39
-			@toVersion = 40
-			@diffed_updates = @updates.slice(2)
-			@rewound_content = "rewound-content"
-			@diff = [ u: "mock-diff" ]
-			
-		describe "with matching versions", ->
-			beforeEach ->
-				@DiffManager.getDocumentBeforeVersion = sinon.stub().callsArgWith(3, null, @rewound_content, @updates)
-				@DiffGenerator.buildDiff = sinon.stub().returns(@diff)
-				@DiffManager.getDiff @project_id, @doc_id, @fromVersion, @toVersion, @callback
-
-			it "should get the latest doc and version with all recent updates", ->
-				@DiffManager.getDocumentBeforeVersion
-					.calledWith(@project_id, @doc_id, @fromVersion)
-					.should.equal true
-
-			it "should generate the diff", ->
-				@DiffGenerator.buildDiff
-					.calledWith(@rewound_content, @diffed_updates.slice().reverse())
-					.should.equal true
-
-			it "should call the callback with the diff", ->
-				@callback.calledWith(null, @diff).should.equal true
-
-		describe "when the updates are inconsistent", ->
-			beforeEach ->
-				@DiffManager.getLatestDocAndUpdates = sinon.stub().callsArgWith(3, null, @content, @version, @updates)
-				@DiffGenerator.buildDiff = sinon.stub().throws(@error = new Error("inconsistent!"))
-				@DiffManager.getDiff @project_id, @doc_id, @fromVersion, @toVersion, @callback
-
-			it "should call the callback with an error", ->
-				@callback
-					.calledWith(@error)
-					.should.equal true
-
-	describe "getDocumentBeforeVersion", ->
-		beforeEach ->
-			@DiffManager._tryGetDocumentBeforeVersion = sinon.stub()
-			@document = "mock-documents"
-			@rewound_updates = "mock-rewound-updates"
-
-		describe "succesfully", ->
-			beforeEach ->
-				@DiffManager._tryGetDocumentBeforeVersion.yields(null, @document, @rewound_updates)
-				@DiffManager.getDocumentBeforeVersion @project_id, @doc_id, @version, @callback
-			
-			it "should call _tryGetDocumentBeforeVersion", ->
-				@DiffManager._tryGetDocumentBeforeVersion
-					.calledWith(@project_id, @doc_id, @version)
-					.should.equal true
-			
-			it "should call the callback with the response", ->
-				@callback.calledWith(null, @document, @rewound_updates).should.equal true
-		
-		describe "with a retry needed", ->
-			beforeEach ->
-				retried = false
-				@DiffManager._tryGetDocumentBeforeVersion = (project_id, doc_id, version, callback) =>
-					if !retried
-						retried = true
-						error = new Error()
-						error.retry = true
-						callback error
-					else
-						callback(null, @document, @rewound_updates)
-				sinon.spy @DiffManager, "_tryGetDocumentBeforeVersion"
-				@DiffManager.getDocumentBeforeVersion @project_id, @doc_id, @version, @callback
-			
-			it "should call _tryGetDocumentBeforeVersion twice", ->
-				@DiffManager._tryGetDocumentBeforeVersion
-					.calledTwice
-					.should.equal true
-			
-			it "should call the callback with the response", ->
-				@callback.calledWith(null, @document, @rewound_updates).should.equal true
-		
-		describe "with a non-retriable error", ->
-			beforeEach ->
-				@error = new Error("oops")
-				@DiffManager._tryGetDocumentBeforeVersion.yields(@error)
-				@DiffManager.getDocumentBeforeVersion @project_id, @doc_id, @version, @callback
-			
-			it "should call _tryGetDocumentBeforeVersion once", ->
-				@DiffManager._tryGetDocumentBeforeVersion
-					.calledOnce
-					.should.equal true
-			
-			it "should call the callback with the error", ->
-				@callback.calledWith(@error).should.equal true
-		
-		describe "when retry limit is matched", ->
-			beforeEach ->
-				@error = new Error("oops")
-				@error.retry = true
-				@DiffManager._tryGetDocumentBeforeVersion.yields(@error)
-				@DiffManager.getDocumentBeforeVersion @project_id, @doc_id, @version, @callback
-			
-			it "should call _tryGetDocumentBeforeVersion three times (max retries)", ->
-				@DiffManager._tryGetDocumentBeforeVersion
-					.calledThrice
-					.should.equal true
-			
-			it "should call the callback with the error", ->
-				@callback.calledWith(@error).should.equal true
-
-	describe "_tryGetDocumentBeforeVersion", ->
-		beforeEach ->
-			@content = "hello world"
-			# Op versions are the version they were applied to, so doc is always one version
-			# ahead.s
-			@version = 43
-			@updates = [
-				{ op: "mock-4", v: 42, meta: { start_ts: new Date(@to.getTime() + 20)} }
-				{ op: "mock-3", v: 41, meta: { start_ts: new Date(@to.getTime() + 10)} }
-				{ op: "mock-2", v: 40, meta: { start_ts: new Date(@to.getTime() - 10)} }
-				{ op: "mock-1", v: 39, meta: { start_ts: new Date(@to.getTime() - 20)} }
-			]
-			@fromVersion = 39
-			@rewound_content = "rewound-content"
-			@diff = [ u: "mock-diff" ]
-			
-		describe "with matching versions", ->
-			beforeEach ->
-				@DiffManager.getLatestDocAndUpdates = sinon.stub().callsArgWith(3, null, @content, @version, @updates)
-				@DiffGenerator.rewindUpdates = sinon.spy (content, updates) =>
-					# the rewindUpdates method reverses the 'updates' array
-					updates.reverse()
-					return @rewound_content
-				@rewindUpdatesWithArgs = @DiffGenerator.rewindUpdates.withArgs(@content, @updates.slice().reverse())
-				@DiffManager._tryGetDocumentBeforeVersion @project_id, @doc_id, @fromVersion, @callback
-
-			it "should get the latest doc and version with all recent updates", ->
-				@DiffManager.getLatestDocAndUpdates
-					.calledWith(@project_id, @doc_id, @fromVersion)
-					.should.equal true
-
-			it "should rewind the diff", ->
-				sinon.assert.calledOnce(@rewindUpdatesWithArgs)
-
-			it "should call the callback with the rewound document and updates", ->
-				@callback.calledWith(null, @rewound_content, @updates).should.equal true
-
-		describe "with mismatching versions", ->
-			beforeEach ->
-				@version = 50
-				@updates = [ { op: "mock-1", v: 40 }, { op: "mock-1", v: 39 } ]
-				@DiffManager.getLatestDocAndUpdates = sinon.stub().callsArgWith(3, null, @content, @version, @updates)
-				@DiffManager._tryGetDocumentBeforeVersion @project_id, @doc_id, @fromVersion, @callback
-
-			it "should call the callback with an error with retry = true set", ->
-				@callback.calledOnce.should.equal true
-				error = @callback.args[0][0]
-				expect(error.retry).to.equal true
-
-		describe "when the updates are inconsistent", ->
-			beforeEach ->
-				@DiffManager.getLatestDocAndUpdates = sinon.stub().callsArgWith(3, null, @content, @version, @updates)
-				@DiffGenerator.rewindUpdates = sinon.stub().throws(@error = new Error("inconsistent!"))
-				@DiffManager.getDocumentBeforeVersion @project_id, @doc_id, @fromVersion, @callback
-
-			it "should call the callback with an error", ->
-				@callback
-					.calledWith(@error)
-					.should.equal true

services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee

@@ -1,70 +0,0 @@
-chai = require('chai')
-chai.should()
-sinon = require("sinon")
-modulePath = "../../../../app/js/MongoAWS.js"
-SandboxedModule = require('sandboxed-module')
-{ObjectId} = require("mongojs")
-MemoryStream = require('memorystream')
-zlib = require "zlib"
-
-describe "MongoAWS", ->
-	beforeEach ->
-		@MongoAWS = SandboxedModule.require modulePath, requires:
-			"settings-sharelatex": @settings =
-				trackchanges:
-					s3:
-						secret: "s3-secret"
-						key: "s3-key"
-					stores:
-						doc_history: "s3-bucket"
-			"child_process": @child_process = {}
-			"mongo-uri": @mongouri = {}
-			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->}
-			"aws-sdk": @awssdk = {}
-			"fs": @fs = {}
-			"s3-streams": @S3S = {}
-			"./mongojs" : { db: @db = {}, ObjectId: ObjectId }
-			"JSONStream": @JSONStream = {}
-			"readline-stream": @readline = sinon.stub()
-			'metrics-sharelatex': {inc: ()->}
-
-		@project_id = ObjectId().toString()
-		@doc_id = ObjectId().toString()
-		@pack_id = ObjectId()
-		@update = { v:123 }
-		@callback = sinon.stub()
-
-	describe "archivePack", ->
-
-		beforeEach (done) ->
-			@awssdk.config = { update: sinon.stub() }
-			@awssdk.S3 = sinon.stub()
-			@S3S.WriteStream = () ->
-				MemoryStream.createWriteStream()
-			@db.docHistory = {}
-			@db.docHistory.findOne = sinon.stub().callsArgWith(1, null, {"pack":"hello"})
-
-			@MongoAWS.archivePack @project_id, @doc_id, @pack_id, (err, result) =>
-				@callback()
-				done()
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-
-	describe "unArchivePack", ->
-
-		beforeEach (done) ->
-			zlib.gzip '{"pack":"123"}', (err, zbuf) =>
-				@awssdk.config = { update: sinon.stub() }
-				@awssdk.S3 = sinon.stub()
-				@S3S.ReadStream = () ->
-					MemoryStream.createReadStream(zbuf, {readable:true})
-				@db.docHistory = {}
-				@db.docHistory.insert = sinon.stub().callsArgWith(1, null, "pack")
-
-				@MongoAWS.unArchivePack @project_id, @doc_id, @pack_id, (err, result) =>
-					@callback()
-					done()
-
-		it "should call db.docHistory.insert", ->
-			@db.docHistory.insert.called.should.equal true

services/track-changes/test/unit/coffee/DocumentUpdaterManager/DocumentUpdaterManagerTests.coffee

@@ -1,95 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/DocumentUpdaterManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "DocumentUpdaterManager", ->
-	beforeEach ->
-		@DocumentUpdaterManager = SandboxedModule.require modulePath, requires:
-			"request": @request = {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
-			'settings-sharelatex': @settings =
-				apis : documentupdater: url : "http://example.com"
-		@callback = sinon.stub()
-		@lines = ["one", "two", "three"]
-		@version = 42
-
-	describe "getDocument", ->
-		describe "successfully", ->
-			beforeEach ->
-				@body = JSON.stringify
-					lines: @lines
-					version: @version
-					ops: []
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @callback
-
-			it 'should get the document from the document updater', ->
-				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}"
-				@request.get.calledWith(url).should.equal true
-
-			it "should call the callback with the content and version", ->
-				@callback.calledWith(null, @lines.join("\n"), @version).should.equal true
-
-		describe "when the document updater API returns an error", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the document updater returns a failure error code", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "")
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @callback
-
-			it "should return the callback with an error", ->
-				@callback
-					.calledWith(sinon.match.has('message', "doc updater returned a non-success status code: 500"))
-					.should.equal true
-
-	describe "setDocument", ->
-		beforeEach ->
-			@content = "mock content"
-			@user_id = "user-id-123"
-
-		describe "successfully", ->
-			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200})
-				@DocumentUpdaterManager.setDocument @project_id, @doc_id, @content, @user_id, @callback
-
-			it 'should set the document in the document updater', ->
-				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}"
-				@request.post
-					.calledWith({
-						url: url
-						json:
-							lines: @content.split("\n")
-							source: "restore"
-							user_id: @user_id
-							undoing: true
-					}).should.equal true
-
-			it "should call the callback", ->
-				@callback.calledWith(null).should.equal true
-
-		describe "when the document updater API returns an error", ->
-			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@DocumentUpdaterManager.setDocument @project_id, @doc_id, @content, @user_id, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the document updater returns a failure error code", ->
-			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "")
-				@DocumentUpdaterManager.setDocument @project_id, @doc_id, @content, @user_id, @callback
-
-			it "should return the callback with an error", ->
-				@callback
-					.calledWith(sinon.match.has('message', "doc updater returned a non-success status code: 500"))
-					.should.equal true

services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee

@@ -1,135 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/HttpController.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "HttpController", ->
-	beforeEach ->
-		@HttpController = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": { log: sinon.stub() }
-			"./UpdatesManager": @UpdatesManager = {}
-			"./DiffManager": @DiffManager = {}
-			"./RestoreManager": @RestoreManager = {}
-			"./PackManager": @PackManager = {}
-			"./DocArchiveManager": @DocArchiveManager = {}
-			"./HealthChecker": @HealthChecker = {}
-		@doc_id = "doc-id-123"
-		@project_id = "project-id-123"
-		@next = sinon.stub()
-		@user_id = "mock-user-123"
-		@now = Date.now()
-
-	describe "flushDoc", ->
-		beforeEach ->
-			@req =
-				params:
-					doc_id: @doc_id
-					project_id: @project_id
-			@res =
-				send: sinon.stub()
-			@UpdatesManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(2)
-			@HttpController.flushDoc @req, @res, @next
-
-		it "should process the updates", ->
-			@UpdatesManager.processUncompressedUpdatesWithLock
-				.calledWith(@project_id, @doc_id)
-				.should.equal true
-
-		it "should return a success code", ->
-			@res.send.calledWith(204).should.equal true
-
-	describe "flushProject", ->
-		beforeEach ->
-			@req =
-				params:
-					project_id: @project_id
-			@res =
-				send: sinon.stub()
-			@UpdatesManager.processUncompressedUpdatesForProject = sinon.stub().callsArg(1)
-			@HttpController.flushProject @req, @res, @next
-
-		it "should process the updates", ->
-			@UpdatesManager.processUncompressedUpdatesForProject
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should return a success code", ->
-			@res.send.calledWith(204).should.equal true
-
-
-	describe "getDiff", ->
-		beforeEach ->
-			@from = 42
-			@to = 45
-			@req =
-				params:
-					doc_id: @doc_id
-					project_id: @project_id
-				query:
-					from: @from.toString()
-					to: @to.toString()
-			@res =
-				json: sinon.stub()
-			@diff = [ u: "mock-diff" ]
-			@DiffManager.getDiff = sinon.stub().callsArgWith(4, null, @diff)
-			@HttpController.getDiff @req, @res, @next
-
-		it "should get the diff", ->
-			@DiffManager.getDiff
-				.calledWith(@project_id, @doc_id, parseInt(@from, 10), parseInt(@to, 10))
-				.should.equal true
-
-		it "should return the diff", ->
-			@res.json.calledWith({diff: @diff}).should.equal true
-
-	describe "getUpdates", ->
-		beforeEach ->
-			@before = Date.now()
-			@nextBeforeTimestamp = @before - 100
-			@min_count = 10
-			@req =
-				params:
-					project_id: @project_id
-				query:
-					before:    @before.toString()
-					min_count: @min_count.toString()
-			@res =
-				json: sinon.stub()
-			@updates = ["mock-summarized-updates"]
-			@UpdatesManager.getSummarizedProjectUpdates = sinon.stub().callsArgWith(2, null, @updates, @nextBeforeTimestamp)
-			@HttpController.getUpdates @req, @res, @next
-
-		it "should get the updates", ->
-			@UpdatesManager.getSummarizedProjectUpdates
-				.calledWith(@project_id, before: @before, min_count: @min_count)
-				.should.equal true
-
-		it "should return the formatted updates", ->
-			@res.json.calledWith({updates: @updates, nextBeforeTimestamp: @nextBeforeTimestamp}).should.equal true
-
-	describe "RestoreManager", ->
-		beforeEach ->
-			@version = "42"
-			@req =
-				params:
-					doc_id: @doc_id
-					project_id: @project_id
-					version: @version
-				headers:
-					"x-user-id": @user_id
-			@res =
-				send: sinon.stub()
-
-			@RestoreManager.restoreToBeforeVersion = sinon.stub().callsArg(4)
-			@HttpController.restore @req, @res, @next
-
-		it "should restore the document", ->
-			@RestoreManager.restoreToBeforeVersion
-				.calledWith(@project_id, @doc_id, parseInt(@version, 10), @user_id)
-				.should.equal true
-
-		it "should return a success code", ->
-			@res.send.calledWith(204).should.equal true
-

services/track-changes/test/unit/coffee/LockManager/LockManagerTests.coffee

@@ -1,198 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/LockManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "LockManager", ->
-	beforeEach ->
-		@Settings = 		
-			redis:
-				lock:{}
-		@LockManager = SandboxedModule.require modulePath, requires:
-			"redis-sharelatex":
-				createClient: () => @rclient =
-					auth: sinon.stub()
-			"settings-sharelatex": @Settings
-			"logger-sharelatex": {error: ->}
-
-		@key = "lock-key"
-		@callback = sinon.stub()
-
-	describe "checkLock", ->
-		describe "when the lock is taken", ->
-			beforeEach ->
-				@rclient.exists = sinon.stub().callsArgWith(1, null, "1")
-				@LockManager.checkLock @key, @callback
-
-			it "should check the lock in redis", ->
-				@rclient.exists
-					.calledWith(@key)
-					.should.equal true
-
-			it "should return the callback with false", ->
-				@callback.calledWith(null, false).should.equal true
-
-		describe "when the lock is free", ->
-			beforeEach ->
-				@rclient.exists = sinon.stub().callsArgWith(1, null, "0")
-				@LockManager.checkLock @key, @callback
-
-			it "should return the callback with true", ->
-				@callback.calledWith(null, true).should.equal true
-
-
-	describe "tryLock", ->
-		describe "when the lock is taken", ->
-			beforeEach ->
-				@rclient.set = sinon.stub().callsArgWith(5, null, null)
-				@LockManager.randomLock = sinon.stub().returns "locked-random-value"
-				@LockManager.tryLock @key, @callback
-
-			it "should check the lock in redis", ->
-				@rclient.set
-					.calledWith(@key, "locked-random-value", "EX", @LockManager.LOCK_TTL, "NX")
-					.should.equal true
-
-			it "should return the callback with false", ->
-				@callback.calledWith(null, false).should.equal true
-
-		describe "when the lock is free", ->
-			beforeEach ->
-				@rclient.set = sinon.stub().callsArgWith(5, null, "OK")
-				@LockManager.tryLock @key, @callback
-
-			it "should return the callback with true", ->
-				@callback.calledWith(null, true).should.equal true
-
-	describe "deleteLock", ->
-		beforeEach -> 
-			beforeEach ->
-				@rclient.del = sinon.stub().callsArg(1)
-				@LockManager.deleteLock @key, @callback
-
-			it "should delete the lock in redis", ->
-				@rclient.del
-					.calledWith(key)
-					.should.equal true
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-	describe "getLock", ->
-		describe "when the lock is not taken", ->
-			beforeEach (done) ->
-				@LockManager.tryLock = sinon.stub().callsArgWith(1, null, true)
-				@LockManager.getLock @key, (args...) =>
-					@callback(args...)
-					done()
-
-			it "should try to get the lock", ->
-				@LockManager.tryLock
-					.calledWith(@key)
-					.should.equal true
-
-			it "should only need to try once", ->
-				@LockManager.tryLock.callCount.should.equal 1
-
-			it "should return the callback", ->
-				@callback.calledWith(null).should.equal true
-
-		describe "when the lock is initially set", ->
-			beforeEach (done) ->
-				startTime = Date.now()
-				@LockManager.LOCK_TEST_INTERVAL = 5
-				@LockManager.tryLock = (doc_id, callback = (error, isFree) ->) ->
-					if Date.now() - startTime < 100
-						callback null, false
-					else
-						callback null, true
-				sinon.spy @LockManager, "tryLock"
-
-				@LockManager.getLock @key, (args...) =>
-					@callback(args...)
-					done()
-
-			it "should call tryLock multiple times until free", ->
-				(@LockManager.tryLock.callCount > 1).should.equal true
-
-			it "should return the callback", ->
-				@callback.calledWith(null).should.equal true
-
-		describe "when the lock times out", ->
-			beforeEach (done) ->
-				time = Date.now()
-				@LockManager.MAX_LOCK_WAIT_TIME = 5
-				@LockManager.tryLock = sinon.stub().callsArgWith(1, null, false)
-				@LockManager.getLock @key, (args...) =>
-					@callback(args...)
-					done()
-
-			it "should return the callback with an error", ->
-				@callback.calledWith(sinon.match.instanceOf(Error)).should.equal true
-
-	describe "runWithLock", ->
-		describe "with successful run", ->
-			beforeEach ->
-				@runner = (releaseLock = (error) ->) ->
-					releaseLock()
-				sinon.spy @, "runner"
-				@LockManager.getLock = sinon.stub().callsArg(1)
-				@LockManager.releaseLock = sinon.stub().callsArg(2)
-				@LockManager.runWithLock @key, @runner, @callback
-
-			it "should get the lock", ->
-				@LockManager.getLock
-					.calledWith(@key)
-					.should.equal true
-
-			it "should run the passed function", ->
-				@runner.called.should.equal true
-
-			it "should release the lock", ->
-				@LockManager.releaseLock
-					.calledWith(@key)
-					.should.equal true
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-		describe "when the runner function returns an error", ->
-			beforeEach ->
-				@error = new Error("oops")
-				@runner = (releaseLock = (error) ->) =>
-					releaseLock(@error)
-				sinon.spy @, "runner"
-				@LockManager.getLock = sinon.stub().callsArg(1)
-				@LockManager.releaseLock = sinon.stub().callsArg(2)
-				@LockManager.runWithLock @key, @runner, @callback
-
-			it "should release the lock", ->
-				@LockManager.releaseLock
-					.calledWith(@key)
-					.should.equal true
-
-			it "should call the callback with the error", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "releaseLock", ->
-			describe "when the lock is current", ->
-				beforeEach ->
-					@rclient.eval = sinon.stub().yields(null, 1)
-					@LockManager.releaseLock @key, @lockValue, @callback
-
-				it 'should clear the data from redis', ->
-					@rclient.eval.calledWith(@LockManager.unlockScript, 1, @key, @lockValue).should.equal true
-
-				it 'should call the callback', ->
-					@callback.called.should.equal true
-
-			describe "when the lock has expired", ->
-				beforeEach ->
-					@rclient.eval = sinon.stub().yields(null, 0)
-					@LockManager.releaseLock @key, @lockValue, @callback
-
-				it 'should return an error if the lock has expired', ->
-					@callback.calledWith(sinon.match.has('message', "tried to release timed out lock")).should.equal true
-

services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee

@@ -1,158 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/MongoManager.js"
-packModulePath = "../../../../app/js/PackManager.js"
-SandboxedModule = require('sandboxed-module')
-{ObjectId} = require("mongojs")
-tk = require "timekeeper"
-
-describe "MongoManager", ->
-	beforeEach ->
-		tk.freeze(new Date())
-		@MongoManager = SandboxedModule.require modulePath, requires:
-			"./mongojs" : { db: @db = {}, ObjectId: ObjectId }
-			"./PackManager" : @PackManager = {}
-			'metrics-sharelatex': {timeAsyncMethod: ()->}
-			'logger-sharelatex': {log: ()->}
-		@callback = sinon.stub()
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-
-	afterEach ->
-		tk.reset()
-
-	describe "getLastCompressedUpdate", ->
-		beforeEach ->
-			@update = "mock-update"
-			@db.docHistory = {}
-			@db.docHistory.find = sinon.stub().returns @db.docHistory
-			@db.docHistory.findOne = sinon.stub().returns @db.docHistory
-			@db.docHistory.sort = sinon.stub().returns @db.docHistory
-			@db.docHistory.limit = sinon.stub().returns @db.docHistory
-			@db.docHistory.toArray = sinon.stub().callsArgWith(0, null, [@update])
-
-			@MongoManager.getLastCompressedUpdate @doc_id, @callback
-
-		it "should find the updates for the doc", ->
-			@db.docHistory.find
-				.calledWith(doc_id: ObjectId(@doc_id))
-				.should.equal true
-
-		it "should limit to one result", ->
-			@db.docHistory.limit
-				.calledWith(1)
-				.should.equal true
-
-		it "should sort in descending version order", ->
-			@db.docHistory.sort
-				.calledWith(v: -1)
-				.should.equal true
-
-		it "should call the call back with the update", ->
-			@callback.calledWith(null, @update).should.equal true
-
-
-	describe "peekLastCompressedUpdate", ->
-		describe "when there is no last update", ->
-			beforeEach ->
-				@PackManager.getLastPackFromIndex =  sinon.stub().callsArgWith(1, null, null)
-				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
-				@MongoManager.peekLastCompressedUpdate @doc_id, @callback
-
-			it "should get the last update", ->
-				@MongoManager.getLastCompressedUpdate
-					.calledWith(@doc_id)
-					.should.equal true
-
-			it "should call the callback with no update", ->
-				@callback.calledWith(null, null).should.equal true
-
-		describe "when there is an update", ->
-			beforeEach ->
-				@update = { _id: Object() }
-				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
-				@MongoManager.peekLastCompressedUpdate @doc_id, @callback
-
-			it "should get the last update", ->
-				@MongoManager.getLastCompressedUpdate
-					.calledWith(@doc_id)
-					.should.equal true
-
-			it "should call the callback with the update", ->
-				@callback.calledWith(null, @update).should.equal true
-
-		describe "when there is a last update in S3", ->
-			beforeEach ->
-				@update = { _id: Object(), v: 12345, v_end: 12345, inS3:true}
-				@PackManager.getLastPackFromIndex =  sinon.stub().callsArgWith(1, null, @update)
-				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null)
-				@MongoManager.peekLastCompressedUpdate @doc_id, @callback
-
-			it "should get the last update", ->
-				@MongoManager.getLastCompressedUpdate
-					.calledWith(@doc_id)
-					.should.equal true
-
-			it "should call the callback with a null update and the correct version", ->
-				@callback.calledWith(null, null, @update.v_end).should.equal true
-
-
-	describe "backportProjectId", ->
-		beforeEach ->
-			@db.docHistory =
-				update: sinon.stub().callsArg(3)
-			@MongoManager.backportProjectId @project_id, @doc_id, @callback
-
-		it "should insert the project_id into all entries for the doc_id which don't have it set", ->
-			@db.docHistory.update
-				.calledWith({
-					doc_id: ObjectId(@doc_id)
-					project_id: { $exists: false }
-				}, {
-					$set: { project_id: ObjectId(@project_id) }
-				}, {
-					multi: true
-				})
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-
-	describe "getProjectMetaData", ->
-		beforeEach ->
-			@metadata = { "mock": "metadata" }
-			@db.projectHistoryMetaData =
-				find: sinon.stub().callsArgWith(1, null, [@metadata])
-			@MongoManager.getProjectMetaData @project_id, @callback
-
-		it "should look up the meta data in the db", ->
-			@db.projectHistoryMetaData.find
-				.calledWith({ project_id: ObjectId(@project_id) })
-				.should.equal true
-
-		it "should return the metadata", ->
-			@callback.calledWith(null, @metadata).should.equal true
-
-	describe "setProjectMetaData", ->
-		beforeEach ->
-			@metadata = { "mock": "metadata" }
-			@db.projectHistoryMetaData =
-				update: sinon.stub().callsArgWith(3, null, [@metadata])
-			@MongoManager.setProjectMetaData @project_id, @metadata, @callback
-
-		it "should upsert the metadata into the DB", ->
-			@db.projectHistoryMetaData.update
-				.calledWith({
-					project_id: ObjectId(@project_id)
-				}, {
-					$set: @metadata
-				}, {
-					upsert: true
-				})
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-

services/track-changes/test/unit/coffee/PackManager/PackManagerTests.coffee

@@ -1,369 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-assert = require('chai').assert
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/PackManager.js"
-SandboxedModule = require('sandboxed-module')
-{ObjectId} = require("mongojs")
-bson = require("bson")
-BSON = new bson.BSONPure()
-_ = require("underscore")
-
-tk = require "timekeeper"
-
-describe "PackManager", ->
-	beforeEach ->
-		tk.freeze(new Date())
-		@PackManager = SandboxedModule.require modulePath, requires:
-			"./mongojs" : { db: @db = {}, ObjectId: ObjectId, BSON: BSON }
-			"./LockManager" : {}
-			"./MongoAWS": {}
-			"logger-sharelatex": { log: sinon.stub(), error: sinon.stub() }
-			'metrics-sharelatex': {inc: ()->}
-			"./ProjectIterator": require("../../../../app/js/ProjectIterator.js") # Cache for speed
-			"settings-sharelatex":
-				redis: lock: key_schema: {}
-		@callback = sinon.stub()
-		@doc_id = ObjectId().toString()
-		@project_id = ObjectId().toString()
-		@PackManager.MAX_COUNT = 512
-
-	afterEach ->
-		tk.reset()
-
-	describe "insertCompressedUpdates", ->
-		beforeEach ->
-			@lastUpdate = {
-				_id: "12345"
-				pack: [
-					{ op: "op-1", meta: "meta-1", v: 1},
-					{ op: "op-2", meta: "meta-2", v: 2}
-				]
-				n : 2
-				sz : 100
-			}
-			@newUpdates = [
-				{ op: "op-3", meta: "meta-3", v: 3},
-				{ op: "op-4", meta: "meta-4", v: 4}
-			]
-			@db.docHistory =
-				save: sinon.stub().callsArg(1)
-				insert: sinon.stub().callsArg(1)
-				findAndModify: sinon.stub().callsArg(1)
-
-		describe "with no last update", ->
-			beforeEach ->
-				@PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
-				@PackManager.insertCompressedUpdates @project_id, @doc_id, null, @newUpdates, true, @callback
-
-			describe "for a small update", ->
-				it "should insert the update into a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates, true).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-			describe "for many small updates", ->
-				beforeEach ->
-					@newUpdates = ({ op: "op-#{i}", meta: "meta-#{i}", v: i} for i in [0..2048])
-					@PackManager.insertCompressedUpdates @project_id, @doc_id, null, @newUpdates, false, @callback
-
-				it "should append the initial updates to the existing pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[0...512], false).should.equal true
-
-				it "should insert the first set remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[512...1024], false).should.equal true
-
-				it "should insert the second set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1024...1536], false).should.equal true
-
-				it "should insert the third set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1536...2048], false).should.equal true
-
-				it "should insert the final set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2048..2048], false).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-
-
-		describe "with an existing pack as the last update", ->
-			beforeEach ->
-				@PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5)
-				@PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
-				@PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
-
-			describe "for a small update", ->
-				it "should append the update to the existing pack", ->
-					@PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates, false).should.equal true
-				it "should not insert any new packs", ->
-					@PackManager.insertUpdatesIntoNewPack.called.should.equal false
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-			describe "for many small updates", ->
-				beforeEach ->
-					@newUpdates = ({ op: "op-#{i}", meta: "meta-#{i}", v: i} for i in [0..2048])
-					@PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
-
-				it "should append the initial updates to the existing pack", ->
-					@PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates[0...510], false).should.equal true
-
-				it "should insert the first set remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[510...1022], false).should.equal true
-
-				it "should insert the second set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1022...1534], false).should.equal true
-
-				it "should insert the third set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1534...2046], false).should.equal true
-
-				it "should insert the final set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2046..2048], false).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-			describe "for many big updates", ->
-				beforeEach ->
-					longString = ("a" for [0 .. (0.75*@PackManager.MAX_SIZE)]).join("")
-					@newUpdates = ({ op: "op-#{i}-#{longString}", meta: "meta-#{i}", v: i} for i in [0..4])
-					@PackManager.insertCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
-
-				it "should append the initial updates to the existing pack", ->
-					@PackManager.appendUpdatesToExistingPack.calledWith(@project_id, @doc_id, @lastUpdate, @newUpdates[0..0], false).should.equal true
-
-				it "should insert the first set remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[1..1], false).should.equal true
-
-				it "should insert the second set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[2..2], false).should.equal true
-
-				it "should insert the third set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[3..3], false).should.equal true
-
-				it "should insert the final set of remaining updates as a new pack", ->
-					@PackManager.insertUpdatesIntoNewPack.calledWith(@project_id, @doc_id, @newUpdates[4..4], false).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-		describe "flushCompressedUpdates", ->
-			describe "when there is no previous update",  ->
-				beforeEach ->
-					@PackManager.flushCompressedUpdates @project_id, @doc_id, null, @newUpdates, true, @callback
-
-				describe "for a small update  that will expire", ->
-					it "should insert the update into mongo", ->
-						@db.docHistory.save.calledWithMatch({
-							pack: @newUpdates,
-							project_id: ObjectId(@project_id),
-							doc_id: ObjectId(@doc_id)
-							n: @newUpdates.length
-							v: @newUpdates[0].v
-							v_end: @newUpdates[@newUpdates.length-1].v
-						}).should.equal true
-
-					it "should set an expiry time in the future", ->
-						@db.docHistory.save.calledWithMatch({
-							expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
-						}).should.equal true
-
-					it "should call the callback", ->
-						@callback.called.should.equal true
-
-		describe "when there is a recent previous update in mongo that expires", ->
-			beforeEach ->
-				@lastUpdate = {
-					_id: "12345"
-					pack: [
-						{ op: "op-1", meta: "meta-1", v: 1},
-						{ op: "op-2", meta: "meta-2", v: 2}
-					]
-					n : 2
-					sz : 100
-					meta: {start_ts: Date.now() - 6 * 3600 * 1000}
-					expiresAt: new Date(Date.now())
-				}
-
-				@PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, true, @callback
-
-			describe "for a small update that will expire", ->
-				it "should append the update in mongo", ->
-					@db.docHistory.findAndModify.calledWithMatch({
-						query: {_id: @lastUpdate._id}
-						update: { $push: {"pack" : {$each: @newUpdates}}, $set: {v_end: @newUpdates[@newUpdates.length-1].v}}
-					}).should.equal true
-
-				it "should set an expiry time in the future", ->
-					@db.docHistory.findAndModify.calledWithMatch({
-						update: {$set: {expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)}}
-					}).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-
-		describe "when there is a recent previous update in mongo that expires", ->
-			beforeEach ->
-				@PackManager.updateIndex = sinon.stub().callsArg(2)
-
-				@lastUpdate = {
-					_id: "12345"
-					pack: [
-						{ op: "op-1", meta: "meta-1", v: 1},
-						{ op: "op-2", meta: "meta-2", v: 2}
-					]
-					n : 2
-					sz : 100
-					meta: {start_ts: Date.now() - 6 * 3600 * 1000}
-					expiresAt: new Date(Date.now())
-				}
-
-				@PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, false, @callback
-
-			describe "for a small update that will not expire", ->
-				it "should insert the update into mongo", ->
-					@db.docHistory.save.calledWithMatch({
-						pack: @newUpdates,
-						project_id: ObjectId(@project_id),
-						doc_id: ObjectId(@doc_id)
-						n: @newUpdates.length
-						v: @newUpdates[0].v
-						v_end: @newUpdates[@newUpdates.length-1].v
-					}).should.equal true
-
-				it "should not set any expiry time", ->
-					@db.docHistory.save.neverCalledWithMatch(sinon.match.has("expiresAt")).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-		describe "when there is an old previous update in mongo", ->
-			beforeEach ->
-				@lastUpdate = {
-					_id: "12345"
-					pack: [
-						{ op: "op-1", meta: "meta-1", v: 1},
-						{ op: "op-2", meta: "meta-2", v: 2}
-					]
-					n : 2
-					sz : 100
-					meta: {start_ts: Date.now() - 30 * 24 * 3600 * 1000}
-					expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000)
-				}
-
-				@PackManager.flushCompressedUpdates @project_id, @doc_id, @lastUpdate, @newUpdates, true, @callback
-
-			describe "for a small update that will expire", ->
-				it "should insert the update into mongo", ->
-					@db.docHistory.save.calledWithMatch({
-						pack: @newUpdates,
-						project_id: ObjectId(@project_id),
-						doc_id: ObjectId(@doc_id)
-						n: @newUpdates.length
-						v: @newUpdates[0].v
-						v_end: @newUpdates[@newUpdates.length-1].v
-					}).should.equal true
-
-				it "should set an expiry time in the future", ->
-					@db.docHistory.save.calledWithMatch({
-						expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
-					}).should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-
-	describe "getOpsByVersionRange", ->
-
-	describe "loadPacksByVersionRange", ->
-
-	describe "fetchPacksIfNeeded", ->
-
-	describe "makeProjectIterator", ->
-
-	describe "getPackById", ->
-
-	describe "increaseTTL", ->
-
-	describe "getIndex", ->
-
-	describe "getPackFromIndex", ->
-# getLastPackFromIndex:
-# getIndexWithKeys
-# initialiseIndex
-# updateIndex
-# findCompletedPacks
-# findUnindexedPacks
-# insertPacksIntoIndexWithLock
-# _insertPacksIntoIndex
-# archivePack
-# checkArchivedPack
-# processOldPack
-# 	updateIndexIfNeeded
-# 	findUnarchivedPacks
-
-	describe "checkArchiveNotInProgress", ->
-
-		describe "when an archive is in progress", ->
-			beforeEach ->
-				@db.docHistoryIndex =
-					findOne: sinon.stub().callsArgWith(2, null, {inS3:false})
-				@PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
-			it "should call the callback", ->
-				@callback.called.should.equal true
-			it "should return an error", ->
-				@callback.calledWith(sinon.match.has('message')).should.equal true
-
-		describe "when an archive is completed", ->
-			beforeEach ->
-				@db.docHistoryIndex =
-					findOne: sinon.stub().callsArgWith(2, null, {inS3:true})
-				@PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
-			it "should call the callback", ->
-				@callback.called.should.equal true
-			it "should return an error", ->
-				@callback.calledWith(sinon.match.has('message')).should.equal true
-
-		describe "when the archive has not started or completed", ->
-			beforeEach ->
-				@db.docHistoryIndex =
-					findOne: sinon.stub().callsArgWith(2, null, {})
-				@PackManager.checkArchiveNotInProgress @project_id, @doc_id, @pack_id, @callback
-			it "should call the callback with no error", ->
-				@callback.called.should.equal true
-			it "should return with no error", ->
-				(typeof @callback.lastCall.args[0]).should.equal 'undefined'
-
-	# describe "setTTLOnArchivedPack", ->
-	# 	beforeEach ->
-	# 		@pack_id = "somepackid"
-	# 		@onedayinms = 86400000
-	# 		@db.docHistory =
-	# 			findAndModify : sinon.stub().callsArgWith(1)
-
-	# 	it "should set expires to 1 day", (done)->
-	# 		#@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
-	# 		@PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
-	# 			args = @db.docHistory.findAndModify.args[0][0]
-	# 			args.query._id.should.equal @pack_id
-	# 			args.update['$set'].expiresAt.should.equal @onedayinms
-	# 			done()
-
-
-	# describe "_getOneDayInFutureWithRandomDelay", ->
-	# 	beforeEach ->
-	# 		@onedayinms = 86400000
-	# 		@thirtyMins = 1000 * 60 * 30
-
-	# 	it "should give 1 day + 30 mins random time", (done)->
-	# 		loops = 10000
-	# 		while --loops > 0
-	# 			randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
-	# 			randomDelay.should.be.above(0)
-	# 			randomDelay.should.be.below(@thirtyMins + 1)
-	# 		done()
-

services/track-changes/test/unit/coffee/RedisManager/RedisManagerTests.coffee

@@ -1,87 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/RedisManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "RedisManager", ->
-	beforeEach ->
-		@RedisManager = SandboxedModule.require modulePath, requires:
-			"redis-sharelatex" : 
-				createClient: () => @rclient =
-					auth: sinon.stub()
-					multi: () => @rclient
-			"settings-sharelatex":
-				redis:
-					history:
-						key_schema:
-							uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:#{doc_id}"
-							docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:#{project_id}"
-		@doc_id = "doc-id-123"
-		@project_id = "project-id-123"
-		@batchSize = 100
-		@callback = sinon.stub()
-
-	describe "getOldestDocUpdates", ->
-		beforeEach ->
-			@rawUpdates = [ {v: 42, op: "mock-op-42"}, { v: 45, op: "mock-op-45" }]
-			@jsonUpdates = (JSON.stringify(update) for update in @rawUpdates)
-			@rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonUpdates)
-			@RedisManager.getOldestDocUpdates @doc_id, @batchSize, @callback
-
-		it "should read the updates from redis", ->
-			@rclient.lrange
-				.calledWith("UncompressedHistoryOps:#{@doc_id}", 0, @batchSize - 1)
-				.should.equal true
-
-		it "should call the callback with the unparsed ops", ->
-			@callback.calledWith(null, @jsonUpdates).should.equal true
-
-
-		describe "expandDocUpdates", ->
-			beforeEach ->
-				@RedisManager.expandDocUpdates @jsonUpdates, @callback
-
-			it "should call the callback with the parsed ops", ->
-				@callback.calledWith(null, @rawUpdates).should.equal true
-
-
-		describe "deleteAppliedDocUpdates", ->
-			beforeEach ->
-				@rclient.lrem = sinon.stub()
-				@rclient.srem = sinon.stub()
-				@rclient.exec = sinon.stub().callsArgWith(0)
-				@RedisManager.deleteAppliedDocUpdates @project_id, @doc_id, @jsonUpdates, @callback
-
-			it "should delete the first update from redis", ->
-				@rclient.lrem
-					.calledWith("UncompressedHistoryOps:#{@doc_id}", 1, @jsonUpdates[0])
-					.should.equal true
-
-			it "should delete the second update from redis", ->
-				@rclient.lrem
-					.calledWith("UncompressedHistoryOps:#{@doc_id}", 1, @jsonUpdates[1])
-					.should.equal true
-
-			it "should delete the doc from the set of docs with history ops", ->
-				@rclient.srem
-					.calledWith("DocsWithHistoryOps:#{@project_id}", @doc_id)
-					.should.equal true
-
-			it "should call the callback ", ->
-				@callback.called.should.equal true
-
-	describe "getDocIdsWithHistoryOps", ->
-		beforeEach ->
-			@doc_ids = ["mock-id-1", "mock-id-2"]
-			@rclient.smembers = sinon.stub().callsArgWith(1, null, @doc_ids)
-			@RedisManager.getDocIdsWithHistoryOps @project_id, @callback
-
-		it "should read the doc_ids from redis", ->
-			@rclient.smembers
-				.calledWith("DocsWithHistoryOps:#{@project_id}")
-				.should.equal true
-
-		it "should call the callback with the doc_ids", ->
-			@callback.calledWith(null, @doc_ids).should.equal true

services/track-changes/test/unit/coffee/RestoreManager/RestoreManagerTests.coffee

@@ -1,39 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/RestoreManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "RestoreManager", ->
-	beforeEach ->
-		@RestoreManager = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
-			"./DocumentUpdaterManager": @DocumentUpdaterManager = {}
-			"./DiffManager": @DiffManager = {}
-		@callback = sinon.stub()
-		@project_id = "mock-project-id"
-		@doc_id = "mock-doc-id"
-		@user_id = "mock-user-id"
-		@version = 42
-
-	describe "restoreToBeforeVersion", ->
-		beforeEach ->
-			@content = "mock content"
-			@DocumentUpdaterManager.setDocument = sinon.stub().callsArg(4)
-			@DiffManager.getDocumentBeforeVersion = sinon.stub().callsArgWith(3, null, @content)
-			@RestoreManager.restoreToBeforeVersion @project_id, @doc_id, @version, @user_id, @callback
-
-		it "should get the content before the requested version", ->
-			@DiffManager.getDocumentBeforeVersion
-				.calledWith(@project_id, @doc_id, @version)
-				.should.equal true
-
-		it "should set the document in the document updater", ->
-			@DocumentUpdaterManager.setDocument
-				.calledWith(@project_id, @doc_id, @content, @user_id)
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-			

services/track-changes/test/unit/coffee/UpdateCompressor/UpdateCompressorTests.coffee

@@ -1,466 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/UpdateCompressor.js"
-SandboxedModule = require('sandboxed-module')
-
-bigstring = ("a" for [0 .. 2*1024*1024]).join("")
-mediumstring = ("a" for [0 .. 1024*1024]).join("")
-
-describe "UpdateCompressor", ->
-	beforeEach ->
-		@UpdateCompressor = SandboxedModule.require modulePath, requires:
-			"../lib/diff_match_patch": require("../../../../app/lib/diff_match_patch")
-		@user_id = "user-id-1"
-		@other_user_id = "user-id-2"
-		@ts1 = Date.now()
-		@ts2 = Date.now() + 1000
-
-	describe "convertToSingleOpUpdates", ->
-		it "should split grouped updates into individual updates", ->
-			expect(@UpdateCompressor.convertToSingleOpUpdates [{
-				op:   [ @op1 = { p: 0, i: "Foo" }, @op2 = { p: 6, i: "bar"} ]
-				meta: { ts: @ts1, user_id: @user_id }
-				v: 42
-			}, {
-				op:   [ @op3 = { p: 10, i: "baz" } ]
-				meta: { ts: @ts2, user_id: @other_user_id }
-				v: 43
-			}])
-			.to.deep.equal [{
-				op: @op1,
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id },
-				v: 42
-			}, {
-				op: @op2,
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id },
-				v: 42
-			}, {
-				op: @op3,
-				meta: { start_ts: @ts2, end_ts: @ts2, user_id: @other_user_id },
-				v: 43
-			}]
-
-		it "should return no-op updates when the op list is empty", ->
-			expect(@UpdateCompressor.convertToSingleOpUpdates [{
-				op:   []
-				meta: { ts: @ts1, user_id: @user_id }
-				v: 42
-			}])
-			.to.deep.equal [{
-				op: @UpdateCompressor.NOOP
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id },
-				v: 42
-			}]
-
-		it "should ignore comment ops", ->
-			expect(@UpdateCompressor.convertToSingleOpUpdates [{
-				op:   [ @op1 = { p: 0, i: "Foo" }, @op2 = { p: 9, c: "baz"}, @op3 = { p: 6, i: "bar"} ]
-				meta: { ts: @ts1, user_id: @user_id }
-				v: 42
-			}])
-			.to.deep.equal [{
-				op: @op1,
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id },
-				v: 42
-			}, {
-				op: @op3,
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id },
-				v: 42
-			}]
-
-	describe "concatUpdatesWithSameVersion", ->
-		it "should concat updates with the same version", ->
-			expect(@UpdateCompressor.concatUpdatesWithSameVersion [{
-				op:   @op1 = { p: 0, i: "Foo" }
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id }
-				v: 42
-			}, {
-				op:   @op2 = { p: 6, i: "bar" }
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id }
-				v: 42
-			}, {
-				op:   @op3 = { p: 10, i: "baz" }
-				meta: { start_ts: @ts2, end_ts: @ts2, user_id: @other_user_id }
-				v: 43
-			}])
-			.to.deep.equal [{
-				op:   [ @op1, @op2 ]
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id }
-				v: 42
-			}, {
-				op:   [ @op3 ]
-				meta: {  start_ts: @ts2, end_ts: @ts2, user_id: @other_user_id }
-				v: 43
-			}]
-
-		it "should turn a noop into an empty op", ->
-			expect(@UpdateCompressor.concatUpdatesWithSameVersion [{
-				op:   @UpdateCompressor.NOOP
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id }
-				v: 42
-			}])
-			.to.deep.equal [{
-				op:   []
-				meta: { start_ts: @ts1, end_ts: @ts1, user_id: @user_id }
-				v: 42
-			}]
-
-	describe "compress", ->
-		describe "insert - insert", ->
-			it "should append one insert to the other", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, i: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foobar" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should insert one insert inside the other", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 5, i: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "fobaro" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should not append separated inserts", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, i: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foo" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, i: "bar" }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should not append inserts that are too big (second op)", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, i: bigstring }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foo" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, i: bigstring }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should not append inserts that are too big (first op)", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: bigstring }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3 + bigstring.length, i: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: bigstring }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3 + bigstring.length, i: "bar" }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should not append inserts that are too big (first and second op)", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: mediumstring }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3 + mediumstring.length, i: mediumstring }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: mediumstring }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3 + mediumstring.length, i: mediumstring }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-
-		describe "delete - delete", ->
-			it "should append one delete to the other", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, d: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3, d: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, d: "foobar" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-				
-			it "should insert one delete inside the other", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, d: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 1, d: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 1, d: "bafoor" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-				
-			it "should not append separated deletes", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, d: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, d: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, d: "foo" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, d: "bar" }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-		describe "insert - delete", ->
-			it "should undo a previous insert", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 5, d: "o" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "fo" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-				
-			it "should remove part of an insert from the middle", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "fobaro" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 5, d: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foo" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should cancel out two opposite updates", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3, d: "foo" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [
-					op: { p: 3, i: "" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				]
-				
-			it "should not combine separated updates", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foo" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, d: "bar" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foo" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 9, d: "bar" }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-			it "should not combine updates with overlap beyond the end", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, i: "foobar" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, d: "bardle" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "foobar" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, d: "bardle" }
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-		
-		describe "delete - insert", ->
-			it "should do a diff of the content", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, d: "one two three four five six seven eight" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3, i: "one 2 three four five six seven eight" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 7, d: "two" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}, {
-					op: { p: 7, i: "2" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-			
-			it "should return a no-op if the delete and insert are the same", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: { p: 3, d: "one two three four five six seven eight" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 3, i: "one two three four five six seven eight" }
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: { p: 3, i: "" }
-					meta: start_ts: @ts1, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]
-
-		describe "noop - insert", ->
-			it "should leave them untouched", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: @UpdateCompressor.NOOP
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, i: "bar" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: @UpdateCompressor.NOOP
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, i: "bar" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 43
-				}]
-
-		describe "noop - delete", ->
-			it "should leave them untouched", ->
-				expect(@UpdateCompressor.compressUpdates [{
-					op: @UpdateCompressor.NOOP
-					meta: ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, d: "bar" }
-					meta: ts: @ts1, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: @UpdateCompressor.NOOP
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, {
-					op: { p: 6, d: "bar" }
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 43
-				}]
-
-	describe "compressRawUpdates", ->
-		describe "merging in-place with an array op", ->
-			it "should not change the existing last updates", ->
-				expect(@UpdateCompressor.compressRawUpdates {
-					op: [ {"p":1000,"d":"hello"}, {"p":1000,"i":"HELLO()"} ]
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				}, [{
-					op: [{ p: 1006, i: "WORLD" }]
-					meta: ts: @ts2, user_id: @user_id
-					v: 43
-				}])
-				.to.deep.equal [{
-					op: [{"p":1000,"d":"hello"}, {"p":1000,"i":"HELLO()"} ]
-					meta: start_ts: @ts1, end_ts: @ts1, user_id: @user_id
-					v: 42
-				},{
-					op: [{"p":1006,"i":"WORLD"}]
-					meta: start_ts: @ts2, end_ts: @ts2, user_id: @user_id
-					v: 43
-				}]

services/track-changes/test/unit/coffee/UpdateTrimmer/UpdateTrimmerTests.coffee

@@ -1,122 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/UpdateTrimmer.js"
-SandboxedModule = require('sandboxed-module')
-tk = require "timekeeper"
-
-describe "UpdateTrimmer", ->
-	beforeEach ->
-		@now = new Date()
-		tk.freeze(@now)
-
-		@UpdateTrimmer = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
-			"./WebApiManager": @WebApiManager = {}
-			"./MongoManager": @MongoManager = {}
-
-		@callback = sinon.stub()
-		@project_id = "mock-project-id"
-
-	afterEach ->
-		tk.reset()
-
-	describe "shouldTrimUpdates", ->
-		beforeEach ->
-			@metadata = {}
-			@details =
-				features: {}
-			@MongoManager.getProjectMetaData = sinon.stub().callsArgWith(1, null, @metadata)
-			@MongoManager.setProjectMetaData = sinon.stub().callsArgWith(2)
-			@MongoManager.upgradeHistory = sinon.stub().callsArgWith(1)
-			@WebApiManager.getProjectDetails = sinon.stub().callsArgWith(1, null, @details)
-
-		describe "with preserveHistory set in the project meta data", ->
-			beforeEach ->
-				@metadata.preserveHistory = true
-				@UpdateTrimmer.shouldTrimUpdates @project_id, @callback
-
-			it "should look up the meta data", ->
-				@MongoManager.getProjectMetaData
-					.calledWith(@project_id)
-					.should.equal true
-
-			it "should not look up the project details", ->
-				@WebApiManager.getProjectDetails
-					.called
-					.should.equal false
-
-			it "should return false", ->
-				@callback.calledWith(null, false).should.equal true
-
-		describe "without preserveHistory set in the project meta data", ->
-			beforeEach ->
-				@metadata.preserveHistory = false
-
-			describe "when the project has the versioning feature", ->
-				beforeEach ->
-					@details.features.versioning = true
-					@UpdateTrimmer.shouldTrimUpdates @project_id, @callback
-
-				it "should look up the meta data", ->
-					@MongoManager.getProjectMetaData
-						.calledWith(@project_id)
-						.should.equal true
-
-				it "should look up the project details", ->
-					@WebApiManager.getProjectDetails
-						.calledWith(@project_id)
-						.should.equal true
-
-				it "should insert preserveHistory into the metadata", ->
-					@MongoManager.setProjectMetaData
-						.calledWith(@project_id, {preserveHistory: true})
-						.should.equal true
-
-				it "should upgrade any existing history", ->
-					@MongoManager.upgradeHistory
-						.calledWith(@project_id)
-						.should.equal true
-
-				it "should return false", ->
-					@callback.calledWith(null, false).should.equal true
-
-			describe "when the project does not have the versioning feature", ->
-				beforeEach ->
-					@details.features.versioning = false
-					@UpdateTrimmer.shouldTrimUpdates @project_id, @callback
-
-				it "should return true", ->
-					@callback.calledWith(null, true).should.equal true
-
-		describe "without any meta data", ->
-			beforeEach ->
-				@MongoManager.getProjectMetaData = sinon.stub().callsArgWith(1, null, null)
-
-			describe "when the project has the versioning feature", ->
-				beforeEach ->
-					@details.features.versioning = true
-					@UpdateTrimmer.shouldTrimUpdates @project_id, @callback
-
-				it "should insert preserveHistory into the metadata", ->
-					@MongoManager.setProjectMetaData
-						.calledWith(@project_id, {preserveHistory: true})
-						.should.equal true
-
-				it "should upgrade any existing history", ->
-					@MongoManager.upgradeHistory
-						.calledWith(@project_id)
-						.should.equal true
-
-				it "should return false", ->
-					@callback.calledWith(null, false).should.equal true
-
-			describe "when the project does not have the versioning feature", ->
-				beforeEach ->
-					@details.features.versioning = false
-					@UpdateTrimmer.shouldTrimUpdates @project_id, @callback
-
-				it "should return true", ->
-					@callback.calledWith(null, true).should.equal true
-

services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee

@@ -1,823 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/UpdatesManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "UpdatesManager", ->
-	beforeEach ->
-		@UpdatesManager = SandboxedModule.require modulePath, requires:
-			"./UpdateCompressor": @UpdateCompressor = {}
-			"./MongoManager" : @MongoManager = {}
-			"./PackManager" : @PackManager = {}
-			"./RedisManager" : @RedisManager = {}
-			"./LockManager"  : @LockManager = {}
-			"./WebApiManager": @WebApiManager = {}
-			"./UpdateTrimmer": @UpdateTrimmer = {}
-			"./DocArchiveManager": @DocArchiveManager = {}
-			"logger-sharelatex": { log: sinon.stub(), error: sinon.stub() }
-			"settings-sharelatex":
-				redis: lock: key_schema:
-					historyLock: ({doc_id}) -> "HistoryLock:#{doc_id}"
-		@doc_id = "doc-id-123"
-		@project_id = "project-id-123"
-		@callback = sinon.stub()
-		@temporary = "temp-mock"
-
-	describe "compressAndSaveRawUpdates", ->
-		describe "when there are no raw ops", ->
-			beforeEach ->
-				@MongoManager.peekLastCompressedUpdate = sinon.stub()
-				@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, [], @temporary, @callback
-
-			it "should not need to access the database", ->
-				@MongoManager.peekLastCompressedUpdate.called.should.equal false
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-		describe "when there is no compressed history to begin with", ->
-			beforeEach ->
-				@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-				@compressedUpdates = [ { v: 13, op: "compressed-op-12" } ]
-
-				@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
-				@PackManager.insertCompressedUpdates = sinon.stub().callsArg(5)
-				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
-				@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-			it "should look at the last compressed op", ->
-				@MongoManager.peekLastCompressedUpdate
-					.calledWith(@doc_id)
-					.should.equal true
-			
-			it "should save the compressed ops as a pack", ->
-				@PackManager.insertCompressedUpdates
-					.calledWith(@project_id, @doc_id, null, @compressedUpdates, @temporary)
-					.should.equal true
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-		describe "when the raw ops need appending to existing history", ->
-			beforeEach ->
-				@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
-				@compressedUpdates = [ { v: 12, op: "compressed-op-11+12" }, { v: 13, op: "compressed-op-12" } ]
-
-				@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate, @lastCompressedUpdate.v)
-				@PackManager.insertCompressedUpdates = sinon.stub().callsArg(5)
-				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
-
-			describe "when the raw ops start where the existing history ends", ->
-				beforeEach ->
-					@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should look at the last compressed op", ->
-					@MongoManager.peekLastCompressedUpdate
-						.calledWith(@doc_id)
-						.should.equal true
-				
-				it "should compress the raw ops", ->
-					@UpdateCompressor.compressRawUpdates
-						.calledWith(null, @rawUpdates)
-						.should.equal true
-
-				it "should save the new compressed ops into a pack", ->
-					@PackManager.insertCompressedUpdates
-						.calledWith(@project_id, @doc_id, @lastCompressedUpdate, @compressedUpdates, @temporary)
-						.should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-			describe "when the raw ops start where the existing history ends and the history is in a pack", ->
-				beforeEach ->
-					@lastCompressedUpdate = {pack: [{ v: 11, op: "compressed-op-11" }], v:11}
-					@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-					@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate, @lastCompressedUpdate.v)
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should look at the last compressed op", ->
-					@MongoManager.peekLastCompressedUpdate
-						.calledWith(@doc_id)
-						.should.equal true
-
-				it "should compress the raw ops", ->
-					@UpdateCompressor.compressRawUpdates
-						.calledWith(null, @rawUpdates)
-						.should.equal true
-
-				it "should save the new compressed ops into a pack", ->
-					@PackManager.insertCompressedUpdates
-						.calledWith(@project_id, @doc_id, @lastCompressedUpdate, @compressedUpdates, @temporary)
-						.should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-			describe "when some raw ops are passed that have already been compressed", ->
-				beforeEach ->
-					@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should only compress the more recent raw ops", ->
-					@UpdateCompressor.compressRawUpdates
-						.calledWith(null, @rawUpdates.slice(-2))
-						.should.equal true
-
-			describe "when the raw ops do not follow from the last compressed op version", ->
-				beforeEach ->
-					@rawUpdates = [{ v: 13, op: "mock-op-13" }]
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should call the callback with an error", ->
-					@callback
-						.calledWith(sinon.match.has('message', "Tried to apply raw op at version 13 to last compressed update with version 11 from unknown time"))
-						.should.equal true
-
-				it "should not insert any update into mongo", ->
-					@PackManager.insertCompressedUpdates.called.should.equal false
-
-			describe "when the raw ops are out of order", ->
-				beforeEach ->
-					@rawUpdates = [{ v: 13, op: "mock-op-13" }, { v: 12, op: "mock-op-12" }]
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should call the callback with an error", ->
-					@callback
-						.calledWith(sinon.match.has('message'))
-						.should.equal true
-
-				it "should not insert any update into mongo", ->
-					@PackManager.insertCompressedUpdates.called.should.equal false
-
-
-		describe "when the raw ops need appending to existing history which is in S3", ->
-			beforeEach ->
-				@lastCompressedUpdate = null
-				@lastVersion = 11
-				@compressedUpdates = [ { v: 13, op: "compressed-op-12" } ]
-
-				@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null, @lastVersion)
-				@PackManager.insertCompressedUpdates = sinon.stub().callsArg(5)
-				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
-
-			describe "when the raw ops start where the existing history ends", ->
-				beforeEach ->
-					@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-					@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback
-
-				it "should try to look at the last compressed op", ->
-					@MongoManager.peekLastCompressedUpdate
-						.calledWith(@doc_id)
-						.should.equal true
-				
-				it "should compress the last compressed op and the raw ops", ->
-					@UpdateCompressor.compressRawUpdates
-						.calledWith(@lastCompressedUpdate, @rawUpdates)
-						.should.equal true
-				
-				it "should save the compressed ops", ->
-					@PackManager.insertCompressedUpdates
-						.calledWith(@project_id, @doc_id, null, @compressedUpdates, @temporary)
-						.should.equal true
-
-				it "should call the callback", ->
-					@callback.called.should.equal true
-
-	describe "processUncompressedUpdates", ->
-		beforeEach ->
-			@UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(4)
-			@RedisManager.deleteAppliedDocUpdates = sinon.stub().callsArg(3)
-			@MongoManager.backportProjectId = sinon.stub().callsArg(2)
-			@UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, @temporary = "temp mock")
-
-		describe "when there is fewer than one batch to send", ->
-			beforeEach ->
-				@updates = ["mock-update"]
-				@RedisManager.getOldestDocUpdates = sinon.stub().callsArgWith(2, null, @updates)
-				@RedisManager.expandDocUpdates = sinon.stub().callsArgWith(1, null, @updates)
-				@UpdatesManager.processUncompressedUpdates @project_id, @doc_id, @temporary, @callback
-
-			it "should get the oldest updates", ->
-				@RedisManager.getOldestDocUpdates
-					.calledWith(@doc_id, @UpdatesManager.REDIS_READ_BATCH_SIZE)
-					.should.equal true
-
-			it "should compress and save the updates", ->
-				@UpdatesManager.compressAndSaveRawUpdates
-					.calledWith(@project_id, @doc_id, @updates, @temporary)
-					.should.equal true
-
-			it "should delete the batch of uncompressed updates that was just processed", ->
-				@RedisManager.deleteAppliedDocUpdates
-					.calledWith(@project_id, @doc_id, @updates)
-					.should.equal true
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-		describe "when there are multiple batches to send", ->
-			beforeEach (done) ->
-				@UpdatesManager.REDIS_READ_BATCH_SIZE = 2
-				@updates = ["mock-update-0", "mock-update-1", "mock-update-2", "mock-update-3", "mock-update-4"]
-				@redisArray = @updates.slice()
-				@RedisManager.getOldestDocUpdates = (doc_id, batchSize, callback = (error, updates) ->) =>
-					updates = @redisArray.slice(0, batchSize)
-					@redisArray = @redisArray.slice(batchSize)
-					callback null, updates
-				sinon.spy @RedisManager, "getOldestDocUpdates"
-				@RedisManager.expandDocUpdates = (jsonUpdates, callback) =>
-					callback null, jsonUpdates
-				sinon.spy @RedisManager, "expandDocUpdates"
-				@UpdatesManager.processUncompressedUpdates @project_id, @doc_id, @temporary, (args...) =>
-					@callback(args...)
-					done()
-
-			it "should get the oldest updates in three batches ", ->
-				@RedisManager.getOldestDocUpdates.callCount.should.equal 3
-
-			it "should compress and save the updates in batches", ->
-				@UpdatesManager.compressAndSaveRawUpdates
-					.calledWith(@project_id, @doc_id, @updates.slice(0,2), @temporary)
-					.should.equal true
-				@UpdatesManager.compressAndSaveRawUpdates
-					.calledWith(@project_id, @doc_id, @updates.slice(2,4), @temporary)
-					.should.equal true
-				@UpdatesManager.compressAndSaveRawUpdates
-					.calledWith(@project_id, @doc_id, @updates.slice(4,5), @temporary)
-					.should.equal true
-
-			it "should delete the batches of uncompressed updates", ->
-				@RedisManager.deleteAppliedDocUpdates.callCount.should.equal 3
-
-			it "should call the callback", ->
-				@callback.called.should.equal true
-
-	describe "processCompressedUpdatesWithLock", ->
-		beforeEach ->
-			@UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, @temporary = "temp mock")
-			@MongoManager.backportProjectId = sinon.stub().callsArg(2)
-			@UpdatesManager._processUncompressedUpdates = sinon.stub().callsArg(3)
-			@LockManager.runWithLock = sinon.stub().callsArg(2)
-			@UpdatesManager.processUncompressedUpdatesWithLock @project_id, @doc_id, @callback
-
-		it "should check if the updates are temporary", ->
-			@UpdateTrimmer.shouldTrimUpdates
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should backport the project id", ->
-			@MongoManager.backportProjectId
-				.calledWith(@project_id, @doc_id)
-				.should.equal true
-
-		it "should run processUncompressedUpdates with the lock", ->
-			@LockManager.runWithLock
-				.calledWith(
-					"HistoryLock:#{@doc_id}"
-				)
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-
-	describe "getDocUpdates", ->
-		beforeEach ->
-			@updates = ["mock-updates"]
-			@options = { to: "mock-to", limit: "mock-limit" }
-			@PackManager.getOpsByVersionRange = sinon.stub().callsArgWith(4, null, @updates)
-			@UpdatesManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(2)
-			@UpdatesManager.getDocUpdates @project_id, @doc_id, @options, @callback
-
-		it "should process outstanding updates", ->
-			@UpdatesManager.processUncompressedUpdatesWithLock
-				.calledWith(@project_id, @doc_id)
-				.should.equal true
-
-		it "should get the updates from the database", ->
-			@PackManager.getOpsByVersionRange
-				.calledWith(@project_id, @doc_id, @options.from, @options.to)
-				.should.equal true
-
-		it "should return the updates", ->
-			@callback
-				.calledWith(null, @updates)
-				.should.equal true
-
-	describe "getDocUpdatesWithUserInfo", ->
-		beforeEach ->
-			@updates = ["mock-updates"]
-			@options = { to: "mock-to", limit: "mock-limit" }
-			@updatesWithUserInfo = ["updates-with-user-info"]
-			@UpdatesManager.getDocUpdates = sinon.stub().callsArgWith(3, null, @updates)
-			@UpdatesManager.fillUserInfo = sinon.stub().callsArgWith(1, null, @updatesWithUserInfo)
-			@UpdatesManager.getDocUpdatesWithUserInfo @project_id, @doc_id, @options, @callback
-
-		it "should get the updates", ->
-			@UpdatesManager.getDocUpdates
-				.calledWith(@project_id, @doc_id, @options)
-				.should.equal true
-
-		it "should file the updates with the user info", ->
-			@UpdatesManager.fillUserInfo
-				.calledWith(@updates)
-				.should.equal true
-
-		it "should return the updates with the filled details", ->
-			@callback.calledWith(null, @updatesWithUserInfo).should.equal true
-
-	describe "processUncompressedUpdatesForProject", ->
-		beforeEach (done) ->
-			@doc_ids = ["mock-id-1", "mock-id-2"]
-			@UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, @temporary = "temp mock")
-			@MongoManager.backportProjectId = sinon.stub().callsArg(2)
-			@UpdatesManager._processUncompressedUpdatesForDocWithLock = sinon.stub().callsArg(3)
-			@RedisManager.getDocIdsWithHistoryOps = sinon.stub().callsArgWith(1, null, @doc_ids)
-			@UpdatesManager.processUncompressedUpdatesForProject @project_id, () =>
-				@callback()
-				done()
-
-		it "should get all the docs with history ops", ->
-			@RedisManager.getDocIdsWithHistoryOps
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should process the doc ops for the each doc_id", ->
-			for doc_id in @doc_ids
-				@UpdatesManager._processUncompressedUpdatesForDocWithLock
-					.calledWith(@project_id, doc_id, @temporary)
-					.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-
-	describe "getSummarizedProjectUpdates", ->
-		beforeEach ->
-			@updates = [{doc_id: 123, v:456, op: "mock-updates", meta: {user_id: 123, start_ts: 1233, end_ts:1234}}]
-			@options = { before: "mock-before", limit: "mock-limit" }
-			@summarizedUpdates = [
-				{meta: {user_ids: [123], start_ts: 1233, end_ts:1234},docs:{"123":{fromV:456,toV:456}}}
-			]
-			@updatesWithUserInfo = ["updates-with-user-info"]
-			@done_state = false
-			@iterator =
-				next: (cb) =>
-					@done_state = true
-					cb(null, @updates)
-				done: () =>
-					@done_state
-			@PackManager.makeProjectIterator = sinon.stub().callsArgWith(2, null, @iterator)
-			@UpdatesManager.processUncompressedUpdatesForProject = sinon.stub().callsArg(1)
-			@UpdatesManager.fillSummarizedUserInfo = sinon.stub().callsArgWith(1, null, @updatesWithUserInfo)
-			@UpdatesManager.getSummarizedProjectUpdates @project_id, @options, @callback
-
-		it "should process any outstanding updates", ->
-			@UpdatesManager.processUncompressedUpdatesForProject
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should get the updates", ->
-			@PackManager.makeProjectIterator
-				.calledWith(@project_id, @options.before)
-				.should.equal true
-
-		it "should fill the updates with the user info", ->
-			@UpdatesManager.fillSummarizedUserInfo
-				.calledWith(@summarizedUpdates)
-				.should.equal true
-
-		it "should return the updates with the filled details", ->
-			@callback.calledWith(null, @updatesWithUserInfo).should.equal true
-
-	# describe "_extendBatchOfSummarizedUpdates", ->
-	# 	beforeEach ->
-	# 		@before = Date.now()
-	# 		@min_count = 2
-	# 		@existingSummarizedUpdates = ["summarized-updates-3"]
-	# 		@summarizedUpdates = ["summarized-updates-3", "summarized-update-2", "summarized-update-1"]
-
-	# 	describe "when there are updates to get", ->
-	# 		beforeEach ->
-	# 			@updates = [
-	# 				{op: "mock-op-1", meta: end_ts: @before - 10},
-	# 				{op: "mock-op-1", meta: end_ts: @nextBeforeTimestamp = @before - 20}
-	# 			]
-	# 			@existingSummarizedUpdates = ["summarized-updates-3"]
-	# 			@summarizedUpdates = ["summarized-updates-3", "summarized-update-2", "summarized-update-1"]
-	# 			@UpdatesManager._summarizeUpdates = sinon.stub().returns(@summarizedUpdates)
-	# 			@UpdatesManager.getProjectUpdatesWithUserInfo = sinon.stub().callsArgWith(2, null, @updates)
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates @project_id, @existingSummarizedUpdates, @before, @min_count, @callback
-
-	# 		it "should get the updates", ->
-	# 			@UpdatesManager.getProjectUpdatesWithUserInfo
-	# 				.calledWith(@project_id, { before: @before, limit: 3 * @min_count })
-	# 				.should.equal true
-
-	# 		it "should summarize the updates", ->
-	# 			@UpdatesManager._summarizeUpdates
-	# 				.calledWith(@updates, @existingSummarizedUpdates)
-	# 				.should.equal true
-
-	# 		it "should call the callback with the summarized updates and the next before timestamp", ->
-	# 			@callback.calledWith(null, @summarizedUpdates, @nextBeforeTimestamp).should.equal true
-
-	# 	describe "when there are no more updates", ->
-	# 		beforeEach ->
-	# 			@updates = []
-	# 			@UpdatesManager._summarizeUpdates = sinon.stub().returns(@summarizedUpdates)
-	# 			@UpdatesManager.getProjectUpdatesWithUserInfo = sinon.stub().callsArgWith(2, null, @updates)
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates @project_id, @existingSummarizedUpdates, @before, @min_count, @callback
-
-	# 		it "should call the callback with the summarized updates and null for nextBeforeTimestamp", ->
-	# 			@callback.calledWith(null, @summarizedUpdates, null).should.equal true
-
-	# describe "getSummarizedProjectUpdates", ->
-	# 	describe "when one batch of updates is enough to meet the limit", ->
-	# 		beforeEach ->
-	# 			@before = Date.now()
-	# 			@min_count = 2
-	# 			@updates = ["summarized-updates-3", "summarized-updates-2"]
-	# 			@nextBeforeTimestamp = @before - 100
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates = sinon.stub().callsArgWith(4, null, @updates, @nextBeforeTimestamp)
-	# 			@UpdatesManager.getSummarizedProjectUpdates @project_id, { before: @before, min_count: @min_count }, @callback
-
-	# 		it "should get the batch of summarized updates", ->
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates
-	# 				.calledWith(@project_id, [], @before, @min_count)
-	# 				.should.equal true
-
-	# 		it "should call the callback with the updates", ->
-	# 			@callback.calledWith(null, @updates, @nextBeforeTimestamp).should.equal true
-
-	# 	describe "when multiple batches are needed to meet the limit", ->
-	# 		beforeEach ->
-	# 			@before = Date.now()
-	# 			@min_count = 4
-	# 			@firstBatch =  [{ toV: 6, fromV: 6 }, { toV: 5, fromV: 5 }]
-	# 			@nextBeforeTimestamp = @before - 100
-	# 			@secondBatch = [{ toV: 4, fromV: 4 }, { toV: 3, fromV: 3 }]
-	# 			@nextNextBeforeTimestamp = @before - 200
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates = (project_id, existingUpdates, before, desiredLength, callback) =>
-	# 				if existingUpdates.length == 0
-	# 					callback null, @firstBatch, @nextBeforeTimestamp
-	# 				else
-	# 					callback null, @firstBatch.concat(@secondBatch), @nextNextBeforeTimestamp
-	# 			sinon.spy @UpdatesManager, "_extendBatchOfSummarizedUpdates"
-	# 			@UpdatesManager.getSummarizedProjectUpdates @project_id, { before: @before, min_count: @min_count }, @callback
-
-	# 		it "should get the first batch of summarized updates", ->
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates
-	# 				.calledWith(@project_id, [], @before, @min_count)
-	# 				.should.equal true
-
-	# 		it "should get the second batch of summarized updates", ->
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates
-	# 				.calledWith(@project_id, @firstBatch, @nextBeforeTimestamp, @min_count)
-	# 				.should.equal true
-
-	# 		it "should call the callback with all the updates", ->
-	# 			@callback.calledWith(null, @firstBatch.concat(@secondBatch), @nextNextBeforeTimestamp).should.equal true
-
-	# 	describe "when the end of the database is hit", ->
-	# 		beforeEach ->
-	# 			@before = Date.now()
-	# 			@min_count = 4
-	# 			@updates =  [{ toV: 6, fromV: 6 }, { toV: 5, fromV: 5 }]
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates = sinon.stub().callsArgWith(4, null, @updates, null)
-	# 			@UpdatesManager.getSummarizedProjectUpdates @project_id, { before: @before, min_count: @min_count }, @callback
-
-	# 		it "should get the batch of summarized updates", ->
-	# 			@UpdatesManager._extendBatchOfSummarizedUpdates
-	# 				.calledWith(@project_id, [], @before, @min_count)
-	# 				.should.equal true
-
-	# 		it "should call the callback with the updates", ->
-	# 			@callback.calledWith(null, @updates, null).should.equal true
-
-	describe "fillUserInfo", ->
-		describe "with valid users", ->
-			beforeEach (done) ->
-				{ObjectId} = require "mongojs"
-				@user_id_1 = ObjectId().toString()
-				@user_id_2 = ObjectId().toString()
-				@updates = [{
-					meta:
-						user_id: @user_id_1
-					op: "mock-op-1"
-				}, {
-					meta:
-						user_id: @user_id_1
-					op: "mock-op-2"
-				}, {
-					meta:
-						user_id: @user_id_2
-					op: "mock-op-3"
-				}]
-				@user_info = {}
-				@user_info[@user_id_1] = email: "user1@sharelatex.com"
-				@user_info[@user_id_2] = email: "user2@sharelatex.com"
-				
-				@WebApiManager.getUserInfo = (user_id, callback = (error, userInfo) ->) =>
-					callback null, @user_info[user_id]
-				sinon.spy @WebApiManager, "getUserInfo"
-
-				@UpdatesManager.fillUserInfo @updates, (error, @results) =>
-					done()
-
-			it "should only call getUserInfo once for each user_id", ->
-				@WebApiManager.getUserInfo.calledTwice.should.equal true
-				@WebApiManager.getUserInfo
-					.calledWith(@user_id_1)
-					.should.equal true
-				@WebApiManager.getUserInfo
-					.calledWith(@user_id_2)
-					.should.equal true
-
-			it "should return the updates with the user info filled", ->
-				expect(@results).to.deep.equal [{
-					meta:
-						user:
-							email: "user1@sharelatex.com"
-					op: "mock-op-1"
-				}, {
-					meta:
-						user:
-							email: "user1@sharelatex.com"
-					op: "mock-op-2"
-				}, {
-					meta:
-						user:
-							email: "user2@sharelatex.com"
-					op: "mock-op-3"
-				}]
-
-
-		describe "with invalid user ids", ->
-			beforeEach (done) ->
-				@updates = [{
-					meta:
-						user_id: null
-					op: "mock-op-1"
-				}, {
-					meta:
-						user_id: "anonymous-user"
-					op: "mock-op-2"
-				}]
-				@WebApiManager.getUserInfo = (user_id, callback = (error, userInfo) ->) =>
-					callback null, @user_info[user_id]
-				sinon.spy @WebApiManager, "getUserInfo"
-
-				@UpdatesManager.fillUserInfo @updates, (error, @results) =>
-					done()
-
-			it "should not call getUserInfo", ->
-				@WebApiManager.getUserInfo.called.should.equal false
-
-			it "should return the updates without the user info filled", ->
-				expect(@results).to.deep.equal [{
-					meta: {}
-					op: "mock-op-1"
-				}, {
-					meta: {}
-					op: "mock-op-2"
-				}]
-
-	describe "_summarizeUpdates", ->
-		beforeEach ->
-			@now = Date.now()
-			@user_1 = { id: "mock-user-1" }
-			@user_2 = { id: "mock-user-2" }
-
-		it "should concat updates that are close in time", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_2.id
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}]
-
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 5
-				meta:
-					user_ids: [@user_1.id, @user_2.id]
-					start_ts: @now
-					end_ts:   @now + 30
-			}]
-
-		it "should leave updates that are far apart in time", ->
-			oneDay = 1000 * 60 * 60 * 24
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_2.id
-					start_ts: @now + oneDay
-					end_ts:   @now + oneDay + 10
-				v: 5
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}]
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 5
-						toV: 5
-				meta:
-					user_ids: [@user_2.id]
-					start_ts: @now + oneDay
-					end_ts:   @now + oneDay + 10
-			}, {
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 4
-				meta:
-					user_ids: [@user_1.id]
-					start_ts: @now
-					end_ts:   @now + 10
-			}]
-
-		it "should concat onto existing summarized updates", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-2"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}, {
-				doc_id: "doc-id-2"
-				meta:
-					user_id: @user_2.id
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}], [{
-				docs: 
-					"doc-id-1": 
-						fromV: 6
-						toV: 8
-				meta:
-					user_ids: [@user_1.id]
-					start_ts: @now + 40
-					end_ts:   @now + 50
-			}]
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						toV: 8
-						fromV: 6
-					"doc-id-2":
-						toV: 5
-						fromV: 4
-				meta:
-					user_ids: [@user_1.id, @user_2.id]
-					start_ts: @now
-					end_ts:   @now + 50
-			}]
-
-		it "should include null user values", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: null
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}]
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 5
-				meta:
-					user_ids: [@user_1.id, null]
-					start_ts: @now
-					end_ts:   @now + 30
-			}]
-
-		it "should include null user values, when the null is earlier in the updates list", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				meta:
-					user_id: null
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}]
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 5
-				meta:
-					user_ids: [null, @user_1.id]
-					start_ts: @now
-					end_ts:   @now + 30
-			}]
-
-		it "should roll several null user values into one", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: null
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: null
-					start_ts: @now + 2
-					end_ts:   @now + 4
-				v: 4
-			}]
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 5
-				meta:
-					user_ids: [@user_1.id, null]
-					start_ts: @now
-					end_ts:   @now + 30
-			}]
-
-		it "should split updates before a big delete", ->
-			result = @UpdatesManager._summarizeUpdates [{
-				doc_id: "doc-id-1"
-				op: [{ d: "this is a long long long long long delete", p: 34 }]
-				meta:
-					user_id: @user_1.id
-					start_ts: @now + 20
-					end_ts:   @now + 30
-				v: 5
-			}, {
-				doc_id: "doc-id-1"
-				meta:
-					user_id: @user_2.id
-					start_ts: @now
-					end_ts:   @now + 10
-				v: 4
-			}]
-
-			expect(result).to.deep.equal [{
-				docs:
-					"doc-id-1":
-						fromV: 5
-						toV: 5
-				meta:
-					user_ids: [@user_1.id]
-					start_ts: @now + 20
-					end_ts:   @now + 30
-			}, {
-				docs:
-					"doc-id-1":
-						fromV: 4
-						toV: 4
-				meta:
-					user_ids: [@user_2.id]
-					start_ts: @now
-					end_ts:   @now + 10
-			}]

services/track-changes/test/unit/coffee/WebApiManager/WebApiManagerTests.coffee

@@ -1,123 +0,0 @@
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../../app/js/WebApiManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "WebApiManager", ->
-	beforeEach ->
-		@WebApiManager = SandboxedModule.require modulePath, requires:
-			"requestretry": @request = {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
-			'settings-sharelatex': @settings =
-				apis:
-					web:
-						url: "http://example.com"
-						user: "sharelatex"
-						pass: "password"
-		@callback = sinon.stub()
-		@user_id = "mock-user-id"
-		@project_id = "mock-project-id"
-		@user_info =
-			email: "leo@sharelatex.com"
-			id: @user_id
-			first_name: "Leo"
-			last_nane: "Lion"
-			extra_param: "blah"
-		@project =
-			features: "mock-features"
-
-	describe "getUserInfo", ->
-		describe "successfully", ->
-			beforeEach ->
-				@body = JSON.stringify @user_info
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
-				@WebApiManager.getUserInfo @user_id, @callback
-
-			it 'should get the user from the web api', ->
-				@request.get
-					.calledWithMatch({
-						url: "#{@settings.apis.web.url}/user/#{@user_id}/personal_info"
-						auth:
-							user: @settings.apis.web.user
-							pass: @settings.apis.web.pass
-							sendImmediately: true
-					})
-					.should.equal true
-
-			it "should call the callback with only the email, id and names", ->
-				@callback.calledWith(null, {
-					id:         @user_id
-					email:      @user_info.email
-					first_name: @user_info.first_name
-					last_name:  @user_info.last_name
-				}).should.equal true
-
-		describe "when the web API returns an error", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@WebApiManager.getUserInfo @user_id, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the web returns a failure error code", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500, attempts: 42}, "")
-				@WebApiManager.getUserInfo @user_id, @callback
-
-			it "should return the callback with an error", ->
-				@callback
-					.calledWith(sinon.match.has('message', "web returned a non-success status code: 500 (attempts: 42)"))
-					.should.equal true
-
-		describe "when the user cannot be found", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 404}, "nothing")
-				@WebApiManager.getUserInfo @user_id, @callback
-
-			it "should return a null value", ->
-				@callback
-					.calledWith(null, null)
-					.should.equal true
-
-
-	describe "getProjectDetails", ->
-		describe "successfully", ->
-			beforeEach ->
-				@body = JSON.stringify @project
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
-				@WebApiManager.getProjectDetails @project_id, @callback
-
-			it 'should get the project from the web api', ->
-				@request.get
-					.calledWithMatch({
-						url: "#{@settings.apis.web.url}/project/#{@project_id}/details"
-						auth:
-							user: @settings.apis.web.user
-							pass: @settings.apis.web.pass
-							sendImmediately: true
-					})
-					.should.equal true
-
-			it "should call the callback with the project", ->
-				@callback.calledWith(null, @project).should.equal true
-
-		describe "when the web API returns an error", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@WebApiManager.getProjectDetails @project_id, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the web returns a failure error code", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500, attempts: 42 }, "")
-				@WebApiManager.getProjectDetails @project_id, @callback
-
-			it "should return the callback with an error", ->
-				@callback
-					.calledWith(sinon.match.has('message', "web returned a non-success status code: 500 (attempts: 42)"))
-					.should.equal true

services/track-changes/test/unit/js/DiffGenerator/DiffGeneratorTests.js

@@ -0,0 +1,462 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/DiffGenerator.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('DiffGenerator', function() {
+  beforeEach(function() {
+    this.DiffGenerator = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': { warn: sinon.stub() }
+      }
+    })
+    this.ts = Date.now()
+    this.user_id = 'mock-user-id'
+    this.user_id_2 = 'mock-user-id-2'
+    return (this.meta = {
+      start_ts: this.ts,
+      end_ts: this.ts,
+      user_id: this.user_id
+    })
+  })
+
+  describe('rewindOp', function() {
+    describe('rewinding an insert', function() {
+      return it('should undo the insert', function() {
+        const content = 'hello world'
+        const rewoundContent = this.DiffGenerator.rewindOp(content, {
+          p: 6,
+          i: 'wo'
+        })
+        return rewoundContent.should.equal('hello rld')
+      })
+    })
+
+    describe('rewinding a delete', function() {
+      return it('should undo the delete', function() {
+        const content = 'hello rld'
+        const rewoundContent = this.DiffGenerator.rewindOp(content, {
+          p: 6,
+          d: 'wo'
+        })
+        return rewoundContent.should.equal('hello world')
+      })
+    })
+
+    describe('with an inconsistent update', function() {
+      return it('should throw an error', function() {
+        const content = 'hello world'
+        return expect(() => {
+          return this.DiffGenerator.rewindOp(content, { p: 6, i: 'foo' })
+        }).to.throw(this.DiffGenerator.ConsistencyError)
+      })
+    })
+
+    return describe('with an update which is beyond the length of the content', function() {
+      return it('should undo the insert as if it were at the end of the content', function() {
+        const content = 'foobar'
+        const rewoundContent = this.DiffGenerator.rewindOp(content, {
+          p: 4,
+          i: 'bar'
+        })
+        return rewoundContent.should.equal('foo')
+      })
+    })
+  })
+
+  describe('rewindUpdate', function() {
+    return it('should rewind ops in reverse', function() {
+      const content = 'aaabbbccc'
+      const update = {
+        op: [
+          { p: 3, i: 'bbb' },
+          { p: 6, i: 'ccc' }
+        ]
+      }
+      const rewoundContent = this.DiffGenerator.rewindUpdate(content, update)
+      return rewoundContent.should.equal('aaa')
+    })
+  })
+
+  describe('rewindUpdates', function() {
+    return it('should rewind updates in reverse', function() {
+      const content = 'aaabbbccc'
+      const updates = [
+        { op: [{ p: 3, i: 'bbb' }] },
+        { op: [{ p: 6, i: 'ccc' }] }
+      ]
+      const rewoundContent = this.DiffGenerator.rewindUpdates(content, updates)
+      return rewoundContent.should.equal('aaa')
+    })
+  })
+
+  describe('buildDiff', function() {
+    beforeEach(function() {
+      this.diff = [{ u: 'mock-diff' }]
+      this.content = 'Hello world'
+      this.updates = [
+        { i: 'mock-update-1' },
+        { i: 'mock-update-2' },
+        { i: 'mock-update-3' }
+      ]
+      this.DiffGenerator.applyUpdateToDiff = sinon.stub().returns(this.diff)
+      this.DiffGenerator.compressDiff = sinon.stub().returns(this.diff)
+      return (this.result = this.DiffGenerator.buildDiff(
+        this.content,
+        this.updates
+      ))
+    })
+
+    it('should return the diff', function() {
+      return this.result.should.deep.equal(this.diff)
+    })
+
+    it('should build the content into an initial diff', function() {
+      return this.DiffGenerator.applyUpdateToDiff
+        .calledWith(
+          [
+            {
+              u: this.content
+            }
+          ],
+          this.updates[0]
+        )
+        .should.equal(true)
+    })
+
+    it('should apply each update', function() {
+      return Array.from(this.updates).map(update =>
+        this.DiffGenerator.applyUpdateToDiff
+          .calledWith(sinon.match.any, update)
+          .should.equal(true)
+      )
+    })
+
+    return it('should compress the diff', function() {
+      return this.DiffGenerator.compressDiff
+        .calledWith(this.diff)
+        .should.equal(true)
+    })
+  })
+
+  describe('compressDiff', function() {
+    describe('with adjacent inserts with the same user_id', function() {
+      return it('should create one update with combined meta data and min/max timestamps', function() {
+        const diff = this.DiffGenerator.compressDiff([
+          {
+            i: 'foo',
+            meta: { start_ts: 10, end_ts: 20, user: { id: this.user_id } }
+          },
+          {
+            i: 'bar',
+            meta: { start_ts: 5, end_ts: 15, user: { id: this.user_id } }
+          }
+        ])
+        return expect(diff).to.deep.equal([
+          {
+            i: 'foobar',
+            meta: { start_ts: 5, end_ts: 20, user: { id: this.user_id } }
+          }
+        ])
+      })
+    })
+
+    describe('with adjacent inserts with different user_ids', function() {
+      return it('should leave the inserts unchanged', function() {
+        const input = [
+          {
+            i: 'foo',
+            meta: { start_ts: 10, end_ts: 20, user: { id: this.user_id } }
+          },
+          {
+            i: 'bar',
+            meta: { start_ts: 5, end_ts: 15, user: { id: this.user_id_2 } }
+          }
+        ]
+        const output = this.DiffGenerator.compressDiff(input)
+        return expect(output).to.deep.equal(input)
+      })
+    })
+
+    describe('with adjacent deletes with the same user_id', function() {
+      return it('should create one update with combined meta data and min/max timestamps', function() {
+        const diff = this.DiffGenerator.compressDiff([
+          {
+            d: 'foo',
+            meta: { start_ts: 10, end_ts: 20, user: { id: this.user_id } }
+          },
+          {
+            d: 'bar',
+            meta: { start_ts: 5, end_ts: 15, user: { id: this.user_id } }
+          }
+        ])
+        return expect(diff).to.deep.equal([
+          {
+            d: 'foobar',
+            meta: { start_ts: 5, end_ts: 20, user: { id: this.user_id } }
+          }
+        ])
+      })
+    })
+
+    return describe('with adjacent deletes with different user_ids', function() {
+      return it('should leave the deletes unchanged', function() {
+        const input = [
+          {
+            d: 'foo',
+            meta: { start_ts: 10, end_ts: 20, user: { id: this.user_id } }
+          },
+          {
+            d: 'bar',
+            meta: { start_ts: 5, end_ts: 15, user: { id: this.user_id_2 } }
+          }
+        ]
+        const output = this.DiffGenerator.compressDiff(input)
+        return expect(output).to.deep.equal(input)
+      })
+    })
+  })
+
+  return describe('applyUpdateToDiff', function() {
+    describe('an insert', function() {
+      it('should insert into the middle of (u)nchanged text', function() {
+        const diff = this.DiffGenerator.applyUpdateToDiff([{ u: 'foobar' }], {
+          op: [{ p: 3, i: 'baz' }],
+          meta: this.meta
+        })
+        return expect(diff).to.deep.equal([
+          { u: 'foo' },
+          { i: 'baz', meta: this.meta },
+          { u: 'bar' }
+        ])
+      })
+
+      it('should insert into the start of (u)changed text', function() {
+        const diff = this.DiffGenerator.applyUpdateToDiff([{ u: 'foobar' }], {
+          op: [{ p: 0, i: 'baz' }],
+          meta: this.meta
+        })
+        return expect(diff).to.deep.equal([
+          { i: 'baz', meta: this.meta },
+          { u: 'foobar' }
+        ])
+      })
+
+      it('should insert into the end of (u)changed text', function() {
+        const diff = this.DiffGenerator.applyUpdateToDiff([{ u: 'foobar' }], {
+          op: [{ p: 6, i: 'baz' }],
+          meta: this.meta
+        })
+        return expect(diff).to.deep.equal([
+          { u: 'foobar' },
+          { i: 'baz', meta: this.meta }
+        ])
+      })
+
+      it('should insert into the middle of (i)inserted text', function() {
+        const diff = this.DiffGenerator.applyUpdateToDiff(
+          [{ i: 'foobar', meta: this.meta }],
+          { op: [{ p: 3, i: 'baz' }], meta: this.meta }
+        )
+        return expect(diff).to.deep.equal([
+          { i: 'foo', meta: this.meta },
+          { i: 'baz', meta: this.meta },
+          { i: 'bar', meta: this.meta }
+        ])
+      })
+
+      return it('should not count deletes in the running length total', function() {
+        const diff = this.DiffGenerator.applyUpdateToDiff(
+          [{ d: 'deleted', meta: this.meta }, { u: 'foobar' }],
+          { op: [{ p: 3, i: 'baz' }], meta: this.meta }
+        )
+        return expect(diff).to.deep.equal([
+          { d: 'deleted', meta: this.meta },
+          { u: 'foo' },
+          { i: 'baz', meta: this.meta },
+          { u: 'bar' }
+        ])
+      })
+    })
+
+    return describe('a delete', function() {
+      describe('deleting unchanged text', function() {
+        it('should delete from the middle of (u)nchanged text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foobazbar' }],
+            { op: [{ p: 3, d: 'baz' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'foo' },
+            { d: 'baz', meta: this.meta },
+            { u: 'bar' }
+          ])
+        })
+
+        it('should delete from the start of (u)nchanged text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foobazbar' }],
+            { op: [{ p: 0, d: 'foo' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { d: 'foo', meta: this.meta },
+            { u: 'bazbar' }
+          ])
+        })
+
+        it('should delete from the end of (u)nchanged text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foobazbar' }],
+            { op: [{ p: 6, d: 'bar' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'foobaz' },
+            { d: 'bar', meta: this.meta }
+          ])
+        })
+
+        return it('should delete across multiple (u)changed text parts', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foo' }, { u: 'baz' }, { u: 'bar' }],
+            { op: [{ p: 2, d: 'obazb' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'fo' },
+            { d: 'o', meta: this.meta },
+            { d: 'baz', meta: this.meta },
+            { d: 'b', meta: this.meta },
+            { u: 'ar' }
+          ])
+        })
+      })
+
+      describe('deleting inserts', function() {
+        it('should delete from the middle of (i)nserted text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ i: 'foobazbar', meta: this.meta }],
+            { op: [{ p: 3, d: 'baz' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { i: 'foo', meta: this.meta },
+            { i: 'bar', meta: this.meta }
+          ])
+        })
+
+        it('should delete from the start of (u)nchanged text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ i: 'foobazbar', meta: this.meta }],
+            { op: [{ p: 0, d: 'foo' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([{ i: 'bazbar', meta: this.meta }])
+        })
+
+        it('should delete from the end of (u)nchanged text', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ i: 'foobazbar', meta: this.meta }],
+            { op: [{ p: 6, d: 'bar' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([{ i: 'foobaz', meta: this.meta }])
+        })
+
+        return it('should delete across multiple (u)changed and (i)nserted text parts', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foo' }, { i: 'baz', meta: this.meta }, { u: 'bar' }],
+            { op: [{ p: 2, d: 'obazb' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'fo' },
+            { d: 'o', meta: this.meta },
+            { d: 'b', meta: this.meta },
+            { u: 'ar' }
+          ])
+        })
+      })
+
+      describe('deleting over existing deletes', function() {
+        return it('should delete across multiple (u)changed and (d)deleted text parts', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foo' }, { d: 'baz', meta: this.meta }, { u: 'bar' }],
+            { op: [{ p: 2, d: 'ob' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'fo' },
+            { d: 'o', meta: this.meta },
+            { d: 'baz', meta: this.meta },
+            { d: 'b', meta: this.meta },
+            { u: 'ar' }
+          ])
+        })
+      })
+
+      describe("deleting when the text doesn't match", function() {
+        it('should throw an error when deleting from the middle of (u)nchanged text', function() {
+          return expect(() =>
+            this.DiffGenerator.applyUpdateToDiff([{ u: 'foobazbar' }], {
+              op: [{ p: 3, d: 'xxx' }],
+              meta: this.meta
+            })
+          ).to.throw(this.DiffGenerator.ConsistencyError)
+        })
+
+        it('should throw an error when deleting from the start of (u)nchanged text', function() {
+          return expect(() =>
+            this.DiffGenerator.applyUpdateToDiff([{ u: 'foobazbar' }], {
+              op: [{ p: 0, d: 'xxx' }],
+              meta: this.meta
+            })
+          ).to.throw(this.DiffGenerator.ConsistencyError)
+        })
+
+        return it('should throw an error when deleting from the end of (u)nchanged text', function() {
+          return expect(() =>
+            this.DiffGenerator.applyUpdateToDiff([{ u: 'foobazbar' }], {
+              op: [{ p: 6, d: 'xxx' }],
+              meta: this.meta
+            })
+          ).to.throw(this.DiffGenerator.ConsistencyError)
+        })
+      })
+
+      describe('when the last update in the existing diff is a delete', function() {
+        return it('should insert the new update before the delete', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ u: 'foo' }, { d: 'bar', meta: this.meta }],
+            { op: [{ p: 3, i: 'baz' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { u: 'foo' },
+            { i: 'baz', meta: this.meta },
+            { d: 'bar', meta: this.meta }
+          ])
+        })
+      })
+
+      return describe('when the only update in the existing diff is a delete', function() {
+        return it('should insert the new update after the delete', function() {
+          const diff = this.DiffGenerator.applyUpdateToDiff(
+            [{ d: 'bar', meta: this.meta }],
+            { op: [{ p: 0, i: 'baz' }], meta: this.meta }
+          )
+          return expect(diff).to.deep.equal([
+            { d: 'bar', meta: this.meta },
+            { i: 'baz', meta: this.meta }
+          ])
+        })
+      })
+    })
+  })
+})

services/track-changes/test/unit/js/DiffManager/DiffManagerTests.js

@@ -0,0 +1,448 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/DiffManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('DiffManager', function() {
+  beforeEach(function() {
+    this.DiffManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          error: sinon.stub(),
+          warn: sinon.stub()
+        }),
+        './UpdatesManager': (this.UpdatesManager = {}),
+        './DocumentUpdaterManager': (this.DocumentUpdaterManager = {}),
+        './DiffGenerator': (this.DiffGenerator = {})
+      }
+    })
+    this.callback = sinon.stub()
+    this.from = new Date()
+    this.to = new Date(Date.now() + 10000)
+    this.project_id = 'mock-project-id'
+    return (this.doc_id = 'mock-doc-id')
+  })
+
+  describe('getLatestDocAndUpdates', function() {
+    beforeEach(function() {
+      this.content = 'hello world'
+      this.version = 42
+      this.updates = ['mock-update-1', 'mock-update-2']
+
+      this.DocumentUpdaterManager.getDocument = sinon
+        .stub()
+        .callsArgWith(2, null, this.content, this.version)
+      return (this.UpdatesManager.getDocUpdatesWithUserInfo = sinon
+        .stub()
+        .callsArgWith(3, null, this.updates))
+    })
+
+    describe('with a fromVersion', function() {
+      beforeEach(function() {
+        return this.DiffManager.getLatestDocAndUpdates(
+          this.project_id,
+          this.doc_id,
+          this.from,
+          this.callback
+        )
+      })
+
+      it('should get the latest version of the doc', function() {
+        return this.DocumentUpdaterManager.getDocument
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should get the latest updates', function() {
+        return this.UpdatesManager.getDocUpdatesWithUserInfo
+          .calledWith(this.project_id, this.doc_id, { from: this.from })
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the content, version and updates', function() {
+        return this.callback
+          .calledWith(null, this.content, this.version, this.updates)
+          .should.equal(true)
+      })
+    })
+
+    return describe('with no fromVersion', function() {
+      beforeEach(function() {
+        return this.DiffManager.getLatestDocAndUpdates(
+          this.project_id,
+          this.doc_id,
+          null,
+          this.callback
+        )
+      })
+
+      it('should get the latest version of the doc', function() {
+        return this.DocumentUpdaterManager.getDocument
+          .calledWith(this.project_id, this.doc_id)
+          .should.equal(true)
+      })
+
+      it('should not get the latest updates', function() {
+        return this.UpdatesManager.getDocUpdatesWithUserInfo.called.should.equal(
+          false
+        )
+      })
+
+      return it('should call the callback with the content, version and blank updates', function() {
+        return this.callback
+          .calledWith(null, this.content, this.version, [])
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('getDiff', function() {
+    beforeEach(function() {
+      this.content = 'hello world'
+      // Op versions are the version they were applied to, so doc is always one version
+      // ahead.s
+      this.version = 43
+      this.updates = [
+        {
+          op: 'mock-4',
+          v: 42,
+          meta: { start_ts: new Date(this.to.getTime() + 20) }
+        },
+        {
+          op: 'mock-3',
+          v: 41,
+          meta: { start_ts: new Date(this.to.getTime() + 10) }
+        },
+        {
+          op: 'mock-2',
+          v: 40,
+          meta: { start_ts: new Date(this.to.getTime() - 10) }
+        },
+        {
+          op: 'mock-1',
+          v: 39,
+          meta: { start_ts: new Date(this.to.getTime() - 20) }
+        }
+      ]
+      this.fromVersion = 39
+      this.toVersion = 40
+      this.diffed_updates = this.updates.slice(2)
+      this.rewound_content = 'rewound-content'
+      return (this.diff = [{ u: 'mock-diff' }])
+    })
+
+    describe('with matching versions', function() {
+      beforeEach(function() {
+        this.DiffManager.getDocumentBeforeVersion = sinon
+          .stub()
+          .callsArgWith(3, null, this.rewound_content, this.updates)
+        this.DiffGenerator.buildDiff = sinon.stub().returns(this.diff)
+        return this.DiffManager.getDiff(
+          this.project_id,
+          this.doc_id,
+          this.fromVersion,
+          this.toVersion,
+          this.callback
+        )
+      })
+
+      it('should get the latest doc and version with all recent updates', function() {
+        return this.DiffManager.getDocumentBeforeVersion
+          .calledWith(this.project_id, this.doc_id, this.fromVersion)
+          .should.equal(true)
+      })
+
+      it('should generate the diff', function() {
+        return this.DiffGenerator.buildDiff
+          .calledWith(
+            this.rewound_content,
+            this.diffed_updates.slice().reverse()
+          )
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the diff', function() {
+        return this.callback.calledWith(null, this.diff).should.equal(true)
+      })
+    })
+
+    return describe('when the updates are inconsistent', function() {
+      beforeEach(function() {
+        this.DiffManager.getLatestDocAndUpdates = sinon
+          .stub()
+          .callsArgWith(3, null, this.content, this.version, this.updates)
+        this.DiffGenerator.buildDiff = sinon
+          .stub()
+          .throws((this.error = new Error('inconsistent!')))
+        return this.DiffManager.getDiff(
+          this.project_id,
+          this.doc_id,
+          this.fromVersion,
+          this.toVersion,
+          this.callback
+        )
+      })
+
+      return it('should call the callback with an error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+  })
+
+  describe('getDocumentBeforeVersion', function() {
+    beforeEach(function() {
+      this.DiffManager._tryGetDocumentBeforeVersion = sinon.stub()
+      this.document = 'mock-documents'
+      return (this.rewound_updates = 'mock-rewound-updates')
+    })
+
+    describe('succesfully', function() {
+      beforeEach(function() {
+        this.DiffManager._tryGetDocumentBeforeVersion.yields(
+          null,
+          this.document,
+          this.rewound_updates
+        )
+        return this.DiffManager.getDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.version,
+          this.callback
+        )
+      })
+
+      it('should call _tryGetDocumentBeforeVersion', function() {
+        return this.DiffManager._tryGetDocumentBeforeVersion
+          .calledWith(this.project_id, this.doc_id, this.version)
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the response', function() {
+        return this.callback
+          .calledWith(null, this.document, this.rewound_updates)
+          .should.equal(true)
+      })
+    })
+
+    describe('with a retry needed', function() {
+      beforeEach(function() {
+        let retried = false
+        this.DiffManager._tryGetDocumentBeforeVersion = (
+          project_id,
+          doc_id,
+          version,
+          callback
+        ) => {
+          if (!retried) {
+            retried = true
+            const error = new Error()
+            error.retry = true
+            return callback(error)
+          } else {
+            return callback(null, this.document, this.rewound_updates)
+          }
+        }
+        sinon.spy(this.DiffManager, '_tryGetDocumentBeforeVersion')
+        return this.DiffManager.getDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.version,
+          this.callback
+        )
+      })
+
+      it('should call _tryGetDocumentBeforeVersion twice', function() {
+        return this.DiffManager._tryGetDocumentBeforeVersion.calledTwice.should.equal(
+          true
+        )
+      })
+
+      return it('should call the callback with the response', function() {
+        return this.callback
+          .calledWith(null, this.document, this.rewound_updates)
+          .should.equal(true)
+      })
+    })
+
+    describe('with a non-retriable error', function() {
+      beforeEach(function() {
+        this.error = new Error('oops')
+        this.DiffManager._tryGetDocumentBeforeVersion.yields(this.error)
+        return this.DiffManager.getDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.version,
+          this.callback
+        )
+      })
+
+      it('should call _tryGetDocumentBeforeVersion once', function() {
+        return this.DiffManager._tryGetDocumentBeforeVersion.calledOnce.should.equal(
+          true
+        )
+      })
+
+      return it('should call the callback with the error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+
+    return describe('when retry limit is matched', function() {
+      beforeEach(function() {
+        this.error = new Error('oops')
+        this.error.retry = true
+        this.DiffManager._tryGetDocumentBeforeVersion.yields(this.error)
+        return this.DiffManager.getDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.version,
+          this.callback
+        )
+      })
+
+      it('should call _tryGetDocumentBeforeVersion three times (max retries)', function() {
+        return this.DiffManager._tryGetDocumentBeforeVersion.calledThrice.should.equal(
+          true
+        )
+      })
+
+      return it('should call the callback with the error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+  })
+
+  return describe('_tryGetDocumentBeforeVersion', function() {
+    beforeEach(function() {
+      this.content = 'hello world'
+      // Op versions are the version they were applied to, so doc is always one version
+      // ahead.s
+      this.version = 43
+      this.updates = [
+        {
+          op: 'mock-4',
+          v: 42,
+          meta: { start_ts: new Date(this.to.getTime() + 20) }
+        },
+        {
+          op: 'mock-3',
+          v: 41,
+          meta: { start_ts: new Date(this.to.getTime() + 10) }
+        },
+        {
+          op: 'mock-2',
+          v: 40,
+          meta: { start_ts: new Date(this.to.getTime() - 10) }
+        },
+        {
+          op: 'mock-1',
+          v: 39,
+          meta: { start_ts: new Date(this.to.getTime() - 20) }
+        }
+      ]
+      this.fromVersion = 39
+      this.rewound_content = 'rewound-content'
+      return (this.diff = [{ u: 'mock-diff' }])
+    })
+
+    describe('with matching versions', function() {
+      beforeEach(function() {
+        this.DiffManager.getLatestDocAndUpdates = sinon
+          .stub()
+          .callsArgWith(3, null, this.content, this.version, this.updates)
+        this.DiffGenerator.rewindUpdates = sinon.spy((content, updates) => {
+          // the rewindUpdates method reverses the 'updates' array
+          updates.reverse()
+          return this.rewound_content
+        })
+        this.rewindUpdatesWithArgs = this.DiffGenerator.rewindUpdates.withArgs(
+          this.content,
+          this.updates.slice().reverse()
+        )
+        return this.DiffManager._tryGetDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.fromVersion,
+          this.callback
+        )
+      })
+
+      it('should get the latest doc and version with all recent updates', function() {
+        return this.DiffManager.getLatestDocAndUpdates
+          .calledWith(this.project_id, this.doc_id, this.fromVersion)
+          .should.equal(true)
+      })
+
+      it('should rewind the diff', function() {
+        return sinon.assert.calledOnce(this.rewindUpdatesWithArgs)
+      })
+
+      return it('should call the callback with the rewound document and updates', function() {
+        return this.callback
+          .calledWith(null, this.rewound_content, this.updates)
+          .should.equal(true)
+      })
+    })
+
+    describe('with mismatching versions', function() {
+      beforeEach(function() {
+        this.version = 50
+        this.updates = [
+          { op: 'mock-1', v: 40 },
+          { op: 'mock-1', v: 39 }
+        ]
+        this.DiffManager.getLatestDocAndUpdates = sinon
+          .stub()
+          .callsArgWith(3, null, this.content, this.version, this.updates)
+        return this.DiffManager._tryGetDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.fromVersion,
+          this.callback
+        )
+      })
+
+      return it('should call the callback with an error with retry = true set', function() {
+        this.callback.calledOnce.should.equal(true)
+        const error = this.callback.args[0][0]
+        return expect(error.retry).to.equal(true)
+      })
+    })
+
+    return describe('when the updates are inconsistent', function() {
+      beforeEach(function() {
+        this.DiffManager.getLatestDocAndUpdates = sinon
+          .stub()
+          .callsArgWith(3, null, this.content, this.version, this.updates)
+        this.DiffGenerator.rewindUpdates = sinon
+          .stub()
+          .throws((this.error = new Error('inconsistent!')))
+        return this.DiffManager.getDocumentBeforeVersion(
+          this.project_id,
+          this.doc_id,
+          this.fromVersion,
+          this.callback
+        )
+      })
+
+      return it('should call the callback with an error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+  })
+})

services/track-changes/test/unit/js/DocArchive/MongoAWS.js

@@ -0,0 +1,112 @@
+/* eslint-disable
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+chai.should()
+const sinon = require('sinon')
+const modulePath = '../../../../app/js/MongoAWS.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongojs')
+const MemoryStream = require('memorystream')
+const zlib = require('zlib')
+
+describe('MongoAWS', function() {
+  beforeEach(function() {
+    this.MongoAWS = SandboxedModule.require(modulePath, {
+      requires: {
+        'settings-sharelatex': (this.settings = {
+          trackchanges: {
+            s3: {
+              secret: 's3-secret',
+              key: 's3-key'
+            },
+            stores: {
+              doc_history: 's3-bucket'
+            }
+          }
+        }),
+        child_process: (this.child_process = {}),
+        'mongo-uri': (this.mongouri = {}),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          error: sinon.stub(),
+          err() {}
+        }),
+        'aws-sdk': (this.awssdk = {}),
+        fs: (this.fs = {}),
+        's3-streams': (this.S3S = {}),
+        './mongojs': { db: (this.db = {}), ObjectId },
+        JSONStream: (this.JSONStream = {}),
+        'readline-stream': (this.readline = sinon.stub()),
+        'metrics-sharelatex': { inc() {} }
+      }
+    })
+
+    this.project_id = ObjectId().toString()
+    this.doc_id = ObjectId().toString()
+    this.pack_id = ObjectId()
+    this.update = { v: 123 }
+    return (this.callback = sinon.stub())
+  })
+
+  describe('archivePack', function() {
+    beforeEach(function(done) {
+      this.awssdk.config = { update: sinon.stub() }
+      this.awssdk.S3 = sinon.stub()
+      this.S3S.WriteStream = () => MemoryStream.createWriteStream()
+      this.db.docHistory = {}
+      this.db.docHistory.findOne = sinon
+        .stub()
+        .callsArgWith(1, null, { pack: 'hello' })
+
+      return this.MongoAWS.archivePack(
+        this.project_id,
+        this.doc_id,
+        this.pack_id,
+        (err, result) => {
+          this.callback()
+          return done()
+        }
+      )
+    })
+
+    return it('should call the callback', function() {
+      return this.callback.called.should.equal(true)
+    })
+  })
+
+  return describe('unArchivePack', function() {
+    beforeEach(function(done) {
+      return zlib.gzip('{"pack":"123"}', (err, zbuf) => {
+        this.awssdk.config = { update: sinon.stub() }
+        this.awssdk.S3 = sinon.stub()
+        this.S3S.ReadStream = () =>
+          MemoryStream.createReadStream(zbuf, { readable: true })
+        this.db.docHistory = {}
+        this.db.docHistory.insert = sinon.stub().callsArgWith(1, null, 'pack')
+
+        return this.MongoAWS.unArchivePack(
+          this.project_id,
+          this.doc_id,
+          this.pack_id,
+          (err, result) => {
+            this.callback()
+            return done()
+          }
+        )
+      })
+    })
+
+    return it('should call db.docHistory.insert', function() {
+      return this.db.docHistory.insert.called.should.equal(true)
+    })
+  })
+})

services/track-changes/test/unit/js/DocumentUpdaterManager/DocumentUpdaterManagerTests.js

@@ -0,0 +1,205 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/DocumentUpdaterManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('DocumentUpdaterManager', function() {
+  beforeEach(function() {
+    this.DocumentUpdaterManager = SandboxedModule.require(modulePath, {
+      requires: {
+        request: (this.request = {}),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          error: sinon.stub()
+        }),
+        'settings-sharelatex': (this.settings = {
+          apis: { documentupdater: { url: 'http://example.com' } }
+        })
+      }
+    })
+    this.callback = sinon.stub()
+    this.lines = ['one', 'two', 'three']
+    return (this.version = 42)
+  })
+
+  describe('getDocument', function() {
+    describe('successfully', function() {
+      beforeEach(function() {
+        this.body = JSON.stringify({
+          lines: this.lines,
+          version: this.version,
+          ops: []
+        })
+        this.request.get = sinon
+          .stub()
+          .callsArgWith(1, null, { statusCode: 200 }, this.body)
+        return this.DocumentUpdaterManager.getDocument(
+          this.project_id,
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      it('should get the document from the document updater', function() {
+        const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}/doc/${this.doc_id}`
+        return this.request.get.calledWith(url).should.equal(true)
+      })
+
+      return it('should call the callback with the content and version', function() {
+        return this.callback
+          .calledWith(null, this.lines.join('\n'), this.version)
+          .should.equal(true)
+      })
+    })
+
+    describe('when the document updater API returns an error', function() {
+      beforeEach(function() {
+        this.request.get = sinon
+          .stub()
+          .callsArgWith(
+            1,
+            (this.error = new Error('something went wrong')),
+            null,
+            null
+          )
+        return this.DocumentUpdaterManager.getDocument(
+          this.project_id,
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      return it('should return an error to the callback', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+
+    return describe('when the document updater returns a failure error code', function() {
+      beforeEach(function() {
+        this.request.get = sinon
+          .stub()
+          .callsArgWith(1, null, { statusCode: 500 }, '')
+        return this.DocumentUpdaterManager.getDocument(
+          this.project_id,
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      return it('should return the callback with an error', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has(
+              'message',
+              'doc updater returned a non-success status code: 500'
+            )
+          )
+          .should.equal(true)
+      })
+    })
+  })
+
+  return describe('setDocument', function() {
+    beforeEach(function() {
+      this.content = 'mock content'
+      return (this.user_id = 'user-id-123')
+    })
+
+    describe('successfully', function() {
+      beforeEach(function() {
+        this.request.post = sinon
+          .stub()
+          .callsArgWith(1, null, { statusCode: 200 })
+        return this.DocumentUpdaterManager.setDocument(
+          this.project_id,
+          this.doc_id,
+          this.content,
+          this.user_id,
+          this.callback
+        )
+      })
+
+      it('should set the document in the document updater', function() {
+        const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}/doc/${this.doc_id}`
+        return this.request.post
+          .calledWith({
+            url,
+            json: {
+              lines: this.content.split('\n'),
+              source: 'restore',
+              user_id: this.user_id,
+              undoing: true
+            }
+          })
+          .should.equal(true)
+      })
+
+      return it('should call the callback', function() {
+        return this.callback.calledWith(null).should.equal(true)
+      })
+    })
+
+    describe('when the document updater API returns an error', function() {
+      beforeEach(function() {
+        this.request.post = sinon
+          .stub()
+          .callsArgWith(
+            1,
+            (this.error = new Error('something went wrong')),
+            null,
+            null
+          )
+        return this.DocumentUpdaterManager.setDocument(
+          this.project_id,
+          this.doc_id,
+          this.content,
+          this.user_id,
+          this.callback
+        )
+      })
+
+      return it('should return an error to the callback', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+
+    return describe('when the document updater returns a failure error code', function() {
+      beforeEach(function() {
+        this.request.post = sinon
+          .stub()
+          .callsArgWith(1, null, { statusCode: 500 }, '')
+        return this.DocumentUpdaterManager.setDocument(
+          this.project_id,
+          this.doc_id,
+          this.content,
+          this.user_id,
+          this.callback
+        )
+      })
+
+      return it('should return the callback with an error', function() {
+        return this.callback
+          .calledWith(
+            sinon.match.has(
+              'message',
+              'doc updater returned a non-success status code: 500'
+            )
+          )
+          .should.equal(true)
+      })
+    })
+  })
+})

services/track-changes/test/unit/js/HttpController/HttpControllerTests.js

@@ -0,0 +1,201 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/HttpController.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('HttpController', function() {
+  beforeEach(function() {
+    this.HttpController = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': { log: sinon.stub() },
+        './UpdatesManager': (this.UpdatesManager = {}),
+        './DiffManager': (this.DiffManager = {}),
+        './RestoreManager': (this.RestoreManager = {}),
+        './PackManager': (this.PackManager = {}),
+        './DocArchiveManager': (this.DocArchiveManager = {}),
+        './HealthChecker': (this.HealthChecker = {})
+      }
+    })
+    this.doc_id = 'doc-id-123'
+    this.project_id = 'project-id-123'
+    this.next = sinon.stub()
+    this.user_id = 'mock-user-123'
+    return (this.now = Date.now())
+  })
+
+  describe('flushDoc', function() {
+    beforeEach(function() {
+      this.req = {
+        params: {
+          doc_id: this.doc_id,
+          project_id: this.project_id
+        }
+      }
+      this.res = { send: sinon.stub() }
+      this.UpdatesManager.processUncompressedUpdatesWithLock = sinon
+        .stub()
+        .callsArg(2)
+      return this.HttpController.flushDoc(this.req, this.res, this.next)
+    })
+
+    it('should process the updates', function() {
+      return this.UpdatesManager.processUncompressedUpdatesWithLock
+        .calledWith(this.project_id, this.doc_id)
+        .should.equal(true)
+    })
+
+    return it('should return a success code', function() {
+      return this.res.send.calledWith(204).should.equal(true)
+    })
+  })
+
+  describe('flushProject', function() {
+    beforeEach(function() {
+      this.req = {
+        params: {
+          project_id: this.project_id
+        }
+      }
+      this.res = { send: sinon.stub() }
+      this.UpdatesManager.processUncompressedUpdatesForProject = sinon
+        .stub()
+        .callsArg(1)
+      return this.HttpController.flushProject(this.req, this.res, this.next)
+    })
+
+    it('should process the updates', function() {
+      return this.UpdatesManager.processUncompressedUpdatesForProject
+        .calledWith(this.project_id)
+        .should.equal(true)
+    })
+
+    return it('should return a success code', function() {
+      return this.res.send.calledWith(204).should.equal(true)
+    })
+  })
+
+  describe('getDiff', function() {
+    beforeEach(function() {
+      this.from = 42
+      this.to = 45
+      this.req = {
+        params: {
+          doc_id: this.doc_id,
+          project_id: this.project_id
+        },
+        query: {
+          from: this.from.toString(),
+          to: this.to.toString()
+        }
+      }
+      this.res = { json: sinon.stub() }
+      this.diff = [{ u: 'mock-diff' }]
+      this.DiffManager.getDiff = sinon.stub().callsArgWith(4, null, this.diff)
+      return this.HttpController.getDiff(this.req, this.res, this.next)
+    })
+
+    it('should get the diff', function() {
+      return this.DiffManager.getDiff
+        .calledWith(
+          this.project_id,
+          this.doc_id,
+          parseInt(this.from, 10),
+          parseInt(this.to, 10)
+        )
+        .should.equal(true)
+    })
+
+    return it('should return the diff', function() {
+      return this.res.json.calledWith({ diff: this.diff }).should.equal(true)
+    })
+  })
+
+  describe('getUpdates', function() {
+    beforeEach(function() {
+      this.before = Date.now()
+      this.nextBeforeTimestamp = this.before - 100
+      this.min_count = 10
+      this.req = {
+        params: {
+          project_id: this.project_id
+        },
+        query: {
+          before: this.before.toString(),
+          min_count: this.min_count.toString()
+        }
+      }
+      this.res = { json: sinon.stub() }
+      this.updates = ['mock-summarized-updates']
+      this.UpdatesManager.getSummarizedProjectUpdates = sinon
+        .stub()
+        .callsArgWith(2, null, this.updates, this.nextBeforeTimestamp)
+      return this.HttpController.getUpdates(this.req, this.res, this.next)
+    })
+
+    it('should get the updates', function() {
+      return this.UpdatesManager.getSummarizedProjectUpdates
+        .calledWith(this.project_id, {
+          before: this.before,
+          min_count: this.min_count
+        })
+        .should.equal(true)
+    })
+
+    return it('should return the formatted updates', function() {
+      return this.res.json
+        .calledWith({
+          updates: this.updates,
+          nextBeforeTimestamp: this.nextBeforeTimestamp
+        })
+        .should.equal(true)
+    })
+  })
+
+  return describe('RestoreManager', function() {
+    beforeEach(function() {
+      this.version = '42'
+      this.req = {
+        params: {
+          doc_id: this.doc_id,
+          project_id: this.project_id,
+          version: this.version
+        },
+        headers: {
+          'x-user-id': this.user_id
+        }
+      }
+      this.res = { send: sinon.stub() }
+
+      this.RestoreManager.restoreToBeforeVersion = sinon.stub().callsArg(4)
+      return this.HttpController.restore(this.req, this.res, this.next)
+    })
+
+    it('should restore the document', function() {
+      return this.RestoreManager.restoreToBeforeVersion
+        .calledWith(
+          this.project_id,
+          this.doc_id,
+          parseInt(this.version, 10),
+          this.user_id
+        )
+        .should.equal(true)
+    })
+
+    return it('should return a success code', function() {
+      return this.res.send.calledWith(204).should.equal(true)
+    })
+  })
+})

services/track-changes/test/unit/js/LockManager/LockManagerTests.js

@@ -0,0 +1,320 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    mocha/no-nested-tests,
+    no-return-assign,
+    no-undef,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/LockManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('LockManager', function() {
+  beforeEach(function() {
+    this.Settings = {
+      redis: {
+        lock: {}
+      }
+    }
+    this.LockManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'redis-sharelatex': {
+          createClient: () => {
+            return (this.rclient = { auth: sinon.stub() })
+          }
+        },
+        'settings-sharelatex': this.Settings,
+        'logger-sharelatex': { error() {} }
+      }
+    })
+
+    this.key = 'lock-key'
+    return (this.callback = sinon.stub())
+  })
+
+  describe('checkLock', function() {
+    describe('when the lock is taken', function() {
+      beforeEach(function() {
+        this.rclient.exists = sinon.stub().callsArgWith(1, null, '1')
+        return this.LockManager.checkLock(this.key, this.callback)
+      })
+
+      it('should check the lock in redis', function() {
+        return this.rclient.exists.calledWith(this.key).should.equal(true)
+      })
+
+      return it('should return the callback with false', function() {
+        return this.callback.calledWith(null, false).should.equal(true)
+      })
+    })
+
+    return describe('when the lock is free', function() {
+      beforeEach(function() {
+        this.rclient.exists = sinon.stub().callsArgWith(1, null, '0')
+        return this.LockManager.checkLock(this.key, this.callback)
+      })
+
+      return it('should return the callback with true', function() {
+        return this.callback.calledWith(null, true).should.equal(true)
+      })
+    })
+  })
+
+  describe('tryLock', function() {
+    describe('when the lock is taken', function() {
+      beforeEach(function() {
+        this.rclient.set = sinon.stub().callsArgWith(5, null, null)
+        this.LockManager.randomLock = sinon
+          .stub()
+          .returns('locked-random-value')
+        return this.LockManager.tryLock(this.key, this.callback)
+      })
+
+      it('should check the lock in redis', function() {
+        return this.rclient.set
+          .calledWith(
+            this.key,
+            'locked-random-value',
+            'EX',
+            this.LockManager.LOCK_TTL,
+            'NX'
+          )
+          .should.equal(true)
+      })
+
+      return it('should return the callback with false', function() {
+        return this.callback.calledWith(null, false).should.equal(true)
+      })
+    })
+
+    return describe('when the lock is free', function() {
+      beforeEach(function() {
+        this.rclient.set = sinon.stub().callsArgWith(5, null, 'OK')
+        return this.LockManager.tryLock(this.key, this.callback)
+      })
+
+      return it('should return the callback with true', function() {
+        return this.callback.calledWith(null, true).should.equal(true)
+      })
+    })
+  })
+
+  describe('deleteLock', function() {
+    return beforeEach(function() {
+      beforeEach(function() {
+        this.rclient.del = sinon.stub().callsArg(1)
+        return this.LockManager.deleteLock(this.key, this.callback)
+      })
+
+      it('should delete the lock in redis', function() {
+        return this.rclient.del.calledWith(key).should.equal(true)
+      })
+
+      return it('should call the callback', function() {
+        return this.callback.called.should.equal(true)
+      })
+    })
+  })
+
+  describe('getLock', function() {
+    describe('when the lock is not taken', function() {
+      beforeEach(function(done) {
+        this.LockManager.tryLock = sinon.stub().callsArgWith(1, null, true)
+        return this.LockManager.getLock(this.key, (...args) => {
+          this.callback(...Array.from(args || []))
+          return done()
+        })
+      })
+
+      it('should try to get the lock', function() {
+        return this.LockManager.tryLock.calledWith(this.key).should.equal(true)
+      })
+
+      it('should only need to try once', function() {
+        return this.LockManager.tryLock.callCount.should.equal(1)
+      })
+
+      return it('should return the callback', function() {
+        return this.callback.calledWith(null).should.equal(true)
+      })
+    })
+
+    describe('when the lock is initially set', function() {
+      beforeEach(function(done) {
+        const startTime = Date.now()
+        this.LockManager.LOCK_TEST_INTERVAL = 5
+        this.LockManager.tryLock = function(doc_id, callback) {
+          if (callback == null) {
+            callback = function(error, isFree) {}
+          }
+          if (Date.now() - startTime < 100) {
+            return callback(null, false)
+          } else {
+            return callback(null, true)
+          }
+        }
+        sinon.spy(this.LockManager, 'tryLock')
+
+        return this.LockManager.getLock(this.key, (...args) => {
+          this.callback(...Array.from(args || []))
+          return done()
+        })
+      })
+
+      it('should call tryLock multiple times until free', function() {
+        return (this.LockManager.tryLock.callCount > 1).should.equal(true)
+      })
+
+      return it('should return the callback', function() {
+        return this.callback.calledWith(null).should.equal(true)
+      })
+    })
+
+    return describe('when the lock times out', function() {
+      beforeEach(function(done) {
+        const time = Date.now()
+        this.LockManager.MAX_LOCK_WAIT_TIME = 5
+        this.LockManager.tryLock = sinon.stub().callsArgWith(1, null, false)
+        return this.LockManager.getLock(this.key, (...args) => {
+          this.callback(...Array.from(args || []))
+          return done()
+        })
+      })
+
+      return it('should return the callback with an error', function() {
+        return this.callback
+          .calledWith(sinon.match.instanceOf(Error))
+          .should.equal(true)
+      })
+    })
+  })
+
+  return describe('runWithLock', function() {
+    describe('with successful run', function() {
+      beforeEach(function() {
+        this.runner = function(releaseLock) {
+          if (releaseLock == null) {
+            releaseLock = function(error) {}
+          }
+          return releaseLock()
+        }
+        sinon.spy(this, 'runner')
+        this.LockManager.getLock = sinon.stub().callsArg(1)
+        this.LockManager.releaseLock = sinon.stub().callsArg(2)
+        return this.LockManager.runWithLock(
+          this.key,
+          this.runner,
+          this.callback
+        )
+      })
+
+      it('should get the lock', function() {
+        return this.LockManager.getLock.calledWith(this.key).should.equal(true)
+      })
+
+      it('should run the passed function', function() {
+        return this.runner.called.should.equal(true)
+      })
+
+      it('should release the lock', function() {
+        return this.LockManager.releaseLock
+          .calledWith(this.key)
+          .should.equal(true)
+      })
+
+      return it('should call the callback', function() {
+        return this.callback.called.should.equal(true)
+      })
+    })
+
+    describe('when the runner function returns an error', function() {
+      beforeEach(function() {
+        this.error = new Error('oops')
+        this.runner = releaseLock => {
+          if (releaseLock == null) {
+            releaseLock = function(error) {}
+          }
+          return releaseLock(this.error)
+        }
+        sinon.spy(this, 'runner')
+        this.LockManager.getLock = sinon.stub().callsArg(1)
+        this.LockManager.releaseLock = sinon.stub().callsArg(2)
+        return this.LockManager.runWithLock(
+          this.key,
+          this.runner,
+          this.callback
+        )
+      })
+
+      it('should release the lock', function() {
+        return this.LockManager.releaseLock
+          .calledWith(this.key)
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the error', function() {
+        return this.callback.calledWith(this.error).should.equal(true)
+      })
+    })
+
+    return describe('releaseLock', function() {
+      describe('when the lock is current', function() {
+        beforeEach(function() {
+          this.rclient.eval = sinon.stub().yields(null, 1)
+          return this.LockManager.releaseLock(
+            this.key,
+            this.lockValue,
+            this.callback
+          )
+        })
+
+        it('should clear the data from redis', function() {
+          return this.rclient.eval
+            .calledWith(
+              this.LockManager.unlockScript,
+              1,
+              this.key,
+              this.lockValue
+            )
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+
+      return describe('when the lock has expired', function() {
+        beforeEach(function() {
+          this.rclient.eval = sinon.stub().yields(null, 0)
+          return this.LockManager.releaseLock(
+            this.key,
+            this.lockValue,
+            this.callback
+          )
+        })
+
+        return it('should return an error if the lock has expired', function() {
+          return this.callback
+            .calledWith(
+              sinon.match.has('message', 'tried to release timed out lock')
+            )
+            .should.equal(true)
+        })
+      })
+    })
+  })
+})

services/track-changes/test/unit/js/MongoManager/MongoManagerTests.js

@@ -0,0 +1,245 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/MongoManager.js'
+const packModulePath = '../../../../app/js/PackManager.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongojs')
+const tk = require('timekeeper')
+
+describe('MongoManager', function() {
+  beforeEach(function() {
+    tk.freeze(new Date())
+    this.MongoManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './mongojs': { db: (this.db = {}), ObjectId },
+        './PackManager': (this.PackManager = {}),
+        'metrics-sharelatex': { timeAsyncMethod() {} },
+        'logger-sharelatex': { log() {} }
+      }
+    })
+    this.callback = sinon.stub()
+    this.doc_id = ObjectId().toString()
+    return (this.project_id = ObjectId().toString())
+  })
+
+  afterEach(function() {
+    return tk.reset()
+  })
+
+  describe('getLastCompressedUpdate', function() {
+    beforeEach(function() {
+      this.update = 'mock-update'
+      this.db.docHistory = {}
+      this.db.docHistory.find = sinon.stub().returns(this.db.docHistory)
+      this.db.docHistory.findOne = sinon.stub().returns(this.db.docHistory)
+      this.db.docHistory.sort = sinon.stub().returns(this.db.docHistory)
+      this.db.docHistory.limit = sinon.stub().returns(this.db.docHistory)
+      this.db.docHistory.toArray = sinon
+        .stub()
+        .callsArgWith(0, null, [this.update])
+
+      return this.MongoManager.getLastCompressedUpdate(
+        this.doc_id,
+        this.callback
+      )
+    })
+
+    it('should find the updates for the doc', function() {
+      return this.db.docHistory.find
+        .calledWith({ doc_id: ObjectId(this.doc_id) })
+        .should.equal(true)
+    })
+
+    it('should limit to one result', function() {
+      return this.db.docHistory.limit.calledWith(1).should.equal(true)
+    })
+
+    it('should sort in descending version order', function() {
+      return this.db.docHistory.sort.calledWith({ v: -1 }).should.equal(true)
+    })
+
+    return it('should call the call back with the update', function() {
+      return this.callback.calledWith(null, this.update).should.equal(true)
+    })
+  })
+
+  describe('peekLastCompressedUpdate', function() {
+    describe('when there is no last update', function() {
+      beforeEach(function() {
+        this.PackManager.getLastPackFromIndex = sinon
+          .stub()
+          .callsArgWith(1, null, null)
+        this.MongoManager.getLastCompressedUpdate = sinon
+          .stub()
+          .callsArgWith(1, null, null)
+        return this.MongoManager.peekLastCompressedUpdate(
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      it('should get the last update', function() {
+        return this.MongoManager.getLastCompressedUpdate
+          .calledWith(this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should call the callback with no update', function() {
+        return this.callback.calledWith(null, null).should.equal(true)
+      })
+    })
+
+    describe('when there is an update', function() {
+      beforeEach(function() {
+        this.update = { _id: Object() }
+        this.MongoManager.getLastCompressedUpdate = sinon
+          .stub()
+          .callsArgWith(1, null, this.update)
+        return this.MongoManager.peekLastCompressedUpdate(
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      it('should get the last update', function() {
+        return this.MongoManager.getLastCompressedUpdate
+          .calledWith(this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should call the callback with the update', function() {
+        return this.callback.calledWith(null, this.update).should.equal(true)
+      })
+    })
+
+    return describe('when there is a last update in S3', function() {
+      beforeEach(function() {
+        this.update = { _id: Object(), v: 12345, v_end: 12345, inS3: true }
+        this.PackManager.getLastPackFromIndex = sinon
+          .stub()
+          .callsArgWith(1, null, this.update)
+        this.MongoManager.getLastCompressedUpdate = sinon
+          .stub()
+          .callsArgWith(1, null)
+        return this.MongoManager.peekLastCompressedUpdate(
+          this.doc_id,
+          this.callback
+        )
+      })
+
+      it('should get the last update', function() {
+        return this.MongoManager.getLastCompressedUpdate
+          .calledWith(this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should call the callback with a null update and the correct version', function() {
+        return this.callback
+          .calledWith(null, null, this.update.v_end)
+          .should.equal(true)
+      })
+    })
+  })
+
+  describe('backportProjectId', function() {
+    beforeEach(function() {
+      this.db.docHistory = { update: sinon.stub().callsArg(3) }
+      return this.MongoManager.backportProjectId(
+        this.project_id,
+        this.doc_id,
+        this.callback
+      )
+    })
+
+    it("should insert the project_id into all entries for the doc_id which don't have it set", function() {
+      return this.db.docHistory.update
+        .calledWith(
+          {
+            doc_id: ObjectId(this.doc_id),
+            project_id: { $exists: false }
+          },
+          {
+            $set: { project_id: ObjectId(this.project_id) }
+          },
+          {
+            multi: true
+          }
+        )
+        .should.equal(true)
+    })
+
+    return it('should call the callback', function() {
+      return this.callback.called.should.equal(true)
+    })
+  })
+
+  describe('getProjectMetaData', function() {
+    beforeEach(function() {
+      this.metadata = { mock: 'metadata' }
+      this.db.projectHistoryMetaData = {
+        find: sinon.stub().callsArgWith(1, null, [this.metadata])
+      }
+      return this.MongoManager.getProjectMetaData(
+        this.project_id,
+        this.callback
+      )
+    })
+
+    it('should look up the meta data in the db', function() {
+      return this.db.projectHistoryMetaData.find
+        .calledWith({ project_id: ObjectId(this.project_id) })
+        .should.equal(true)
+    })
+
+    return it('should return the metadata', function() {
+      return this.callback.calledWith(null, this.metadata).should.equal(true)
+    })
+  })
+
+  return describe('setProjectMetaData', function() {
+    beforeEach(function() {
+      this.metadata = { mock: 'metadata' }
+      this.db.projectHistoryMetaData = {
+        update: sinon.stub().callsArgWith(3, null, [this.metadata])
+      }
+      return this.MongoManager.setProjectMetaData(
+        this.project_id,
+        this.metadata,
+        this.callback
+      )
+    })
+
+    it('should upsert the metadata into the DB', function() {
+      return this.db.projectHistoryMetaData.update
+        .calledWith(
+          {
+            project_id: ObjectId(this.project_id)
+          },
+          {
+            $set: this.metadata
+          },
+          {
+            upsert: true
+          }
+        )
+        .should.equal(true)
+    })
+
+    return it('should call the callback', function() {
+      return this.callback.called.should.equal(true)
+    })
+  })
+})

services/track-changes/test/unit/js/PackManager/PackManagerTests.js

@@ -0,0 +1,710 @@
+/* eslint-disable
+    mocha/no-identical-title,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const { assert } = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/PackManager.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongojs')
+const bson = require('bson')
+const BSON = new bson.BSONPure()
+const _ = require('underscore')
+
+const tk = require('timekeeper')
+
+describe('PackManager', function() {
+  beforeEach(function() {
+    tk.freeze(new Date())
+    this.PackManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './mongojs': { db: (this.db = {}), ObjectId, BSON },
+        './LockManager': {},
+        './MongoAWS': {},
+        'logger-sharelatex': { log: sinon.stub(), error: sinon.stub() },
+        'metrics-sharelatex': { inc() {} },
+        './ProjectIterator': require('../../../../app/js/ProjectIterator.js'), // Cache for speed
+        'settings-sharelatex': {
+          redis: { lock: { key_schema: {} } }
+        }
+      }
+    })
+    this.callback = sinon.stub()
+    this.doc_id = ObjectId().toString()
+    this.project_id = ObjectId().toString()
+    return (this.PackManager.MAX_COUNT = 512)
+  })
+
+  afterEach(function() {
+    return tk.reset()
+  })
+
+  describe('insertCompressedUpdates', function() {
+    beforeEach(function() {
+      this.lastUpdate = {
+        _id: '12345',
+        pack: [
+          { op: 'op-1', meta: 'meta-1', v: 1 },
+          { op: 'op-2', meta: 'meta-2', v: 2 }
+        ],
+        n: 2,
+        sz: 100
+      }
+      this.newUpdates = [
+        { op: 'op-3', meta: 'meta-3', v: 3 },
+        { op: 'op-4', meta: 'meta-4', v: 4 }
+      ]
+      return (this.db.docHistory = {
+        save: sinon.stub().callsArg(1),
+        insert: sinon.stub().callsArg(1),
+        findAndModify: sinon.stub().callsArg(1)
+      })
+    })
+
+    describe('with no last update', function() {
+      beforeEach(function() {
+        this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
+        return this.PackManager.insertCompressedUpdates(
+          this.project_id,
+          this.doc_id,
+          null,
+          this.newUpdates,
+          true,
+          this.callback
+        )
+      })
+
+      describe('for a small update', function() {
+        it('should insert the update into a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(this.project_id, this.doc_id, this.newUpdates, true)
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+
+      return describe('for many small updates', function() {
+        beforeEach(function() {
+          this.newUpdates = __range__(0, 2048, true).map(i => ({
+            op: `op-${i}`,
+            meta: `meta-${i}`,
+            v: i
+          }))
+          return this.PackManager.insertCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            null,
+            this.newUpdates,
+            false,
+            this.callback
+          )
+        })
+
+        it('should append the initial updates to the existing pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(0, 512),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the first set remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(512, 1024),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the second set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(1024, 1536),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the third set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(1536, 2048),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the final set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(2048, 2049),
+              false
+            )
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+    })
+
+    describe('with an existing pack as the last update', function() {
+      beforeEach(function() {
+        this.PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5)
+        this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
+        return this.PackManager.insertCompressedUpdates(
+          this.project_id,
+          this.doc_id,
+          this.lastUpdate,
+          this.newUpdates,
+          false,
+          this.callback
+        )
+      })
+
+      describe('for a small update', function() {
+        it('should append the update to the existing pack', function() {
+          return this.PackManager.appendUpdatesToExistingPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.lastUpdate,
+              this.newUpdates,
+              false
+            )
+            .should.equal(true)
+        })
+        it('should not insert any new packs', function() {
+          return this.PackManager.insertUpdatesIntoNewPack.called.should.equal(
+            false
+          )
+        })
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+
+      describe('for many small updates', function() {
+        beforeEach(function() {
+          this.newUpdates = __range__(0, 2048, true).map(i => ({
+            op: `op-${i}`,
+            meta: `meta-${i}`,
+            v: i
+          }))
+          return this.PackManager.insertCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            this.lastUpdate,
+            this.newUpdates,
+            false,
+            this.callback
+          )
+        })
+
+        it('should append the initial updates to the existing pack', function() {
+          return this.PackManager.appendUpdatesToExistingPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.lastUpdate,
+              this.newUpdates.slice(0, 510),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the first set remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(510, 1022),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the second set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(1022, 1534),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the third set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(1534, 2046),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the final set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(2046, 2049),
+              false
+            )
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+
+      return describe('for many big updates', function() {
+        beforeEach(function() {
+          const longString = __range__(
+            0,
+            0.75 * this.PackManager.MAX_SIZE,
+            true
+          )
+            .map(j => 'a')
+            .join('')
+          this.newUpdates = [0, 1, 2, 3, 4].map(i => ({
+            op: `op-${i}-${longString}`,
+            meta: `meta-${i}`,
+            v: i
+          }))
+          return this.PackManager.insertCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            this.lastUpdate,
+            this.newUpdates,
+            false,
+            this.callback
+          )
+        })
+
+        it('should append the initial updates to the existing pack', function() {
+          return this.PackManager.appendUpdatesToExistingPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.lastUpdate,
+              this.newUpdates.slice(0, 1),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the first set remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(1, 2),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the second set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(2, 3),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the third set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(3, 4),
+              false
+            )
+            .should.equal(true)
+        })
+
+        it('should insert the final set of remaining updates as a new pack', function() {
+          return this.PackManager.insertUpdatesIntoNewPack
+            .calledWith(
+              this.project_id,
+              this.doc_id,
+              this.newUpdates.slice(4, 5),
+              false
+            )
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+    })
+
+    describe('flushCompressedUpdates', function() {
+      return describe('when there is no previous update', function() {
+        beforeEach(function() {
+          return this.PackManager.flushCompressedUpdates(
+            this.project_id,
+            this.doc_id,
+            null,
+            this.newUpdates,
+            true,
+            this.callback
+          )
+        })
+
+        return describe('for a small update  that will expire', function() {
+          it('should insert the update into mongo', function() {
+            return this.db.docHistory.save
+              .calledWithMatch({
+                pack: this.newUpdates,
+                project_id: ObjectId(this.project_id),
+                doc_id: ObjectId(this.doc_id),
+                n: this.newUpdates.length,
+                v: this.newUpdates[0].v,
+                v_end: this.newUpdates[this.newUpdates.length - 1].v
+              })
+              .should.equal(true)
+          })
+
+          it('should set an expiry time in the future', function() {
+            return this.db.docHistory.save
+              .calledWithMatch({
+                expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
+              })
+              .should.equal(true)
+          })
+
+          return it('should call the callback', function() {
+            return this.callback.called.should.equal(true)
+          })
+        })
+      })
+    })
+
+    describe('when there is a recent previous update in mongo that expires', function() {
+      beforeEach(function() {
+        this.lastUpdate = {
+          _id: '12345',
+          pack: [
+            { op: 'op-1', meta: 'meta-1', v: 1 },
+            { op: 'op-2', meta: 'meta-2', v: 2 }
+          ],
+          n: 2,
+          sz: 100,
+          meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
+          expiresAt: new Date(Date.now())
+        }
+
+        return this.PackManager.flushCompressedUpdates(
+          this.project_id,
+          this.doc_id,
+          this.lastUpdate,
+          this.newUpdates,
+          true,
+          this.callback
+        )
+      })
+
+      return describe('for a small update that will expire', function() {
+        it('should append the update in mongo', function() {
+          return this.db.docHistory.findAndModify
+            .calledWithMatch({
+              query: { _id: this.lastUpdate._id },
+              update: {
+                $push: { pack: { $each: this.newUpdates } },
+                $set: { v_end: this.newUpdates[this.newUpdates.length - 1].v }
+              }
+            })
+            .should.equal(true)
+        })
+
+        it('should set an expiry time in the future', function() {
+          return this.db.docHistory.findAndModify
+            .calledWithMatch({
+              update: {
+                $set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) }
+              }
+            })
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+    })
+
+    describe('when there is a recent previous update in mongo that expires', function() {
+      beforeEach(function() {
+        this.PackManager.updateIndex = sinon.stub().callsArg(2)
+
+        this.lastUpdate = {
+          _id: '12345',
+          pack: [
+            { op: 'op-1', meta: 'meta-1', v: 1 },
+            { op: 'op-2', meta: 'meta-2', v: 2 }
+          ],
+          n: 2,
+          sz: 100,
+          meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
+          expiresAt: new Date(Date.now())
+        }
+
+        return this.PackManager.flushCompressedUpdates(
+          this.project_id,
+          this.doc_id,
+          this.lastUpdate,
+          this.newUpdates,
+          false,
+          this.callback
+        )
+      })
+
+      return describe('for a small update that will not expire', function() {
+        it('should insert the update into mongo', function() {
+          return this.db.docHistory.save
+            .calledWithMatch({
+              pack: this.newUpdates,
+              project_id: ObjectId(this.project_id),
+              doc_id: ObjectId(this.doc_id),
+              n: this.newUpdates.length,
+              v: this.newUpdates[0].v,
+              v_end: this.newUpdates[this.newUpdates.length - 1].v
+            })
+            .should.equal(true)
+        })
+
+        it('should not set any expiry time', function() {
+          return this.db.docHistory.save
+            .neverCalledWithMatch(sinon.match.has('expiresAt'))
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+    })
+
+    return describe('when there is an old previous update in mongo', function() {
+      beforeEach(function() {
+        this.lastUpdate = {
+          _id: '12345',
+          pack: [
+            { op: 'op-1', meta: 'meta-1', v: 1 },
+            { op: 'op-2', meta: 'meta-2', v: 2 }
+          ],
+          n: 2,
+          sz: 100,
+          meta: { start_ts: Date.now() - 30 * 24 * 3600 * 1000 },
+          expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000)
+        }
+
+        return this.PackManager.flushCompressedUpdates(
+          this.project_id,
+          this.doc_id,
+          this.lastUpdate,
+          this.newUpdates,
+          true,
+          this.callback
+        )
+      })
+
+      return describe('for a small update that will expire', function() {
+        it('should insert the update into mongo', function() {
+          return this.db.docHistory.save
+            .calledWithMatch({
+              pack: this.newUpdates,
+              project_id: ObjectId(this.project_id),
+              doc_id: ObjectId(this.doc_id),
+              n: this.newUpdates.length,
+              v: this.newUpdates[0].v,
+              v_end: this.newUpdates[this.newUpdates.length - 1].v
+            })
+            .should.equal(true)
+        })
+
+        it('should set an expiry time in the future', function() {
+          return this.db.docHistory.save
+            .calledWithMatch({
+              expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
+            })
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
+        })
+      })
+    })
+  })
+
+  describe('getOpsByVersionRange', function() {})
+
+  describe('loadPacksByVersionRange', function() {})
+
+  describe('fetchPacksIfNeeded', function() {})
+
+  describe('makeProjectIterator', function() {})
+
+  describe('getPackById', function() {})
+
+  describe('increaseTTL', function() {})
+
+  describe('getIndex', function() {})
+
+  describe('getPackFromIndex', function() {})
+  // getLastPackFromIndex:
+  // getIndexWithKeys
+  // initialiseIndex
+  // updateIndex
+  // findCompletedPacks
+  // findUnindexedPacks
+  // insertPacksIntoIndexWithLock
+  // _insertPacksIntoIndex
+  // archivePack
+  // checkArchivedPack
+  // processOldPack
+  // 	updateIndexIfNeeded
+  // 	findUnarchivedPacks
+
+  return describe('checkArchiveNotInProgress', function() {
+    describe('when an archive is in progress', function() {
+      beforeEach(function() {
+        this.db.docHistoryIndex = {
+          findOne: sinon.stub().callsArgWith(2, null, { inS3: false })
+        }
+        return this.PackManager.checkArchiveNotInProgress(
+          this.project_id,
+          this.doc_id,
+          this.pack_id,
+          this.callback
+        )
+      })
+      it('should call the callback', function() {
+        return this.callback.called.should.equal(true)
+      })
+      return it('should return an error', function() {
+        return this.callback
+          .calledWith(sinon.match.has('message'))
+          .should.equal(true)
+      })
+    })
+
+    describe('when an archive is completed', function() {
+      beforeEach(function() {
+        this.db.docHistoryIndex = {
+          findOne: sinon.stub().callsArgWith(2, null, { inS3: true })
+        }
+        return this.PackManager.checkArchiveNotInProgress(
+          this.project_id,
+          this.doc_id,
+          this.pack_id,
+          this.callback
+        )
+      })
+      it('should call the callback', function() {
+        return this.callback.called.should.equal(true)
+      })
+      return it('should return an error', function() {
+        return this.callback
+          .calledWith(sinon.match.has('message'))
+          .should.equal(true)
+      })
+    })
+
+    return describe('when the archive has not started or completed', function() {
+      beforeEach(function() {
+        this.db.docHistoryIndex = {
+          findOne: sinon.stub().callsArgWith(2, null, {})
+        }
+        return this.PackManager.checkArchiveNotInProgress(
+          this.project_id,
+          this.doc_id,
+          this.pack_id,
+          this.callback
+        )
+      })
+      it('should call the callback with no error', function() {
+        return this.callback.called.should.equal(true)
+      })
+      return it('should return with no error', function() {
+        return (typeof this.callback.lastCall.args[0]).should.equal('undefined')
+      })
+    })
+  })
+})
+
+// describe "setTTLOnArchivedPack", ->
+// 	beforeEach ->
+// 		@pack_id = "somepackid"
+// 		@onedayinms = 86400000
+// 		@db.docHistory =
+// 			findAndModify : sinon.stub().callsArgWith(1)
+
+// 	it "should set expires to 1 day", (done)->
+// 		#@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
+// 		@PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
+// 			args = @db.docHistory.findAndModify.args[0][0]
+// 			args.query._id.should.equal @pack_id
+// 			args.update['$set'].expiresAt.should.equal @onedayinms
+// 			done()
+
+// describe "_getOneDayInFutureWithRandomDelay", ->
+// 	beforeEach ->
+// 		@onedayinms = 86400000
+// 		@thirtyMins = 1000 * 60 * 30
+
+// 	it "should give 1 day + 30 mins random time", (done)->
+// 		loops = 10000
+// 		while --loops > 0
+// 			randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
+// 			randomDelay.should.be.above(0)
+// 			randomDelay.should.be.below(@thirtyMins + 1)
+// 		done()
+
+function __range__(left, right, inclusive) {
+  const range = []
+  const ascending = left < right
+  const end = !inclusive ? right : ascending ? right + 1 : right - 1
+  for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
+    range.push(i)
+  }
+  return range
+}

services/track-changes/test/unit/js/RedisManager/RedisManagerTests.js

@@ -0,0 +1,166 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/RedisManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('RedisManager', function() {
+  beforeEach(function() {
+    this.RedisManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'redis-sharelatex': {
+          createClient: () => {
+            return (this.rclient = {
+              auth: sinon.stub(),
+              multi: () => this.rclient
+            })
+          }
+        },
+        'settings-sharelatex': {
+          redis: {
+            history: {
+              key_schema: {
+                uncompressedHistoryOps({ doc_id }) {
+                  return `UncompressedHistoryOps:${doc_id}`
+                },
+                docsWithHistoryOps({ project_id }) {
+                  return `DocsWithHistoryOps:${project_id}`
+                }
+              }
+            }
+          }
+        }
+      }
+    })
+    this.doc_id = 'doc-id-123'
+    this.project_id = 'project-id-123'
+    this.batchSize = 100
+    return (this.callback = sinon.stub())
+  })
+
+  describe('getOldestDocUpdates', function() {
+    beforeEach(function() {
+      this.rawUpdates = [
+        { v: 42, op: 'mock-op-42' },
+        { v: 45, op: 'mock-op-45' }
+      ]
+      this.jsonUpdates = Array.from(this.rawUpdates).map(update =>
+        JSON.stringify(update)
+      )
+      this.rclient.lrange = sinon.stub().callsArgWith(3, null, this.jsonUpdates)
+      return this.RedisManager.getOldestDocUpdates(
+        this.doc_id,
+        this.batchSize,
+        this.callback
+      )
+    })
+
+    it('should read the updates from redis', function() {
+      return this.rclient.lrange
+        .calledWith(
+          `UncompressedHistoryOps:${this.doc_id}`,
+          0,
+          this.batchSize - 1
+        )
+        .should.equal(true)
+    })
+
+    it('should call the callback with the unparsed ops', function() {
+      return this.callback.calledWith(null, this.jsonUpdates).should.equal(true)
+    })
+
+    describe('expandDocUpdates', function() {
+      beforeEach(function() {
+        return this.RedisManager.expandDocUpdates(
+          this.jsonUpdates,
+          this.callback
+        )
+      })
+
+      return it('should call the callback with the parsed ops', function() {
+        return this.callback
+          .calledWith(null, this.rawUpdates)
+          .should.equal(true)
+      })
+    })
+
+    return describe('deleteAppliedDocUpdates', function() {
+      beforeEach(function() {
+        this.rclient.lrem = sinon.stub()
+        this.rclient.srem = sinon.stub()
+        this.rclient.exec = sinon.stub().callsArgWith(0)
+        return this.RedisManager.deleteAppliedDocUpdates(
+          this.project_id,
+          this.doc_id,
+          this.jsonUpdates,
+          this.callback
+        )
+      })
+
+      it('should delete the first update from redis', function() {
+        return this.rclient.lrem
+          .calledWith(
+            `UncompressedHistoryOps:${this.doc_id}`,
+            1,
+            this.jsonUpdates[0]
+          )
+          .should.equal(true)
+      })
+
+      it('should delete the second update from redis', function() {
+        return this.rclient.lrem
+          .calledWith(
+            `UncompressedHistoryOps:${this.doc_id}`,
+            1,
+            this.jsonUpdates[1]
+          )
+          .should.equal(true)
+      })
+
+      it('should delete the doc from the set of docs with history ops', function() {
+        return this.rclient.srem
+          .calledWith(`DocsWithHistoryOps:${this.project_id}`, this.doc_id)
+          .should.equal(true)
+      })
+
+      return it('should call the callback ', function() {
+        return this.callback.called.should.equal(true)
+      })
+    })
+  })
+
+  return describe('getDocIdsWithHistoryOps', function() {
+    beforeEach(function() {
+      this.doc_ids = ['mock-id-1', 'mock-id-2']
+      this.rclient.smembers = sinon.stub().callsArgWith(1, null, this.doc_ids)
+      return this.RedisManager.getDocIdsWithHistoryOps(
+        this.project_id,
+        this.callback
+      )
+    })
+
+    it('should read the doc_ids from redis', function() {
+      return this.rclient.smembers
+        .calledWith(`DocsWithHistoryOps:${this.project_id}`)
+        .should.equal(true)
+    })
+
+    return it('should call the callback with the doc_ids', function() {
+      return this.callback.calledWith(null, this.doc_ids).should.equal(true)
+    })
+  })
+})

services/track-changes/test/unit/js/RestoreManager/RestoreManagerTests.js

@@ -0,0 +1,70 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/RestoreManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('RestoreManager', function() {
+  beforeEach(function() {
+    this.RestoreManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          error: sinon.stub()
+        }),
+        './DocumentUpdaterManager': (this.DocumentUpdaterManager = {}),
+        './DiffManager': (this.DiffManager = {})
+      }
+    })
+    this.callback = sinon.stub()
+    this.project_id = 'mock-project-id'
+    this.doc_id = 'mock-doc-id'
+    this.user_id = 'mock-user-id'
+    return (this.version = 42)
+  })
+
+  return describe('restoreToBeforeVersion', function() {
+    beforeEach(function() {
+      this.content = 'mock content'
+      this.DocumentUpdaterManager.setDocument = sinon.stub().callsArg(4)
+      this.DiffManager.getDocumentBeforeVersion = sinon
+        .stub()
+        .callsArgWith(3, null, this.content)
+      return this.RestoreManager.restoreToBeforeVersion(
+        this.project_id,
+        this.doc_id,
+        this.version,
+        this.user_id,
+        this.callback
+      )
+    })
+
+    it('should get the content before the requested version', function() {
+      return this.DiffManager.getDocumentBeforeVersion
+        .calledWith(this.project_id, this.doc_id, this.version)
+        .should.equal(true)
+    })
+
+    it('should set the document in the document updater', function() {
+      return this.DocumentUpdaterManager.setDocument
+        .calledWith(this.project_id, this.doc_id, this.content, this.user_id)
+        .should.equal(true)
+    })
+
+    return it('should call the callback', function() {
+      return this.callback.called.should.equal(true)
+    })
+  })
+})

services/track-changes/test/unit/js/UpdateCompressor/UpdateCompressorTests.js

@@ -0,0 +1,850 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../../app/js/UpdateCompressor.js'
+const SandboxedModule = require('sandboxed-module')
+
+const bigstring = __range__(0, 2 * 1024 * 1024, true)
+  .map(i => 'a')
+  .join('')
+const mediumstring = __range__(0, 1024 * 1024, true)
+  .map(j => 'a')
+  .join('')
+
+describe('UpdateCompressor', function() {
+  beforeEach(function() {
+    this.UpdateCompressor = SandboxedModule.require(modulePath, {
+      requires: {
+        '../lib/diff_match_patch': require('../../../../app/lib/diff_match_patch')
+      }
+    })
+    this.user_id = 'user-id-1'
+    this.other_user_id = 'user-id-2'
+    this.ts1 = Date.now()
+    return (this.ts2 = Date.now() + 1000)
+  })
+
+  describe('convertToSingleOpUpdates', function() {
+    it('should split grouped updates into individual updates', function() {
+      return expect(
+        this.UpdateCompressor.convertToSingleOpUpdates([
+          {
+            op: [
+              (this.op1 = { p: 0, i: 'Foo' }),
+              (this.op2 = { p: 6, i: 'bar' })
+            ],
+            meta: { ts: this.ts1, user_id: this.user_id },
+            v: 42
+          },
+          {
+            op: [(this.op3 = { p: 10, i: 'baz' })],
+            meta: { ts: this.ts2, user_id: this.other_user_id },
+            v: 43
+          }
+        ])
+      ).to.deep.equal([
+        {
+          op: this.op1,
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        },
+        {
+          op: this.op2,
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        },
+        {
+          op: this.op3,
+          meta: {
+            start_ts: this.ts2,
+            end_ts: this.ts2,
+            user_id: this.other_user_id
+          },
+          v: 43
+        }
+      ])
+    })
+
+    it('should return no-op updates when the op list is empty', function() {
+      return expect(
+        this.UpdateCompressor.convertToSingleOpUpdates([
+          {
+            op: [],
+            meta: { ts: this.ts1, user_id: this.user_id },
+            v: 42
+          }
+        ])
+      ).to.deep.equal([
+        {
+          op: this.UpdateCompressor.NOOP,
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        }
+      ])
+    })
+
+    return it('should ignore comment ops', function() {
+      return expect(
+        this.UpdateCompressor.convertToSingleOpUpdates([
+          {
+            op: [
+              (this.op1 = { p: 0, i: 'Foo' }),
+              (this.op2 = { p: 9, c: 'baz' }),
+              (this.op3 = { p: 6, i: 'bar' })
+            ],
+            meta: { ts: this.ts1, user_id: this.user_id },
+            v: 42
+          }
+        ])
+      ).to.deep.equal([
+        {
+          op: this.op1,
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        },
+        {
+          op: this.op3,
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        }
+      ])
+    })
+  })
+
+  describe('concatUpdatesWithSameVersion', function() {
+    it('should concat updates with the same version', function() {
+      return expect(
+        this.UpdateCompressor.concatUpdatesWithSameVersion([
+          {
+            op: (this.op1 = { p: 0, i: 'Foo' }),
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: (this.op2 = { p: 6, i: 'bar' }),
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: (this.op3 = { p: 10, i: 'baz' }),
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.other_user_id
+            },
+            v: 43
+          }
+        ])
+      ).to.deep.equal([
+        {
+          op: [this.op1, this.op2],
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        },
+        {
+          op: [this.op3],
+          meta: {
+            start_ts: this.ts2,
+            end_ts: this.ts2,
+            user_id: this.other_user_id
+          },
+          v: 43
+        }
+      ])
+    })
+
+    return it('should turn a noop into an empty op', function() {
+      return expect(
+        this.UpdateCompressor.concatUpdatesWithSameVersion([
+          {
+            op: this.UpdateCompressor.NOOP,
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          }
+        ])
+      ).to.deep.equal([
+        {
+          op: [],
+          meta: { start_ts: this.ts1, end_ts: this.ts1, user_id: this.user_id },
+          v: 42
+        }
+      ])
+    })
+  })
+
+  describe('compress', function() {
+    describe('insert - insert', function() {
+      it('should append one insert to the other', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 6, i: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foobar' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should insert one insert inside the other', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 5, i: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'fobaro' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should not append separated inserts', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 9, i: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 9, i: 'bar' },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should not append inserts that are too big (second op)', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 6, i: bigstring },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 6, i: bigstring },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should not append inserts that are too big (first op)', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: bigstring },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3 + bigstring.length, i: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: bigstring },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 3 + bigstring.length, i: 'bar' },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      return it('should not append inserts that are too big (first and second op)', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: mediumstring },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3 + mediumstring.length, i: mediumstring },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: mediumstring },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 3 + mediumstring.length, i: mediumstring },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+
+    describe('delete - delete', function() {
+      it('should append one delete to the other', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, d: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3, d: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, d: 'foobar' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should insert one delete inside the other', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, d: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 1, d: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 1, d: 'bafoor' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      return it('should not append separated deletes', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, d: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 9, d: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, d: 'foo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 9, d: 'bar' },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+
+    describe('insert - delete', function() {
+      it('should undo a previous insert', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 5, d: 'o' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'fo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should remove part of an insert from the middle', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'fobaro' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 5, d: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should cancel out two opposite updates', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3, d: 'foo' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: '' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      it('should not combine separated updates', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foo' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 9, d: 'bar' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foo' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 9, d: 'bar' },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      return it('should not combine updates with overlap beyond the end', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, i: 'foobar' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 6, d: 'bardle' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: 'foobar' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 6, d: 'bardle' },
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+
+    describe('delete - insert', function() {
+      it('should do a diff of the content', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, d: 'one two three four five six seven eight' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3, i: 'one 2 three four five six seven eight' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 7, d: 'two' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          },
+          {
+            op: { p: 7, i: '2' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+
+      return it('should return a no-op if the delete and insert are the same', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: { p: 3, d: 'one two three four five six seven eight' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 3, i: 'one two three four five six seven eight' },
+              meta: { ts: this.ts2, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: { p: 3, i: '' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+
+    describe('noop - insert', function() {
+      return it('should leave them untouched', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: this.UpdateCompressor.NOOP,
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 6, i: 'bar' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: this.UpdateCompressor.NOOP,
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 6, i: 'bar' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+
+    return describe('noop - delete', function() {
+      return it('should leave them untouched', function() {
+        return expect(
+          this.UpdateCompressor.compressUpdates([
+            {
+              op: this.UpdateCompressor.NOOP,
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 42
+            },
+            {
+              op: { p: 6, d: 'bar' },
+              meta: { ts: this.ts1, user_id: this.user_id },
+              v: 43
+            }
+          ])
+        ).to.deep.equal([
+          {
+            op: this.UpdateCompressor.NOOP,
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: { p: 6, d: 'bar' },
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+  })
+
+  return describe('compressRawUpdates', function() {
+    return describe('merging in-place with an array op', function() {
+      return it('should not change the existing last updates', function() {
+        return expect(
+          this.UpdateCompressor.compressRawUpdates(
+            {
+              op: [
+                { p: 1000, d: 'hello' },
+                { p: 1000, i: 'HELLO()' }
+              ],
+              meta: {
+                start_ts: this.ts1,
+                end_ts: this.ts1,
+                user_id: this.user_id
+              },
+              v: 42
+            },
+            [
+              {
+                op: [{ p: 1006, i: 'WORLD' }],
+                meta: { ts: this.ts2, user_id: this.user_id },
+                v: 43
+              }
+            ]
+          )
+        ).to.deep.equal([
+          {
+            op: [
+              { p: 1000, d: 'hello' },
+              { p: 1000, i: 'HELLO()' }
+            ],
+            meta: {
+              start_ts: this.ts1,
+              end_ts: this.ts1,
+              user_id: this.user_id
+            },
+            v: 42
+          },
+          {
+            op: [{ p: 1006, i: 'WORLD' }],
+            meta: {
+              start_ts: this.ts2,
+              end_ts: this.ts2,
+              user_id: this.user_id
+            },
+            v: 43
+          }
+        ])
+      })
+    })
+  })
+})
+
+function __range__(left, right, inclusive) {
+  const range = []
+  const ascending = left < right
+  const end = !inclusive ? right : ascending ? right + 1 : right - 1
+  for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
+    range.push(i)
+  }
+  return range
+}