decaffeinate: Convert DiffGenerator.coffee and 17 other files to JS

services/track-changes/app/coffee/DiffGenerator.js

@@ -1,227 +1,293 @@
-ConsistencyError = (message) ->
-	error = new Error(message)
-	error.name = "ConsistencyError"
-	error.__proto__ = ConsistencyError.prototype
-	return error
-ConsistencyError.prototype.__proto__ = Error.prototype
-
-logger = require "logger-sharelatex"
-
-module.exports = DiffGenerator =
-	ConsistencyError: ConsistencyError
-
-	rewindUpdate: (content, update) ->
-		for op, i in update.op by -1 when op.broken isnt true
-			try
-				content = DiffGenerator.rewindOp content, op
-			catch e
-				if e instanceof ConsistencyError and i = update.op.length - 1
-					# catch known case where the last op in an array has been
-					# merged into a later op
-					logger.error {err: e, update, op: JSON.stringify(op)}, "marking op as broken"
-					op.broken = true
-				else
-					throw e # rethrow the execption
-		return content
-
-	rewindOp: (content, op) ->
-		if op.i?
-			# ShareJS will accept an op where p > content.length when applied,
-			# and it applies as though p == content.length. However, the op is
-			# passed to us with the original p > content.length. Detect if that
-			# is the case with this op, and shift p back appropriately to match
-			# ShareJS if so.
-			p = op.p
-			max_p = content.length - op.i.length
-			if p > max_p
-				logger.warn {max_p, p}, "truncating position to content length"
-				p = max_p
-
-			textToBeRemoved = content.slice(p, p + op.i.length)
-			if op.i != textToBeRemoved
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DiffGenerator;
+var ConsistencyError = function(message) {
+	const error = new Error(message);
+	error.name = "ConsistencyError";
+	error.__proto__ = ConsistencyError.prototype;
+	return error;
+};
+ConsistencyError.prototype.__proto__ = Error.prototype;
+
+const logger = require("logger-sharelatex");
+
+module.exports = (DiffGenerator = {
+	ConsistencyError,
+
+	rewindUpdate(content, update) {
+		for (let j = update.op.length - 1, i = j; j >= 0; j--, i = j) {
+			const op = update.op[i];
+			if (op.broken !== true) {
+				try {
+					content = DiffGenerator.rewindOp(content, op);
+				} catch (e) {
+					if (e instanceof ConsistencyError && (i = update.op.length - 1)) {
+						// catch known case where the last op in an array has been
+						// merged into a later op
+						logger.error({err: e, update, op: JSON.stringify(op)}, "marking op as broken");
+						op.broken = true;
+					} else {
+						throw e; // rethrow the execption
+					}
+				}
+			}
+		}
+		return content;
+	},
+
+	rewindOp(content, op) {
+		let p;
+		if (op.i != null) {
+			// ShareJS will accept an op where p > content.length when applied,
+			// and it applies as though p == content.length. However, the op is
+			// passed to us with the original p > content.length. Detect if that
+			// is the case with this op, and shift p back appropriately to match
+			// ShareJS if so.
+			({ p } = op);
+			const max_p = content.length - op.i.length;
+			if (p > max_p) {
+				logger.warn({max_p, p}, "truncating position to content length");
+				p = max_p;
+			}
+
+			const textToBeRemoved = content.slice(p, p + op.i.length);
+			if (op.i !== textToBeRemoved) {
 				throw new ConsistencyError(
-					"Inserted content, '#{op.i}', does not match text to be removed, '#{textToBeRemoved}'"
-				)
+					`Inserted content, '${op.i}', does not match text to be removed, '${textToBeRemoved}'`
+				);
+			}
 
-			return content.slice(0, p) + content.slice(p + op.i.length)
+			return content.slice(0, p) + content.slice(p + op.i.length);
 
-		else if op.d?
-			return content.slice(0, op.p) + op.d + content.slice(op.p)
+		} else if (op.d != null) {
+			return content.slice(0, op.p) + op.d + content.slice(op.p);
 		
-		else
-			return content
-
-	rewindUpdates: (content, updates) ->
-		for update in updates.reverse()
-			try
-				content = DiffGenerator.rewindUpdate(content, update)
-			catch e
-				e.attempted_update = update # keep a record of the attempted update
-				throw e # rethrow the exception
-		return content
-
-	buildDiff: (initialContent, updates) ->
-		diff = [ u: initialContent ]
-		for update in updates
-			diff = DiffGenerator.applyUpdateToDiff diff, update
-		diff = DiffGenerator.compressDiff diff
-		return diff
-
-	compressDiff: (diff) ->
-		newDiff = []
-		for part in diff
-			lastPart = newDiff[newDiff.length - 1]
-			if lastPart? and lastPart.meta?.user? and part.meta?.user?
-				if lastPart.i? and part.i? and lastPart.meta.user.id == part.meta.user.id
-					lastPart.i += part.i
-					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts)
-					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts)
-				else if lastPart.d? and part.d? and lastPart.meta.user.id == part.meta.user.id
-					lastPart.d += part.d
-					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts)
-					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts)
-				else
-					newDiff.push part
-			else
-				newDiff.push part
-		return newDiff
-
-	applyOpToDiff: (diff, op, meta) ->
-		position = 0
-
-		remainingDiff = diff.slice()
-		{consumedDiff, remainingDiff} = DiffGenerator._consumeToOffset(remainingDiff, op.p)
-		newDiff = consumedDiff
-
-		if op.i?
-			newDiff.push
-				i: op.i
-				meta: meta
-		else if op.d?
-			{consumedDiff, remainingDiff} = DiffGenerator._consumeDiffAffectedByDeleteOp remainingDiff, op, meta
-			newDiff.push(consumedDiff...)
-
-		newDiff.push(remainingDiff...)
-
-		return newDiff
-
-	applyUpdateToDiff: (diff, update) ->
-		for op in update.op when op.broken isnt true
-			diff = DiffGenerator.applyOpToDiff diff, op, update.meta
-		return diff
-
-	_consumeToOffset: (remainingDiff, totalOffset) ->
-		consumedDiff = []
-		position = 0
-		while part = remainingDiff.shift()
-			length = DiffGenerator._getLengthOfDiffPart part
-			if part.d?
-				consumedDiff.push part
-			else if position + length >= totalOffset
-				partOffset = totalOffset - position
-				if partOffset > 0
-					consumedDiff.push DiffGenerator._slicePart part, 0, partOffset
-				if partOffset < length
-					remainingDiff.unshift DiffGenerator._slicePart part, partOffset
-				break
-			else
-				position += length
-				consumedDiff.push part
-
-		return {
-			consumedDiff: consumedDiff
-			remainingDiff: remainingDiff
+		} else {
+			return content;
 		}
-
-	_consumeDiffAffectedByDeleteOp: (remainingDiff, deleteOp, meta) ->
-		consumedDiff = []
-		remainingOp = deleteOp
-		while remainingOp and remainingDiff.length > 0
-			{newPart, remainingDiff, remainingOp} = DiffGenerator._consumeDeletedPart remainingDiff, remainingOp, meta
-			consumedDiff.push newPart if newPart?
-		return {
-			consumedDiff: consumedDiff
-			remainingDiff: remainingDiff
+	},
+
+	rewindUpdates(content, updates) {
+		for (let update of Array.from(updates.reverse())) {
+			try {
+				content = DiffGenerator.rewindUpdate(content, update);
+			} catch (e) {
+				e.attempted_update = update; // keep a record of the attempted update
+				throw e; // rethrow the exception
+			}
 		}
+		return content;
+	},
 
-	_consumeDeletedPart: (remainingDiff, op, meta) ->
-		part = remainingDiff.shift()
-		partLength = DiffGenerator._getLengthOfDiffPart part
-
-		if part.d?
-			# Skip existing deletes
-			remainingOp = op
-			newPart = part
-
-		else if partLength > op.d.length
-			# Only the first bit of the part has been deleted
-			remainingPart = DiffGenerator._slicePart part, op.d.length
-			remainingDiff.unshift remainingPart
-
-			deletedContent = DiffGenerator._getContentOfPart(part).slice(0, op.d.length)
-			if deletedContent != op.d
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{op.d}'")
-
-			if part.u?
-				newPart =
-					d: op.d
-					meta: meta
-			else if part.i?
-				newPart = null
-
-			remainingOp = null
-
-		else if partLength == op.d.length
-			# The entire part has been deleted, but it is the last part
-
-			deletedContent = DiffGenerator._getContentOfPart(part)
-			if deletedContent != op.d
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{op.d}'")
-
-			if part.u?
-				newPart =
-					d: op.d
-					meta: meta
-			else if part.i?
-				newPart = null
+	buildDiff(initialContent, updates) {
+		let diff = [ {u: initialContent} ];
+		for (let update of Array.from(updates)) {
+			diff = DiffGenerator.applyUpdateToDiff(diff, update);
+		}
+		diff = DiffGenerator.compressDiff(diff);
+		return diff;
+	},
+
+	compressDiff(diff) {
+		const newDiff = [];
+		for (let part of Array.from(diff)) {
+			const lastPart = newDiff[newDiff.length - 1];
+			if ((lastPart != null) && ((lastPart.meta != null ? lastPart.meta.user : undefined) != null) && ((part.meta != null ? part.meta.user : undefined) != null)) {
+				if ((lastPart.i != null) && (part.i != null) && (lastPart.meta.user.id === part.meta.user.id)) {
+					lastPart.i += part.i;
+					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts);
+					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts);
+				} else if ((lastPart.d != null) && (part.d != null) && (lastPart.meta.user.id === part.meta.user.id)) {
+					lastPart.d += part.d;
+					lastPart.meta.start_ts = Math.min(lastPart.meta.start_ts, part.meta.start_ts);
+					lastPart.meta.end_ts = Math.max(lastPart.meta.end_ts, part.meta.end_ts);
+				} else {
+					newDiff.push(part);
+				}
+			} else {
+				newDiff.push(part);
+			}
+		}
+		return newDiff;
+	},
+
+	applyOpToDiff(diff, op, meta) {
+		let consumedDiff;
+		const position = 0;
+
+		let remainingDiff = diff.slice();
+		({consumedDiff, remainingDiff} = DiffGenerator._consumeToOffset(remainingDiff, op.p));
+		const newDiff = consumedDiff;
+
+		if (op.i != null) {
+			newDiff.push({
+				i: op.i,
+				meta
+			});
+		} else if (op.d != null) {
+			({consumedDiff, remainingDiff} = DiffGenerator._consumeDiffAffectedByDeleteOp(remainingDiff, op, meta));
+			newDiff.push(...Array.from(consumedDiff || []));
+		}
 
-			remainingOp = null
+		newDiff.push(...Array.from(remainingDiff || []));
 
-		else if partLength < op.d.length
-			# The entire part has been deleted and there is more
+		return newDiff;
+	},
 
-			deletedContent = DiffGenerator._getContentOfPart(part)
-			opContent = op.d.slice(0, deletedContent.length)
-			if deletedContent != opContent
-				throw new ConsistencyError("deleted content, '#{deletedContent}', does not match delete op, '#{opContent}'")
+	applyUpdateToDiff(diff, update) {
+		for (let op of Array.from(update.op)) {
+			if (op.broken !== true) {
+				diff = DiffGenerator.applyOpToDiff(diff, op, update.meta);
+			}
+		}
+		return diff;
+	},
+
+	_consumeToOffset(remainingDiff, totalOffset) {
+		let part;
+		const consumedDiff = [];
+		let position = 0;
+		while ((part = remainingDiff.shift())) {
+			const length = DiffGenerator._getLengthOfDiffPart(part);
+			if (part.d != null) {
+				consumedDiff.push(part);
+			} else if ((position + length) >= totalOffset) {
+				const partOffset = totalOffset - position;
+				if (partOffset > 0) {
+					consumedDiff.push(DiffGenerator._slicePart(part, 0, partOffset));
+				}
+				if (partOffset < length) {
+					remainingDiff.unshift(DiffGenerator._slicePart(part, partOffset));
+				}
+				break;
+			} else {
+				position += length;
+				consumedDiff.push(part);
+			}
+		}
 
-			if part.u
-				newPart =
-					d: part.u
-					meta: meta
-			else if part.i?
-				newPart = null
+		return {
+			consumedDiff,
+			remainingDiff
+		};
+	},
+
+	_consumeDiffAffectedByDeleteOp(remainingDiff, deleteOp, meta) {
+		const consumedDiff = [];
+		let remainingOp = deleteOp;
+		while (remainingOp && (remainingDiff.length > 0)) {
+			let newPart;
+			({newPart, remainingDiff, remainingOp} = DiffGenerator._consumeDeletedPart(remainingDiff, remainingOp, meta));
+			if (newPart != null) { consumedDiff.push(newPart); }
+		}
+		return {
+			consumedDiff,
+			remainingDiff
+		};
+	},
+
+	_consumeDeletedPart(remainingDiff, op, meta) {
+		let deletedContent, newPart, remainingOp;
+		const part = remainingDiff.shift();
+		const partLength = DiffGenerator._getLengthOfDiffPart(part);
+
+		if (part.d != null) {
+			// Skip existing deletes
+			remainingOp = op;
+			newPart = part;
+
+		} else if (partLength > op.d.length) {
+			// Only the first bit of the part has been deleted
+			const remainingPart = DiffGenerator._slicePart(part, op.d.length);
+			remainingDiff.unshift(remainingPart);
+
+			deletedContent = DiffGenerator._getContentOfPart(part).slice(0, op.d.length);
+			if (deletedContent !== op.d) {
+				throw new ConsistencyError(`deleted content, '${deletedContent}', does not match delete op, '${op.d}'`);
+			}
+
+			if (part.u != null) {
+				newPart = {
+					d: op.d,
+					meta
+				};
+			} else if (part.i != null) {
+				newPart = null;
+			}
+
+			remainingOp = null;
+
+		} else if (partLength === op.d.length) {
+			// The entire part has been deleted, but it is the last part
+
+			deletedContent = DiffGenerator._getContentOfPart(part);
+			if (deletedContent !== op.d) {
+				throw new ConsistencyError(`deleted content, '${deletedContent}', does not match delete op, '${op.d}'`);
+			}
+
+			if (part.u != null) {
+				newPart = {
+					d: op.d,
+					meta
+				};
+			} else if (part.i != null) {
+				newPart = null;
+			}
+
+			remainingOp = null;
+
+		} else if (partLength < op.d.length) {
+			// The entire part has been deleted and there is more
+
+			deletedContent = DiffGenerator._getContentOfPart(part);
+			const opContent = op.d.slice(0, deletedContent.length);
+			if (deletedContent !== opContent) {
+				throw new ConsistencyError(`deleted content, '${deletedContent}', does not match delete op, '${opContent}'`);
+			}
+
+			if (part.u) {
+				newPart = {
+					d: part.u,
+					meta
+				};
+			} else if (part.i != null) {
+				newPart = null;
+			}
 
 			remainingOp =
-				p: op.p, d: op.d.slice(DiffGenerator._getLengthOfDiffPart(part))
+				{p: op.p, d: op.d.slice(DiffGenerator._getLengthOfDiffPart(part))};
+		}
 
 		return {
-			newPart: newPart
-			remainingDiff: remainingDiff
-			remainingOp: remainingOp
+			newPart,
+			remainingDiff,
+			remainingOp
+		};
+	},
+
+	_slicePart(basePart, from, to) {
+		let part;
+		if (basePart.u != null) {
+			part = { u: basePart.u.slice(from, to) };
+		} else if (basePart.i != null) {
+			part = { i: basePart.i.slice(from, to) };
 		}
+		if (basePart.meta != null) {
+			part.meta = basePart.meta;
+		}
+		return part;
+	},
 
-	_slicePart: (basePart, from, to) ->
-		if basePart.u?
-			part = { u: basePart.u.slice(from, to) }
-		else if basePart.i?
-			part = { i: basePart.i.slice(from, to) }
-		if basePart.meta?
-			part.meta = basePart.meta
-		return part
-
-	_getLengthOfDiffPart: (part) ->
-		(part.u or part.d or part.i or '').length
+	_getLengthOfDiffPart(part) {
+		return (part.u || part.d || part.i || '').length;
+	},
 
-	_getContentOfPart: (part) ->
-		part.u or part.d or part.i or ''
+	_getContentOfPart(part) {
+		return part.u || part.d || part.i || '';
+	}
+});

services/track-changes/app/coffee/DiffManager.js

@@ -1,88 +1,128 @@
-UpdatesManager = require "./UpdatesManager"
-DocumentUpdaterManager = require "./DocumentUpdaterManager"
-DiffGenerator = require "./DiffGenerator"
-logger = require "logger-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DiffManager;
+const UpdatesManager = require("./UpdatesManager");
+const DocumentUpdaterManager = require("./DocumentUpdaterManager");
+const DiffGenerator = require("./DiffGenerator");
+const logger = require("logger-sharelatex");
 
-module.exports = DiffManager =
-	getLatestDocAndUpdates: (project_id, doc_id, fromVersion, callback = (error, content, version, updates) ->) ->
-		# Get updates last, since then they must be ahead and it
-		# might be possible to rewind to the same version as the doc.
-		DocumentUpdaterManager.getDocument project_id, doc_id, (error, content, version) ->
-			return callback(error) if error?
-			if !fromVersion? # If we haven't been given a version, just return lastest doc and no updates
-				return callback(null, content, version, [])
-			UpdatesManager.getDocUpdatesWithUserInfo project_id, doc_id, from: fromVersion, (error, updates) ->
-				return callback(error) if error?
-				callback(null, content, version, updates)
+module.exports = (DiffManager = {
+	getLatestDocAndUpdates(project_id, doc_id, fromVersion, callback) {
+		// Get updates last, since then they must be ahead and it
+		// might be possible to rewind to the same version as the doc.
+		if (callback == null) { callback = function(error, content, version, updates) {}; }
+		return DocumentUpdaterManager.getDocument(project_id, doc_id, function(error, content, version) {
+			if (error != null) { return callback(error); }
+			if ((fromVersion == null)) { // If we haven't been given a version, just return lastest doc and no updates
+				return callback(null, content, version, []);
+			}
+			return UpdatesManager.getDocUpdatesWithUserInfo(project_id, doc_id, {from: fromVersion}, function(error, updates) {
+				if (error != null) { return callback(error); }
+				return callback(null, content, version, updates);
+			});
+		});
+	},
 	
-	getDiff: (project_id, doc_id, fromVersion, toVersion, callback = (error, diff) ->) ->
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, fromVersion, (error, startingContent, updates) ->
-			if error?
-				if error.message == "broken-history"
-					return callback(null, "history unavailable")
-				else
-					return callback(error)
+	getDiff(project_id, doc_id, fromVersion, toVersion, callback) {
+		if (callback == null) { callback = function(error, diff) {}; }
+		return DiffManager.getDocumentBeforeVersion(project_id, doc_id, fromVersion, function(error, startingContent, updates) {
+			let diff;
+			if (error != null) {
+				if (error.message === "broken-history") {
+					return callback(null, "history unavailable");
+				} else {
+					return callback(error);
+				}
+			}
 
-			updatesToApply = []
-			for update in updates.slice().reverse()
-				if update.v <= toVersion
-					updatesToApply.push update
+			const updatesToApply = [];
+			for (let update of Array.from(updates.slice().reverse())) {
+				if (update.v <= toVersion) {
+					updatesToApply.push(update);
+				}
+			}
 
-			try
-				diff = DiffGenerator.buildDiff startingContent, updatesToApply
-			catch e
-				return callback(e)
+			try {
+				diff = DiffGenerator.buildDiff(startingContent, updatesToApply);
+			} catch (e) {
+				return callback(e);
+			}
 			
-			callback(null, diff)
+			return callback(null, diff);
+		});
+	},
 
-	getDocumentBeforeVersion: (project_id, doc_id, version, _callback = (error, document, rewoundUpdates) ->) ->
-		# Whichever order we get the latest document and the latest updates,
-		# there is potential for updates to be applied between them so that
-		# they do not return the same 'latest' versions.
-		# If this happens, we just retry and hopefully get them at the compatible
-		# versions.
-		retries = 3
-		callback = (error, args...) ->
-			if error?
-				if error.retry and retries > 0
-					logger.warn {error, project_id, doc_id, version, retries}, "retrying getDocumentBeforeVersion"
-					retry()
-				else
-					_callback(error)
-			else
-				_callback(null, args...)
+	getDocumentBeforeVersion(project_id, doc_id, version, _callback) {
+		// Whichever order we get the latest document and the latest updates,
+		// there is potential for updates to be applied between them so that
+		// they do not return the same 'latest' versions.
+		// If this happens, we just retry and hopefully get them at the compatible
+		// versions.
+		let retry;
+		if (_callback == null) { _callback = function(error, document, rewoundUpdates) {}; }
+		let retries = 3;
+		const callback = function(error, ...args) {
+			if (error != null) {
+				if (error.retry && (retries > 0)) {
+					logger.warn({error, project_id, doc_id, version, retries}, "retrying getDocumentBeforeVersion");
+					return retry();
+				} else {
+					return _callback(error);
+				}
+			} else {
+				return _callback(null, ...Array.from(args));
+			}
+		};
 
-		do retry = () ->
-			retries--
-			DiffManager._tryGetDocumentBeforeVersion(project_id, doc_id, version, callback)
+		return (retry = function() {
+			retries--;
+			return DiffManager._tryGetDocumentBeforeVersion(project_id, doc_id, version, callback);
+		})();
+	},
 
-	_tryGetDocumentBeforeVersion: (project_id, doc_id, version, callback = (error, document, rewoundUpdates) ->) ->
-		logger.log project_id: project_id, doc_id: doc_id, version: version, "getting document before version"
-		DiffManager.getLatestDocAndUpdates project_id, doc_id, version, (error, content, version, updates) ->
-			return callback(error) if error?
+	_tryGetDocumentBeforeVersion(project_id, doc_id, version, callback) {
+		if (callback == null) { callback = function(error, document, rewoundUpdates) {}; }
+		logger.log({project_id, doc_id, version}, "getting document before version");
+		return DiffManager.getLatestDocAndUpdates(project_id, doc_id, version, function(error, content, version, updates) {
+			let startingContent;
+			if (error != null) { return callback(error); }
 
-			# bail out if we hit a broken update
-			for u in updates when u.broken
-				return callback new Error "broken-history"
+			// bail out if we hit a broken update
+			for (let u of Array.from(updates)) {
+				if (u.broken) {
+					return callback(new Error("broken-history"));
+				}
+			}
 
-			# discard any updates which are ahead of this document version
-			while updates[0]?.v >= version
-				updates.shift()
+			// discard any updates which are ahead of this document version
+			while ((updates[0] != null ? updates[0].v : undefined) >= version) {
+				updates.shift();
+			}
 
-			lastUpdate = updates[0]
-			if lastUpdate? and lastUpdate.v != version - 1
-				error = new Error("latest update version, #{lastUpdate.v}, does not match doc version, #{version}")
-				error.retry = true
-				return callback error
+			const lastUpdate = updates[0];
+			if ((lastUpdate != null) && (lastUpdate.v !== (version - 1))) {
+				error = new Error(`latest update version, ${lastUpdate.v}, does not match doc version, ${version}`);
+				error.retry = true;
+				return callback(error);
+			}
 			
-			logger.log {docVersion: version, lastUpdateVersion: lastUpdate?.v, updateCount: updates.length}, "rewinding updates"
+			logger.log({docVersion: version, lastUpdateVersion: (lastUpdate != null ? lastUpdate.v : undefined), updateCount: updates.length}, "rewinding updates");
 
-			tryUpdates = updates.slice().reverse()
+			const tryUpdates = updates.slice().reverse();
 
-			try
-				startingContent = DiffGenerator.rewindUpdates content, tryUpdates
-				# tryUpdates is reversed, and any unapplied ops are marked as broken
-			catch e
-				return callback(e)
+			try {
+				startingContent = DiffGenerator.rewindUpdates(content, tryUpdates);
+				// tryUpdates is reversed, and any unapplied ops are marked as broken
+			} catch (e) {
+				return callback(e);
+			}
 
-			callback(null, startingContent, tryUpdates)
+			return callback(null, startingContent, tryUpdates);
+		});
+	}
+});

services/track-changes/app/coffee/DocumentUpdaterManager.js

@@ -1,42 +1,63 @@
-request = require "request"
-logger = require "logger-sharelatex"
-Settings = require "settings-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocumentUpdaterManager;
+const request = require("request");
+const logger = require("logger-sharelatex");
+const Settings = require("settings-sharelatex");
 
-module.exports = DocumentUpdaterManager =
-	getDocument: (project_id, doc_id, callback = (error, content, version) ->) ->
-		url = "#{Settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}"
-		logger.log project_id:project_id, doc_id: doc_id, "getting doc from document updater"
-		request.get url, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode >= 200 and res.statusCode < 300
-				try
-					body = JSON.parse(body)
-				catch error
-					return callback(error)
-				logger.log {project_id, doc_id, version: body.version}, "got doc from document updater"
-				callback null, body.lines.join("\n"), body.version
-			else
-				error = new Error("doc updater returned a non-success status code: #{res.statusCode}")
-				logger.error err: error, project_id:project_id, doc_id:doc_id, url: url, "error accessing doc updater"
-				callback error
+module.exports = (DocumentUpdaterManager = {
+	getDocument(project_id, doc_id, callback) {
+		if (callback == null) { callback = function(error, content, version) {}; }
+		const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`;
+		logger.log({project_id, doc_id}, "getting doc from document updater");
+		return request.get(url, function(error, res, body){
+			if (error != null) {
+				return callback(error);
+			}
+			if ((res.statusCode >= 200) && (res.statusCode < 300)) {
+				try {
+					body = JSON.parse(body);
+				} catch (error1) {
+					error = error1;
+					return callback(error);
+				}
+				logger.log({project_id, doc_id, version: body.version}, "got doc from document updater");
+				return callback(null, body.lines.join("\n"), body.version);
+			} else {
+				error = new Error(`doc updater returned a non-success status code: ${res.statusCode}`);
+				logger.error({err: error, project_id, doc_id, url}, "error accessing doc updater");
+				return callback(error);
+			}
+		});
+	},
 
-	setDocument: (project_id, doc_id, content, user_id, callback = (error) ->) ->
-		url = "#{Settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}"
-		logger.log project_id:project_id, doc_id: doc_id, "setting doc in document updater"
-		request.post {
-			url: url
-			json:
-				lines: content.split("\n")
-				source: "restore"
-				user_id: user_id
+	setDocument(project_id, doc_id, content, user_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`;
+		logger.log({project_id, doc_id}, "setting doc in document updater");
+		return request.post({
+			url,
+			json: {
+				lines: content.split("\n"),
+				source: "restore",
+				user_id,
 				undoing: true
-		}, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode >= 200 and res.statusCode < 300
-				callback null
-			else
-				error = new Error("doc updater returned a non-success status code: #{res.statusCode}")
-				logger.error err: error, project_id:project_id, doc_id:doc_id, url: url, "error accessing doc updater"
-				callback error
+			}
+		}, function(error, res, body){
+			if (error != null) {
+				return callback(error);
+			}
+			if ((res.statusCode >= 200) && (res.statusCode < 300)) {
+				return callback(null);
+			} else {
+				error = new Error(`doc updater returned a non-success status code: ${res.statusCode}`);
+				logger.error({err: error, project_id, doc_id, url}, "error accessing doc updater");
+				return callback(error);
+			}
+		});
+	}
+});

services/track-changes/app/coffee/HealthChecker.js

@@ -1,46 +1,64 @@
-ObjectId = require("mongojs").ObjectId
-request = require("request")
-async = require("async")
-settings = require("settings-sharelatex")
-port = settings.internal.trackchanges.port
-logger = require "logger-sharelatex"
-LockManager = require "./LockManager"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { ObjectId } = require("mongojs");
+const request = require("request");
+const async = require("async");
+const settings = require("settings-sharelatex");
+const { port } = settings.internal.trackchanges;
+const logger = require("logger-sharelatex");
+const LockManager = require("./LockManager");
 
-module.exports =
-	check : (callback)->
-		project_id = ObjectId(settings.trackchanges.healthCheck.project_id)
-		url = "http://localhost:#{port}/project/#{project_id}"
-		logger.log project_id:project_id, "running health check"
-		jobs = [
-			(cb)->
-				request.get {url:"http://localhost:#{port}/check_lock", timeout:3000}, (err, res, body) ->
-					if err?
-						logger.err err:err, project_id:project_id, "error checking lock for health check"
-						cb(err)
-					else if res?.statusCode != 200
-						cb("status code not 200, it's #{res.statusCode}")
-					else
-						cb()
-			(cb)->
-				request.post {url:"#{url}/flush", timeout:10000}, (err, res, body) ->
-					if err?
-						logger.err err:err, project_id:project_id, "error flushing for health check"
-						cb(err)
-					else if res?.statusCode != 204
-						cb("status code not 204, it's #{res.statusCode}")
-					else
-						cb()
-			(cb)->
-				request.get {url:"#{url}/updates", timeout:10000}, (err, res, body)->
-					if err?
-						logger.err err:err, project_id:project_id, "error getting updates for health check"
-						cb(err)
-					else if res?.statusCode != 200
-						cb("status code not 200, it's #{res.statusCode}")
-					else
-						cb()
-		]
-		async.series jobs, callback
+module.exports = {
+	check(callback){
+		const project_id = ObjectId(settings.trackchanges.healthCheck.project_id);
+		const url = `http://localhost:${port}/project/${project_id}`;
+		logger.log({project_id}, "running health check");
+		const jobs = [
+			cb=>
+				request.get({url:`http://localhost:${port}/check_lock`, timeout:3000}, function(err, res, body) {
+					if (err != null) {
+						logger.err({err, project_id}, "error checking lock for health check");
+						return cb(err);
+					} else if ((res != null ? res.statusCode : undefined) !== 200) {
+						return cb(`status code not 200, it's ${res.statusCode}`);
+					} else {
+						return cb();
+					}
+				})
+			,
+			cb=>
+				request.post({url:`${url}/flush`, timeout:10000}, function(err, res, body) {
+					if (err != null) {
+						logger.err({err, project_id}, "error flushing for health check");
+						return cb(err);
+					} else if ((res != null ? res.statusCode : undefined) !== 204) {
+						return cb(`status code not 204, it's ${res.statusCode}`);
+					} else {
+						return cb();
+					}
+				})
+			,
+			cb=>
+				request.get({url:`${url}/updates`, timeout:10000}, function(err, res, body){
+					if (err != null) {
+						logger.err({err, project_id}, "error getting updates for health check");
+						return cb(err);
+					} else if ((res != null ? res.statusCode : undefined) !== 200) {
+						return cb(`status code not 200, it's ${res.statusCode}`);
+					} else {
+						return cb();
+					}
+				})
+			
+		];
+		return async.series(jobs, callback);
+	},
 
-	checkLock: (callback) ->
-		LockManager.healthCheck callback
+	checkLock(callback) {
+		return LockManager.healthCheck(callback);
+	}
+};

services/track-changes/app/coffee/HttpController.js

@@ -1,137 +1,195 @@
-UpdatesManager = require "./UpdatesManager"
-DiffManager = require "./DiffManager"
-PackManager = require "./PackManager"
-RestoreManager = require "./RestoreManager"
-logger = require "logger-sharelatex"
-HealthChecker = require "./HealthChecker"
-_ = require "underscore"
-
-module.exports = HttpController =
-	flushDoc: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-		logger.log project_id: project_id, doc_id: doc_id, "compressing doc history"
-		UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	flushProject: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		logger.log project_id: project_id, "compressing project history"
-		UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	flushAll: (req, res, next = (error) ->) ->
-		# limit on projects to flush or -1 for all (default)
-		limit = if req.query.limit? then parseInt(req.query.limit, 10) else -1
-		logger.log {limit: limit}, "flushing all projects"
-		UpdatesManager.flushAll limit, (error, result) ->
-			return next(error) if error?
-			{failed, succeeded, all} = result
-			status = "#{succeeded.length} succeeded, #{failed.length} failed"
-			if limit == 0
-				res.status(200).send "#{status}\nwould flush:\n#{all.join('\n')}\n"
-			else if failed.length > 0
-				logger.log {failed: failed, succeeded: succeeded}, "error flushing projects"
-				res.status(500).send "#{status}\nfailed to flush:\n#{failed.join('\n')}\n"
-			else
-				res.status(200).send "#{status}\nflushed #{succeeded.length} projects of #{all.length}\n"
-
-	checkDanglingUpdates: (req, res, next = (error) ->) ->
-		logger.log "checking dangling updates"
-		UpdatesManager.getDanglingUpdates (error, result) ->
-			return next(error) if error?
-			if result.length > 0
-				logger.log {dangling: result}, "found dangling updates"
-				res.status(500).send "dangling updates:\n#{result.join('\n')}\n"
-			else
-				res.status(200).send "no dangling updates found\n"
-
-	checkDoc: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-		logger.log project_id: project_id, doc_id: doc_id, "checking doc history"
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, 1, (error, document, rewoundUpdates) ->
-			return next(error) if error?
-			broken = []
-			for update in rewoundUpdates
-				for op in update.op when op.broken is true
-					broken.push op
-			if broken.length > 0
-				res.send broken
-			else
-				res.send 204
-
-	getDiff: (req, res, next = (error) ->) ->
-		doc_id = req.params.doc_id
-		project_id = req.params.project_id
-
-		if req.query.from?
-			from = parseInt(req.query.from, 10)
-		else
-			from = null
-		if req.query.to?
-			to = parseInt(req.query.to, 10)
-		else
-			to = null
-
-		logger.log {project_id, doc_id, from, to}, "getting diff"
-		DiffManager.getDiff project_id, doc_id, from, to, (error, diff) ->
-			return next(error) if error?
-			res.json {diff: diff}
-
-	getUpdates: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-
-		if req.query.before?
-			before = parseInt(req.query.before, 10)
-		if req.query.min_count?
-			min_count = parseInt(req.query.min_count, 10)
-
-		UpdatesManager.getSummarizedProjectUpdates project_id, before: before, min_count: min_count, (error, updates, nextBeforeTimestamp) ->
-			return next(error) if error?
-			res.json {
-				updates: updates
-				nextBeforeTimestamp: nextBeforeTimestamp
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HttpController;
+const UpdatesManager = require("./UpdatesManager");
+const DiffManager = require("./DiffManager");
+const PackManager = require("./PackManager");
+const RestoreManager = require("./RestoreManager");
+const logger = require("logger-sharelatex");
+const HealthChecker = require("./HealthChecker");
+const _ = require("underscore");
+
+module.exports = (HttpController = {
+	flushDoc(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		const { doc_id } = req.params;
+		const { project_id } = req.params;
+		logger.log({project_id, doc_id}, "compressing doc history");
+		return UpdatesManager.processUncompressedUpdatesWithLock(project_id, doc_id, function(error) {
+			if (error != null) { return next(error); }
+			return res.send(204);
+		});
+	},
+
+	flushProject(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		const { project_id } = req.params;
+		logger.log({project_id}, "compressing project history");
+		return UpdatesManager.processUncompressedUpdatesForProject(project_id, function(error) {
+			if (error != null) { return next(error); }
+			return res.send(204);
+		});
+	},
+
+	flushAll(req, res, next) {
+		// limit on projects to flush or -1 for all (default)
+		if (next == null) { next = function(error) {}; }
+		const limit = (req.query.limit != null) ? parseInt(req.query.limit, 10) : -1;
+		logger.log({limit}, "flushing all projects");
+		return UpdatesManager.flushAll(limit, function(error, result) {
+			if (error != null) { return next(error); }
+			const {failed, succeeded, all} = result;
+			const status = `${succeeded.length} succeeded, ${failed.length} failed`;
+			if (limit === 0) {
+				return res.status(200).send(`${status}\nwould flush:\n${all.join('\n')}\n`);
+			} else if (failed.length > 0) {
+				logger.log({failed, succeeded}, "error flushing projects");
+				return res.status(500).send(`${status}\nfailed to flush:\n${failed.join('\n')}\n`);
+			} else {
+				return res.status(200).send(`${status}\nflushed ${succeeded.length} projects of ${all.length}\n`);
 			}
-
-	restore: (req, res, next = (error) ->) ->
-		{doc_id, project_id, version} = req.params
-		user_id = req.headers["x-user-id"]
-		version = parseInt(version, 10)
-		RestoreManager.restoreToBeforeVersion project_id, doc_id, version, user_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	pushDocHistory: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id = req.params.doc_id
-		logger.log {project_id, doc_id}, "pushing all finalised changes to s3"
-		PackManager.pushOldPacks project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	pullDocHistory: (req, res, next = (error) ->) ->
-		project_id = req.params.project_id
-		doc_id = req.params.doc_id
-		logger.log {project_id, doc_id}, "pulling all packs from s3"
-		PackManager.pullOldPacks project_id, doc_id, (error) ->
-			return next(error) if error?
-			res.send 204
-
-	healthCheck: (req, res)->
-		HealthChecker.check (err)->
-			if err?
-				logger.err err:err, "error performing health check"
-				res.send 500
-			else
-				res.send 200
-
-	checkLock: (req, res)->
-		HealthChecker.checkLock (err) ->
-			if err?
-				logger.err err:err, "error performing lock check"
-				res.send 500
-			else
-				res.send 200
+		});
+	},
+
+	checkDanglingUpdates(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		logger.log("checking dangling updates");
+		return UpdatesManager.getDanglingUpdates(function(error, result) {
+			if (error != null) { return next(error); }
+			if (result.length > 0) {
+				logger.log({dangling: result}, "found dangling updates");
+				return res.status(500).send(`dangling updates:\n${result.join('\n')}\n`);
+			} else {
+				return res.status(200).send("no dangling updates found\n");
+			}
+		});
+	},
+
+	checkDoc(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		const { doc_id } = req.params;
+		const { project_id } = req.params;
+		logger.log({project_id, doc_id}, "checking doc history");
+		return DiffManager.getDocumentBeforeVersion(project_id, doc_id, 1, function(error, document, rewoundUpdates) {
+			if (error != null) { return next(error); }
+			const broken = [];
+			for (let update of Array.from(rewoundUpdates)) {
+				for (let op of Array.from(update.op)) {
+					if (op.broken === true) {
+						broken.push(op);
+					}
+				}
+			}
+			if (broken.length > 0) {
+				return res.send(broken);
+			} else {
+				return res.send(204);
+			}
+		});
+	},
+
+	getDiff(req, res, next) {
+		let from, to;
+		if (next == null) { next = function(error) {}; }
+		const { doc_id } = req.params;
+		const { project_id } = req.params;
+
+		if (req.query.from != null) {
+			from = parseInt(req.query.from, 10);
+		} else {
+			from = null;
+		}
+		if (req.query.to != null) {
+			to = parseInt(req.query.to, 10);
+		} else {
+			to = null;
+		}
+
+		logger.log({project_id, doc_id, from, to}, "getting diff");
+		return DiffManager.getDiff(project_id, doc_id, from, to, function(error, diff) {
+			if (error != null) { return next(error); }
+			return res.json({diff});
+	});
+	},
+
+	getUpdates(req, res, next) {
+		let before, min_count;
+		if (next == null) { next = function(error) {}; }
+		const { project_id } = req.params;
+
+		if (req.query.before != null) {
+			before = parseInt(req.query.before, 10);
+		}
+		if (req.query.min_count != null) {
+			min_count = parseInt(req.query.min_count, 10);
+		}
+
+		return UpdatesManager.getSummarizedProjectUpdates(project_id, {before, min_count}, function(error, updates, nextBeforeTimestamp) {
+			if (error != null) { return next(error); }
+			return res.json({
+				updates,
+				nextBeforeTimestamp
+			});
+	});
+	},
+
+	restore(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		let {doc_id, project_id, version} = req.params;
+		const user_id = req.headers["x-user-id"];
+		version = parseInt(version, 10);
+		return RestoreManager.restoreToBeforeVersion(project_id, doc_id, version, user_id, function(error) {
+			if (error != null) { return next(error); }
+			return res.send(204);
+		});
+	},
+
+	pushDocHistory(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		const { project_id } = req.params;
+		const { doc_id } = req.params;
+		logger.log({project_id, doc_id}, "pushing all finalised changes to s3");
+		return PackManager.pushOldPacks(project_id, doc_id, function(error) {
+			if (error != null) { return next(error); }
+			return res.send(204);
+		});
+	},
+
+	pullDocHistory(req, res, next) {
+		if (next == null) { next = function(error) {}; }
+		const { project_id } = req.params;
+		const { doc_id } = req.params;
+		logger.log({project_id, doc_id}, "pulling all packs from s3");
+		return PackManager.pullOldPacks(project_id, doc_id, function(error) {
+			if (error != null) { return next(error); }
+			return res.send(204);
+		});
+	},
+
+	healthCheck(req, res){
+		return HealthChecker.check(function(err){
+			if (err != null) {
+				logger.err({err}, "error performing health check");
+				return res.send(500);
+			} else {
+				return res.send(200);
+			}
+		});
+	},
+
+	checkLock(req, res){
+		return HealthChecker.checkLock(function(err) {
+			if (err != null) {
+				logger.err({err}, "error performing lock check");
+				return res.send(500);
+			} else {
+				return res.send(200);
+			}
+		});
+	}
+});

services/track-changes/app/coffee/LockManager.js

@@ -1,85 +1,119 @@
-Settings = require "settings-sharelatex"
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.lock)
-os = require "os"
-crypto = require "crypto"
-logger = require "logger-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let LockManager;
+const Settings = require("settings-sharelatex");
+const redis = require("redis-sharelatex");
+const rclient = redis.createClient(Settings.redis.lock);
+const os = require("os");
+const crypto = require("crypto");
+const logger = require("logger-sharelatex");
 
-HOST = os.hostname()
-PID = process.pid
-RND = crypto.randomBytes(4).toString('hex')
-COUNT = 0
+const HOST = os.hostname();
+const PID = process.pid;
+const RND = crypto.randomBytes(4).toString('hex');
+let COUNT = 0;
 
-module.exports = LockManager =
-	LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock
-	MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock
-	LOCK_TTL: 300 # seconds (allow 5 minutes for any operation to complete)
+module.exports = (LockManager = {
+	LOCK_TEST_INTERVAL: 50, // 50ms between each test of the lock
+	MAX_LOCK_WAIT_TIME: 10000, // 10s maximum time to spend trying to get the lock
+	LOCK_TTL: 300, // seconds (allow 5 minutes for any operation to complete)
 
-	# Use a signed lock value as described in
-	# http://redis.io/topics/distlock#correct-implementation-with-a-single-instance
-	# to prevent accidental unlocking by multiple processes
-	randomLock : () ->
-		time = Date.now()
-		return "locked:host=#{HOST}:pid=#{PID}:random=#{RND}:time=#{time}:count=#{COUNT++}"
+	// Use a signed lock value as described in
+	// http://redis.io/topics/distlock#correct-implementation-with-a-single-instance
+	// to prevent accidental unlocking by multiple processes
+	randomLock() {
+		const time = Date.now();
+		return `locked:host=${HOST}:pid=${PID}:random=${RND}:time=${time}:count=${COUNT++}`;
+	},
 
-	unlockScript: 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
+	unlockScript: 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end',
 
-	tryLock : (key, callback = (err, gotLock) ->) ->
-		lockValue = LockManager.randomLock()
-		rclient.set key, lockValue, "EX", @LOCK_TTL, "NX", (err, gotLock)->
-			return callback(err) if err?
-			if gotLock == "OK"
-				callback err, true, lockValue
-			else
-				callback err, false
+	tryLock(key, callback) {
+		if (callback == null) { callback = function(err, gotLock) {}; }
+		const lockValue = LockManager.randomLock();
+		return rclient.set(key, lockValue, "EX", this.LOCK_TTL, "NX", function(err, gotLock){
+			if (err != null) { return callback(err); }
+			if (gotLock === "OK") {
+				return callback(err, true, lockValue);
+			} else {
+				return callback(err, false);
+			}
+		});
+	},
 
-	getLock: (key, callback = (error) ->) ->
-		startTime = Date.now()
-		do attempt = () ->
-			if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
-				e = new Error("Timeout")
-				e.key = key
-				return callback(e)
+	getLock(key, callback) {
+		let attempt;
+		if (callback == null) { callback = function(error) {}; }
+		const startTime = Date.now();
+		return (attempt = function() {
+			if ((Date.now() - startTime) > LockManager.MAX_LOCK_WAIT_TIME) {
+				const e = new Error("Timeout");
+				e.key = key;
+				return callback(e);
+			}
 
-			LockManager.tryLock key, (error, gotLock, lockValue) ->
-				return callback(error) if error?
-				if gotLock
-					callback(null, lockValue)
-				else
-					setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
+			return LockManager.tryLock(key, function(error, gotLock, lockValue) {
+				if (error != null) { return callback(error); }
+				if (gotLock) {
+					return callback(null, lockValue);
+				} else {
+					return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL);
+				}
+			});
+		})();
+	},
 
-	checkLock: (key, callback = (err, isFree) ->) ->
-		rclient.exists key, (err, exists) ->
-			return callback(err) if err?
-			exists = parseInt exists
-			if exists == 1
-				callback err, false
-			else
-				callback err, true
+	checkLock(key, callback) {
+		if (callback == null) { callback = function(err, isFree) {}; }
+		return rclient.exists(key, function(err, exists) {
+			if (err != null) { return callback(err); }
+			exists = parseInt(exists);
+			if (exists === 1) {
+				return callback(err, false);
+			} else {
+				return callback(err, true);
+			}
+		});
+	},
 
-	releaseLock: (key, lockValue, callback) ->
-		rclient.eval LockManager.unlockScript, 1, key, lockValue, (err, result) ->
-			if err?
-				return callback(err)
-			if result? and result isnt 1 # successful unlock should release exactly one key
-				logger.error {key:key, lockValue:lockValue, redis_err:err, redis_result:result}, "unlocking error"
-				return callback(new Error("tried to release timed out lock"))
-			callback(err,result)
+	releaseLock(key, lockValue, callback) {
+		return rclient.eval(LockManager.unlockScript, 1, key, lockValue, function(err, result) {
+			if (err != null) {
+				return callback(err);
+			}
+			if ((result != null) && (result !== 1)) { // successful unlock should release exactly one key
+				logger.error({key, lockValue, redis_err:err, redis_result:result}, "unlocking error");
+				return callback(new Error("tried to release timed out lock"));
+			}
+			return callback(err,result);
+		});
+	},
 
-	runWithLock: (key, runner, callback = ( (error) -> )) ->
-		LockManager.getLock key, (error, lockValue) ->
-			return callback(error) if error?
-			runner (error1) ->
-				LockManager.releaseLock key, lockValue, (error2) ->
-					error = error1 or error2
-					return callback(error) if error?
-					callback()
+	runWithLock(key, runner, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return LockManager.getLock(key, function(error, lockValue) {
+			if (error != null) { return callback(error); }
+			return runner(error1 =>
+				LockManager.releaseLock(key, lockValue, function(error2) {
+					error = error1 || error2;
+					if (error != null) { return callback(error); }
+					return callback();
+				})
+			);
+		});
+	},
 
-	healthCheck: (callback) ->
-		action = (releaseLock) ->
-			releaseLock()
-		LockManager.runWithLock	"HistoryLock:HealthCheck:host=#{HOST}:pid=#{PID}:random=#{RND}", action, callback
+	healthCheck(callback) {
+		const action = releaseLock => releaseLock();
+		return LockManager.runWithLock(`HistoryLock:HealthCheck:host=${HOST}:pid=${PID}:random=${RND}`, action, callback);
+	},
 
-	close: (callback) ->
-		rclient.quit()
-		rclient.once 'end', callback
+	close(callback) {
+		rclient.quit();
+		return rclient.once('end', callback);
+	}
+});

services/track-changes/app/coffee/MongoAWS.js

@@ -1,118 +1,141 @@
-settings = require "settings-sharelatex"
-logger = require "logger-sharelatex"
-AWS = require 'aws-sdk'
-S3S = require 's3-streams'
-{db, ObjectId} = require "./mongojs"
-JSONStream = require "JSONStream"
-ReadlineStream = require "byline"
-zlib = require "zlib"
-Metrics = require "metrics-sharelatex"
-
-DAYS = 24 * 3600 * 1000 # one day in milliseconds
-
-createStream = (streamConstructor, project_id, doc_id, pack_id) ->
-	AWS_CONFIG =
-		accessKeyId: settings.trackchanges.s3.key
-		secretAccessKey: settings.trackchanges.s3.secret
-		endpoint: settings.trackchanges.s3.endpoint
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MongoAWS;
+const settings = require("settings-sharelatex");
+const logger = require("logger-sharelatex");
+const AWS = require('aws-sdk');
+const S3S = require('s3-streams');
+const {db, ObjectId} = require("./mongojs");
+const JSONStream = require("JSONStream");
+const ReadlineStream = require("byline");
+const zlib = require("zlib");
+const Metrics = require("metrics-sharelatex");
+
+const DAYS = 24 * 3600 * 1000; // one day in milliseconds
+
+const createStream = function(streamConstructor, project_id, doc_id, pack_id) {
+	const AWS_CONFIG = {
+		accessKeyId: settings.trackchanges.s3.key,
+		secretAccessKey: settings.trackchanges.s3.secret,
+		endpoint: settings.trackchanges.s3.endpoint,
 		s3ForcePathStyle: settings.trackchanges.s3.pathStyle
+	};
 
-	return streamConstructor new AWS.S3(AWS_CONFIG), {
+	return streamConstructor(new AWS.S3(AWS_CONFIG), {
 		"Bucket": settings.trackchanges.stores.doc_history,
 		"Key": project_id+"/changes-"+doc_id+"/pack-"+pack_id
-	}
+	});
+};
 
-module.exports = MongoAWS =
+module.exports = (MongoAWS = {
 
-	archivePack: (project_id, doc_id, pack_id, _callback = (error) ->) ->
+	archivePack(project_id, doc_id, pack_id, _callback) {
 
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
+		if (_callback == null) { _callback = function(error) {}; }
+		const callback = function(...args) {
+			_callback(...Array.from(args || []));
+			return _callback = function() {};
+		};
 
-		query = {
-			_id: ObjectId(pack_id)
+		const query = {
+			_id: ObjectId(pack_id),
 			doc_id: ObjectId(doc_id)
-		}
-
-		return callback new Error("invalid project id") if not project_id?
-		return callback new Error("invalid doc id") if not doc_id?
-		return callback new Error("invalid pack id") if not pack_id?
-
-		logger.log {project_id, doc_id, pack_id}, "uploading data to s3"
-
-		upload = createStream S3S.WriteStream, project_id, doc_id, pack_id
-
-		db.docHistory.findOne query, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("cannot find pack to send to s3") if not result?
-			return callback new Error("refusing to send pack with TTL to s3") if result.expiresAt?
-			uncompressedData = JSON.stringify(result)
-			if uncompressedData.indexOf("\u0000") != -1
-				error = new Error("null bytes found in upload")
-				logger.error err: error, project_id: project_id, doc_id: doc_id, pack_id: pack_id, error.message
-				return callback(error)
-			zlib.gzip uncompressedData, (err, buf) ->
-				logger.log {project_id, doc_id, pack_id, origSize: uncompressedData.length, newSize: buf.length}, "compressed pack"
-				return callback(err) if err?
-				upload.on 'error', (err) ->
-					callback(err)
-				upload.on 'finish', () ->
-					Metrics.inc("archive-pack")
-					logger.log {project_id, doc_id, pack_id}, "upload to s3 completed"
-					callback(null)
-				upload.write buf
-				upload.end()
-
-	readArchivedPack: (project_id, doc_id, pack_id, _callback = (error, result) ->) ->
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
-
-		return callback new Error("invalid project id") if not project_id?
-		return callback new Error("invalid doc id") if not doc_id?
-		return callback new Error("invalid pack id") if not pack_id?
-
-		logger.log {project_id, doc_id, pack_id}, "downloading data from s3"
-
-		download = createStream S3S.ReadStream, project_id, doc_id, pack_id
-
-		inputStream = download
-			.on 'open', (obj) ->
-				return 1
-			.on 'error', (err) ->
-				callback(err)
-
-		gunzip = zlib.createGunzip()
-		gunzip.setEncoding('utf8')
-		gunzip.on 'error', (err) ->
-			logger.log {project_id, doc_id, pack_id, err}, "error uncompressing gzip stream"
-			callback(err)
-
-		outputStream = inputStream.pipe gunzip
-		parts = []
-		outputStream.on 'error', (err) ->
-			return callback(err)
-		outputStream.on 'end', () ->
-			logger.log {project_id, doc_id, pack_id}, "download from s3 completed"
-			try
-				object = JSON.parse parts.join('')
-			catch e
-				return callback(e)
-			object._id = ObjectId(object._id)
-			object.doc_id = ObjectId(object.doc_id)
-			object.project_id = ObjectId(object.project_id)
-			for op in object.pack
-				op._id = ObjectId(op._id) if op._id?
-			callback null, object
-		outputStream.on 'data', (data) ->
-			parts.push data
-
-	unArchivePack: (project_id, doc_id, pack_id, callback = (error) ->) ->
-		MongoAWS.readArchivedPack project_id, doc_id, pack_id, (err, object) ->
-			return callback(err) if err?
-			Metrics.inc("unarchive-pack")
-			# allow the object to expire, we can always retrieve it again
-			object.expiresAt = new Date(Date.now() + 7 * DAYS)
-			logger.log {project_id, doc_id, pack_id}, "inserting object from s3"
-			db.docHistory.insert object, callback
+		};
+
+		if ((project_id == null)) { return callback(new Error("invalid project id")); }
+		if ((doc_id == null)) { return callback(new Error("invalid doc id")); }
+		if ((pack_id == null)) { return callback(new Error("invalid pack id")); }
+
+		logger.log({project_id, doc_id, pack_id}, "uploading data to s3");
+
+		const upload = createStream(S3S.WriteStream, project_id, doc_id, pack_id);
+
+		return db.docHistory.findOne(query, function(err, result) {
+			if (err != null) { return callback(err); }
+			if ((result == null)) { return callback(new Error("cannot find pack to send to s3")); }
+			if (result.expiresAt != null) { return callback(new Error("refusing to send pack with TTL to s3")); }
+			const uncompressedData = JSON.stringify(result);
+			if (uncompressedData.indexOf("\u0000") !== -1) {
+				const error = new Error("null bytes found in upload");
+				logger.error({err: error, project_id, doc_id, pack_id}, error.message);
+				return callback(error);
+			}
+			return zlib.gzip(uncompressedData, function(err, buf) {
+				logger.log({project_id, doc_id, pack_id, origSize: uncompressedData.length, newSize: buf.length}, "compressed pack");
+				if (err != null) { return callback(err); }
+				upload.on('error', err => callback(err));
+				upload.on('finish', function() {
+					Metrics.inc("archive-pack");
+					logger.log({project_id, doc_id, pack_id}, "upload to s3 completed");
+					return callback(null);
+				});
+				upload.write(buf);
+				return upload.end();
+			});
+		});
+	},
+
+	readArchivedPack(project_id, doc_id, pack_id, _callback) {
+		if (_callback == null) { _callback = function(error, result) {}; }
+		const callback = function(...args) {
+			_callback(...Array.from(args || []));
+			return _callback = function() {};
+		};
+
+		if ((project_id == null)) { return callback(new Error("invalid project id")); }
+		if ((doc_id == null)) { return callback(new Error("invalid doc id")); }
+		if ((pack_id == null)) { return callback(new Error("invalid pack id")); }
+
+		logger.log({project_id, doc_id, pack_id}, "downloading data from s3");
+
+		const download = createStream(S3S.ReadStream, project_id, doc_id, pack_id);
+
+		const inputStream = download
+			.on('open', obj => 1).on('error', err => callback(err));
+
+		const gunzip = zlib.createGunzip();
+		gunzip.setEncoding('utf8');
+		gunzip.on('error', function(err) {
+			logger.log({project_id, doc_id, pack_id, err}, "error uncompressing gzip stream");
+			return callback(err);
+		});
+
+		const outputStream = inputStream.pipe(gunzip);
+		const parts = [];
+		outputStream.on('error', err => callback(err));
+		outputStream.on('end', function() {
+			let object;
+			logger.log({project_id, doc_id, pack_id}, "download from s3 completed");
+			try {
+				object = JSON.parse(parts.join(''));
+			} catch (e) {
+				return callback(e);
+			}
+			object._id = ObjectId(object._id);
+			object.doc_id = ObjectId(object.doc_id);
+			object.project_id = ObjectId(object.project_id);
+			for (let op of Array.from(object.pack)) {
+				if (op._id != null) { op._id = ObjectId(op._id); }
+			}
+			return callback(null, object);
+		});
+		return outputStream.on('data', data => parts.push(data));
+	},
+
+	unArchivePack(project_id, doc_id, pack_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return MongoAWS.readArchivedPack(project_id, doc_id, pack_id, function(err, object) {
+			if (err != null) { return callback(err); }
+			Metrics.inc("unarchive-pack");
+			// allow the object to expire, we can always retrieve it again
+			object.expiresAt = new Date(Date.now() + (7 * DAYS));
+			logger.log({project_id, doc_id, pack_id}, "inserting object from s3");
+			return db.docHistory.insert(object, callback);
+		});
+	}
+});

services/track-changes/app/coffee/MongoManager.js

@@ -1,104 +1,131 @@
-{db, ObjectId} = require "./mongojs"
-PackManager = require "./PackManager"
-async = require "async"
-_ = require "underscore"
-metrics = require 'metrics-sharelatex'
-logger = require 'logger-sharelatex'
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MongoManager;
+const {db, ObjectId} = require("./mongojs");
+const PackManager = require("./PackManager");
+const async = require("async");
+const _ = require("underscore");
+const metrics = require('metrics-sharelatex');
+const logger = require('logger-sharelatex');
 
-module.exports = MongoManager =
-	getLastCompressedUpdate: (doc_id, callback = (error, update) ->) ->
-		db.docHistory
-			.find(doc_id: ObjectId(doc_id.toString()), {pack: {$slice:-1}}) # only return the last entry in a pack
-			.sort( v: -1 )
+module.exports = (MongoManager = {
+	getLastCompressedUpdate(doc_id, callback) {
+		if (callback == null) { callback = function(error, update) {}; }
+		return db.docHistory
+			.find({doc_id: ObjectId(doc_id.toString())}, {pack: {$slice:-1}}) // only return the last entry in a pack
+			.sort({ v: -1 })
 			.limit(1)
-			.toArray (error, compressedUpdates) ->
-				return callback(error) if error?
-				callback null, compressedUpdates[0] or null
+			.toArray(function(error, compressedUpdates) {
+				if (error != null) { return callback(error); }
+				return callback(null, compressedUpdates[0] || null);
+		});
+	},
 
-	peekLastCompressedUpdate: (doc_id, callback = (error, update, version) ->) ->
-		# under normal use we pass back the last update as
-		# callback(null,update,version).
-		#
-		# when we have an existing last update but want to force a new one
-		# to start, we pass it back as callback(null,null,version), just
-		# giving the version so we can check consistency.
-		MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
-			return callback(error) if error?
-			if update?
-				if update.broken # marked as broken so we will force a new op
-					return callback null, null
-				else if update.pack?
-					if update.finalised # no more ops can be appended
-						return callback null, null, update.pack[0]?.v
-					else
-						return callback null, update, update.pack[0]?.v
-				else
-					return callback null, update, update.v
-			else
-				PackManager.getLastPackFromIndex doc_id, (error, pack) ->
-					return callback(error) if error?
-					return callback(null, null, pack.v_end) if pack?.inS3? and pack?.v_end?
-					callback null, null
+	peekLastCompressedUpdate(doc_id, callback) {
+		// under normal use we pass back the last update as
+		// callback(null,update,version).
+		//
+		// when we have an existing last update but want to force a new one
+		// to start, we pass it back as callback(null,null,version), just
+		// giving the version so we can check consistency.
+		if (callback == null) { callback = function(error, update, version) {}; }
+		return MongoManager.getLastCompressedUpdate(doc_id, function(error, update) {
+			if (error != null) { return callback(error); }
+			if (update != null) {
+				if (update.broken) { // marked as broken so we will force a new op
+					return callback(null, null);
+				} else if (update.pack != null) {
+					if (update.finalised) { // no more ops can be appended
+						return callback(null, null, update.pack[0] != null ? update.pack[0].v : undefined);
+					} else {
+						return callback(null, update, update.pack[0] != null ? update.pack[0].v : undefined);
+					}
+				} else {
+					return callback(null, update, update.v);
+				}
+			} else {
+				return PackManager.getLastPackFromIndex(doc_id, function(error, pack) {
+					if (error != null) { return callback(error); }
+					if (((pack != null ? pack.inS3 : undefined) != null) && ((pack != null ? pack.v_end : undefined) != null)) { return callback(null, null, pack.v_end); }
+					return callback(null, null);
+				});
+			}
+		});
+	},
 
-	backportProjectId: (project_id, doc_id, callback = (error) ->) ->
-		db.docHistory.update {
-			doc_id: ObjectId(doc_id.toString())
+	backportProjectId(project_id, doc_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return db.docHistory.update({
+			doc_id: ObjectId(doc_id.toString()),
 			project_id: { $exists: false }
 		}, {
 			$set: { project_id: ObjectId(project_id.toString()) }
 		}, {
 			multi: true
-		}, callback
+		}, callback);
+	},
 
-	getProjectMetaData: (project_id, callback = (error, metadata) ->) ->
-		db.projectHistoryMetaData.find {
+	getProjectMetaData(project_id, callback) {
+		if (callback == null) { callback = function(error, metadata) {}; }
+		return db.projectHistoryMetaData.find({
 			project_id: ObjectId(project_id.toString())
-		}, (error, results) ->
-			return callback(error) if error?
-			callback null, results[0]
+		}, function(error, results) {
+			if (error != null) { return callback(error); }
+			return callback(null, results[0]);
+	});
+	},
 
-	setProjectMetaData: (project_id, metadata, callback = (error) ->) ->
-		db.projectHistoryMetaData.update {
+	setProjectMetaData(project_id, metadata, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return db.projectHistoryMetaData.update({
 			project_id: ObjectId(project_id)
 		}, {
 			$set: metadata
 		}, {
 			upsert: true
-		}, callback
+		}, callback);
+	},
 
-	upgradeHistory: (project_id, callback = (error) ->) ->
-		# preserve the project's existing history
-		db.docHistory.update {
-			project_id: ObjectId(project_id)
-			temporary: true
+	upgradeHistory(project_id, callback) {
+		// preserve the project's existing history
+		if (callback == null) { callback = function(error) {}; }
+		return db.docHistory.update({
+			project_id: ObjectId(project_id),
+			temporary: true,
 			expiresAt: {$exists: true}
 		}, {
-			$set: {temporary: false}
+			$set: {temporary: false},
 			$unset: {expiresAt: ""}
 		}, {
 			multi: true
-		}, callback
+		}, callback);
+	},
 
-	ensureIndices: () ->
-		# For finding all updates that go into a diff for a doc
-		db.docHistory.ensureIndex { doc_id: 1, v: 1 }, { background: true }
-		# For finding all updates that affect a project
-		db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, { background: true }
-		# For finding updates that don't yet have a project_id and need it inserting
-		db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, { background: true }
-		# For finding project meta-data
-		db.projectHistoryMetaData.ensureIndex { project_id: 1 }, { background: true }
-		# TTL index for auto deleting week old temporary ops
-		db.docHistory.ensureIndex { expiresAt: 1 }, { expireAfterSeconds: 0, background: true }
-		# For finding packs to be checked for archiving
-		db.docHistory.ensureIndex { last_checked: 1 }, { background: true }
-		# For finding archived packs
-		db.docHistoryIndex.ensureIndex { project_id: 1 }, { background: true }
+	ensureIndices() {
+		// For finding all updates that go into a diff for a doc
+		db.docHistory.ensureIndex({ doc_id: 1, v: 1 }, { background: true });
+		// For finding all updates that affect a project
+		db.docHistory.ensureIndex({ project_id: 1, "meta.end_ts": 1 }, { background: true });
+		// For finding updates that don't yet have a project_id and need it inserting
+		db.docHistory.ensureIndex({ doc_id: 1, project_id: 1 }, { background: true });
+		// For finding project meta-data
+		db.projectHistoryMetaData.ensureIndex({ project_id: 1 }, { background: true });
+		// TTL index for auto deleting week old temporary ops
+		db.docHistory.ensureIndex({ expiresAt: 1 }, { expireAfterSeconds: 0, background: true });
+		// For finding packs to be checked for archiving
+		db.docHistory.ensureIndex({ last_checked: 1 }, { background: true });
+		// For finding archived packs
+		return db.docHistoryIndex.ensureIndex({ project_id: 1 }, { background: true });
+	}
+});
 
 
 [
 	'getLastCompressedUpdate',
 	'getProjectMetaData',
 	'setProjectMetaData'
-].map (method) ->
-	metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger)
+].map(method => metrics.timeAsyncMethod(MongoManager, method, 'mongo.MongoManager', logger));

services/track-changes/app/coffee/PackManager.js

@@ -1,540 +1,706 @@
-async = require "async"
-_ = require "underscore"
-{db, ObjectId, BSON} = require "./mongojs"
-logger = require "logger-sharelatex"
-LockManager = require "./LockManager"
-MongoAWS = require "./MongoAWS"
-Metrics = require "metrics-sharelatex"
-ProjectIterator = require "./ProjectIterator"
-Settings = require "settings-sharelatex"
-keys = Settings.redis.lock.key_schema
-
-# Sharejs operations are stored in a 'pack' object
-#
-#  e.g.  a single sharejs update looks like
-#
-#   {
-#     "doc_id" : 549dae9e0a2a615c0c7f0c98,
-#     "project_id" : 549dae9c0a2a615c0c7f0c8c,
-#     "op" : [ {"p" : 6981,	"d" : "?"	} ],
-#     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
-#     "v" : 17082
-#   }
-#
-#  and a pack looks like this
-#
-#   {
-#     "doc_id" : 549dae9e0a2a615c0c7f0c98,
-#     "project_id" : 549dae9c0a2a615c0c7f0c8c,
-#     "pack" : [ U1, U2, U3, ...., UN],
-#     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
-#     "v" : 17082
-#     "v_end" : ...
-#   }
-#
-#  where U1, U2, U3, .... are single updates stripped of their
-#  doc_id and project_id fields (which are the same for all the
-#  updates in the pack).
-#
-#  The pack itself has v and meta fields, this makes it possible to
-#  treat packs and single updates in a similar way.
-#
-#  The v field of the pack itself is from the first entry U1, the
-#  v_end field from UN.  The meta.end_ts field of the pack itself is
-#  from the last entry UN, the meta.start_ts field from U1.
-
-DAYS = 24 * 3600 * 1000 # one day in milliseconds
-
-module.exports = PackManager =
-
-	MAX_SIZE:  1024*1024 # make these configurable parameters
-	MAX_COUNT: 1024
-
-	insertCompressedUpdates: (project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		return callback() if newUpdates.length == 0
-
-		# never append permanent ops to a pack that will expire
-		lastUpdate = null if lastUpdate?.expiresAt? and not temporary
-
-		updatesToFlush = []
-		updatesRemaining = newUpdates.slice()
-
-		n = lastUpdate?.n || 0
-		sz = lastUpdate?.sz || 0
-
-		while updatesRemaining.length and n < PackManager.MAX_COUNT and sz < PackManager.MAX_SIZE
-			nextUpdate = updatesRemaining[0]
-			nextUpdateSize = BSON.calculateObjectSize(nextUpdate)
-			if nextUpdateSize + sz > PackManager.MAX_SIZE and n > 0
-				break
-			n++
-			sz += nextUpdateSize
-			updatesToFlush.push updatesRemaining.shift()
-
-		PackManager.flushCompressedUpdates project_id, doc_id, lastUpdate, updatesToFlush, temporary, (error) ->
-			return callback(error) if error?
-			PackManager.insertCompressedUpdates project_id, doc_id, null, updatesRemaining, temporary, callback
-
-	flushCompressedUpdates:	(project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		return callback() if newUpdates.length == 0
-
-		canAppend = false
-		# check if it is safe to append to an existing pack
-		if lastUpdate?
-			if not temporary and not lastUpdate.expiresAt?
-				# permanent pack appends to permanent pack
-				canAppend = true
-			age = Date.now() - lastUpdate.meta?.start_ts
-			if temporary and lastUpdate.expiresAt? and age < 1 * DAYS
-				# temporary pack appends to temporary pack if same day
-				canAppend = true
-
-		if canAppend
-			PackManager.appendUpdatesToExistingPack project_id, doc_id, lastUpdate, newUpdates, temporary, callback
-		else
-			PackManager.insertUpdatesIntoNewPack project_id, doc_id, newUpdates, temporary, callback
-
-	insertUpdatesIntoNewPack: (project_id, doc_id, newUpdates, temporary, callback = (error) ->) ->
-		first = newUpdates[0]
-		last = newUpdates[newUpdates.length - 1]
-		n = newUpdates.length
-		sz = BSON.calculateObjectSize(newUpdates)
-		newPack =
-			project_id: ObjectId(project_id.toString())
-			doc_id: ObjectId(doc_id.toString())
-			pack: newUpdates
-			n: n
-			sz: sz
-			meta:
-				start_ts: first.meta.start_ts
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let PackManager;
+const async = require("async");
+const _ = require("underscore");
+const {db, ObjectId, BSON} = require("./mongojs");
+const logger = require("logger-sharelatex");
+const LockManager = require("./LockManager");
+const MongoAWS = require("./MongoAWS");
+const Metrics = require("metrics-sharelatex");
+const ProjectIterator = require("./ProjectIterator");
+const Settings = require("settings-sharelatex");
+const keys = Settings.redis.lock.key_schema;
+
+// Sharejs operations are stored in a 'pack' object
+//
+//  e.g.  a single sharejs update looks like
+//
+//   {
+//     "doc_id" : 549dae9e0a2a615c0c7f0c98,
+//     "project_id" : 549dae9c0a2a615c0c7f0c8c,
+//     "op" : [ {"p" : 6981,	"d" : "?"	} ],
+//     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
+//     "v" : 17082
+//   }
+//
+//  and a pack looks like this
+//
+//   {
+//     "doc_id" : 549dae9e0a2a615c0c7f0c98,
+//     "project_id" : 549dae9c0a2a615c0c7f0c8c,
+//     "pack" : [ U1, U2, U3, ...., UN],
+//     "meta" : {	"user_id" : 52933..., "start_ts" : 1422310693931,	"end_ts" : 1422310693931 },
+//     "v" : 17082
+//     "v_end" : ...
+//   }
+//
+//  where U1, U2, U3, .... are single updates stripped of their
+//  doc_id and project_id fields (which are the same for all the
+//  updates in the pack).
+//
+//  The pack itself has v and meta fields, this makes it possible to
+//  treat packs and single updates in a similar way.
+//
+//  The v field of the pack itself is from the first entry U1, the
+//  v_end field from UN.  The meta.end_ts field of the pack itself is
+//  from the last entry UN, the meta.start_ts field from U1.
+
+const DAYS = 24 * 3600 * 1000; // one day in milliseconds
+
+module.exports = (PackManager = {
+
+	MAX_SIZE:  1024*1024, // make these configurable parameters
+	MAX_COUNT: 1024,
+
+	insertCompressedUpdates(project_id, doc_id, lastUpdate, newUpdates, temporary, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		if (newUpdates.length === 0) { return callback(); }
+
+		// never append permanent ops to a pack that will expire
+		if (((lastUpdate != null ? lastUpdate.expiresAt : undefined) != null) && !temporary) { lastUpdate = null; }
+
+		const updatesToFlush = [];
+		const updatesRemaining = newUpdates.slice();
+
+		let n = (lastUpdate != null ? lastUpdate.n : undefined) || 0;
+		let sz = (lastUpdate != null ? lastUpdate.sz : undefined) || 0;
+
+		while (updatesRemaining.length && (n < PackManager.MAX_COUNT) && (sz < PackManager.MAX_SIZE)) {
+			const nextUpdate = updatesRemaining[0];
+			const nextUpdateSize = BSON.calculateObjectSize(nextUpdate);
+			if (((nextUpdateSize + sz) > PackManager.MAX_SIZE) && (n > 0)) {
+				break;
+			}
+			n++;
+			sz += nextUpdateSize;
+			updatesToFlush.push(updatesRemaining.shift());
+		}
+
+		return PackManager.flushCompressedUpdates(project_id, doc_id, lastUpdate, updatesToFlush, temporary, function(error) {
+			if (error != null) { return callback(error); }
+			return PackManager.insertCompressedUpdates(project_id, doc_id, null, updatesRemaining, temporary, callback);
+		});
+	},
+
+	flushCompressedUpdates(project_id, doc_id, lastUpdate, newUpdates, temporary, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		if (newUpdates.length === 0) { return callback(); }
+
+		let canAppend = false;
+		// check if it is safe to append to an existing pack
+		if (lastUpdate != null) {
+			if (!temporary && (lastUpdate.expiresAt == null)) {
+				// permanent pack appends to permanent pack
+				canAppend = true;
+			}
+			const age = Date.now() - (lastUpdate.meta != null ? lastUpdate.meta.start_ts : undefined);
+			if (temporary && (lastUpdate.expiresAt != null) && (age < (1 * DAYS))) {
+				// temporary pack appends to temporary pack if same day
+				canAppend = true;
+			}
+		}
+
+		if (canAppend) {
+			return PackManager.appendUpdatesToExistingPack(project_id, doc_id, lastUpdate, newUpdates, temporary, callback);
+		} else {
+			return PackManager.insertUpdatesIntoNewPack(project_id, doc_id, newUpdates, temporary, callback);
+		}
+	},
+
+	insertUpdatesIntoNewPack(project_id, doc_id, newUpdates, temporary, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		const first = newUpdates[0];
+		const last = newUpdates[newUpdates.length - 1];
+		const n = newUpdates.length;
+		const sz = BSON.calculateObjectSize(newUpdates);
+		const newPack = {
+			project_id: ObjectId(project_id.toString()),
+			doc_id: ObjectId(doc_id.toString()),
+			pack: newUpdates,
+			n,
+			sz,
+			meta: {
+				start_ts: first.meta.start_ts,
 				end_ts: last.meta.end_ts
-			v: first.v
-			v_end: last.v
-			temporary: temporary
-		if temporary
-			newPack.expiresAt = new Date(Date.now() + 7 * DAYS)
-			newPack.last_checked = new Date(Date.now() + 30 * DAYS) # never check temporary packs
-		logger.log {project_id, doc_id, newUpdates}, "inserting updates into new pack"
-		db.docHistory.save newPack, (err, result) ->
-			return callback(err) if err?
-			Metrics.inc("insert-pack-" + if temporary then "temporary" else "permanent")
-			if temporary
-				return callback()
-			else
-				PackManager.updateIndex project_id, doc_id, callback
-
-	appendUpdatesToExistingPack: (project_id, doc_id, lastUpdate, newUpdates, temporary, callback = (error) ->) ->
-		first = newUpdates[0]
-		last = newUpdates[newUpdates.length - 1]
-		n = newUpdates.length
-		sz = BSON.calculateObjectSize(newUpdates)
-		query =
-			_id: lastUpdate._id
-			project_id: ObjectId(project_id.toString())
-			doc_id: ObjectId(doc_id.toString())
+			},
+			v: first.v,
+			v_end: last.v,
+			temporary
+		};
+		if (temporary) {
+			newPack.expiresAt = new Date(Date.now() + (7 * DAYS));
+			newPack.last_checked = new Date(Date.now() + (30 * DAYS)); // never check temporary packs
+		}
+		logger.log({project_id, doc_id, newUpdates}, "inserting updates into new pack");
+		return db.docHistory.save(newPack, function(err, result) {
+			if (err != null) { return callback(err); }
+			Metrics.inc(`insert-pack-${temporary ? "temporary" : "permanent"}`);
+			if (temporary) {
+				return callback();
+			} else {
+				return PackManager.updateIndex(project_id, doc_id, callback);
+			}
+		});
+	},
+
+	appendUpdatesToExistingPack(project_id, doc_id, lastUpdate, newUpdates, temporary, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		const first = newUpdates[0];
+		const last = newUpdates[newUpdates.length - 1];
+		const n = newUpdates.length;
+		const sz = BSON.calculateObjectSize(newUpdates);
+		const query = {
+			_id: lastUpdate._id,
+			project_id: ObjectId(project_id.toString()),
+			doc_id: ObjectId(doc_id.toString()),
 			pack: {$exists: true}
-		update =
-			$push:
+		};
+		const update = {
+			$push: {
 				"pack": {$each: newUpdates}
-			$inc:
-				"n": n
+			},
+			$inc: {
+				"n": n,
 				"sz":  sz
-			$set:
-				"meta.end_ts": last.meta.end_ts
+			},
+			$set: {
+				"meta.end_ts": last.meta.end_ts,
 				"v_end": last.v
-		if lastUpdate.expiresAt and temporary
-			update.$set.expiresAt = new Date(Date.now() + 7 * DAYS)
-		logger.log {project_id, doc_id, lastUpdate, newUpdates}, "appending updates to existing pack"
-		Metrics.inc("append-pack-" + if temporary then "temporary" else "permanent")
-		db.docHistory.findAndModify {query, update,	new:true, fields:{meta:1,v_end:1}}, callback
-
-	# Retrieve all changes for a document
-
-	getOpsByVersionRange: (project_id, doc_id, fromVersion, toVersion, callback = (error, updates) ->) ->
-		PackManager.loadPacksByVersionRange project_id, doc_id, fromVersion, toVersion, (error) ->
-			query = {doc_id:ObjectId(doc_id.toString())}
-			query.v = {$lte:toVersion} if toVersion?
-			query.v_end = {$gte:fromVersion} if fromVersion?
-			#console.log "query:", query
-			db.docHistory.find(query).sort {v:-1}, (err, result) ->
-				return callback(err) if err?
-				#console.log "getOpsByVersionRange:", err, result
-				updates = []
-				opInRange = (op, from, to) ->
-					return false if fromVersion? and op.v < fromVersion
-					return false if toVersion? and op.v > toVersion
-					return true
-				for docHistory in result
-					#console.log 'adding', docHistory.pack
-					for op in docHistory.pack.reverse() when opInRange(op, fromVersion, toVersion)
-						op.project_id = docHistory.project_id
-						op.doc_id = docHistory.doc_id
-						#console.log "added op", op.v, fromVersion, toVersion
-						updates.push op
-				callback(null, updates)
-
-	loadPacksByVersionRange: (project_id, doc_id, fromVersion, toVersion, callback) ->
-		PackManager.getIndex doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			indexPacks = indexResult?.packs or []
-			packInRange = (pack, from, to) ->
-				return false if fromVersion? and pack.v_end < fromVersion
-				return false if toVersion? and pack.v > toVersion
-				return true
-			neededIds = (pack._id for pack in indexPacks when packInRange(pack, fromVersion, toVersion))
-			if neededIds.length
-				PackManager.fetchPacksIfNeeded project_id, doc_id, neededIds, callback
-			else
-				callback()
-
-	fetchPacksIfNeeded: (project_id, doc_id, pack_ids, callback) ->
-		db.docHistory.find {_id: {$in: (ObjectId(id) for id in pack_ids)}}, {_id:1}, (err, loadedPacks) ->
-			return callback(err) if err?
-			allPackIds = (id.toString() for id in pack_ids)
-			loadedPackIds = (pack._id.toString() for pack in loadedPacks)
-			packIdsToFetch = _.difference allPackIds, loadedPackIds
-			logger.log {project_id, doc_id, loadedPackIds, allPackIds, packIdsToFetch}, "analysed packs"
-			return callback() if packIdsToFetch.length is 0
-			async.eachLimit packIdsToFetch, 4, (pack_id, cb) ->
-				MongoAWS.unArchivePack project_id, doc_id, pack_id, cb
-			, (err) ->
-				return callback(err) if err?
-				logger.log {project_id, doc_id}, "done unarchiving"
-				callback()
-
-	# Retrieve all changes across a project
-
-	makeProjectIterator: (project_id, before, callback) ->
-		# get all the docHistory Entries
-		db.docHistory.find({project_id: ObjectId(project_id)},{pack:false}).sort {"meta.end_ts":-1}, (err, packs) ->
-			return callback(err) if err?
-			allPacks = []
-			seenIds = {}
-			for pack in packs
-				allPacks.push pack
-				seenIds[pack._id] = true
-			db.docHistoryIndex.find {project_id: ObjectId(project_id)}, (err, indexes) ->
-				return callback(err) if err?
-				for index in indexes
-					for pack in index.packs when not seenIds[pack._id]
-						pack.project_id = index.project_id
-						pack.doc_id = index._id
-						pack.fromIndex = true
-						allPacks.push pack
-						seenIds[pack._id] = true
-				callback(null, new ProjectIterator(allPacks, before, PackManager.getPackById))
-
-	getPackById: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findOne {_id: pack_id}, (err, pack) ->
-			return callback(err) if err?
-			if not pack?
-				MongoAWS.unArchivePack project_id, doc_id, pack_id, callback
-			else if pack.expiresAt? and pack.temporary is false
-				# we only need to touch the TTL when listing the changes in the project
-				# because diffs on individual documents are always done after that
-				PackManager.increaseTTL pack, callback
-				# only do this for cached packs, not temporary ones to avoid older packs
-				# being kept longer than newer ones (which messes up the last update version)
-			else
-				callback(null, pack)
-
-	increaseTTL: (pack, callback) ->
-		if pack.expiresAt < new Date(Date.now() + 6 * DAYS)
-			# update cache expiry since we are using this pack
-			db.docHistory.findAndModify {
-				query: {_id: pack._id}
-				update: {$set: {expiresAt: new Date(Date.now() + 7 * DAYS)}}
-			}, (err) ->
-				return callback(err, pack)
-		else
-			callback(null, pack)
-
-	# Manage docHistoryIndex collection
-
-	getIndex: (doc_id, callback) ->
-		db.docHistoryIndex.findOne {_id:ObjectId(doc_id.toString())}, callback
-
-	getPackFromIndex: (doc_id, pack_id, callback) ->
-		db.docHistoryIndex.findOne {_id:ObjectId(doc_id.toString()), "packs._id": pack_id}, {"packs.$":1}, callback
-
-	getLastPackFromIndex: (doc_id, callback) ->
-		db.docHistoryIndex.findOne {_id: ObjectId(doc_id.toString())}, {packs:{$slice:-1}}, (err, indexPack) ->
-			return callback(err) if err?
-			return callback() if not indexPack?
-			callback(null,indexPack[0])
-
-	getIndexWithKeys: (doc_id, callback) ->
-		PackManager.getIndex doc_id, (err, index) ->
-			return callback(err) if err?
-			return callback() if not index?
-			for pack in index?.packs or []
-				index[pack._id] = pack
-			callback(null, index)
-
-	initialiseIndex: (project_id, doc_id, callback) ->
-		PackManager.findCompletedPacks project_id, doc_id, (err, packs) ->
-			#console.log 'err', err, 'packs', packs, packs?.length
-			return callback(err) if err?
-			return callback() if not packs?
-			PackManager.insertPacksIntoIndexWithLock project_id, doc_id, packs, callback
-
-	updateIndex: (project_id, doc_id, callback) ->
-		# find all packs prior to current pack
-		PackManager.findUnindexedPacks project_id, doc_id, (err, newPacks) ->
-			return callback(err) if err?
-			return callback() if not newPacks? or newPacks.length is 0
-			PackManager.insertPacksIntoIndexWithLock project_id, doc_id, newPacks, (err) ->
-				return callback(err) if err?
-				logger.log {project_id, doc_id, newPacks}, "added new packs to index"
-				callback()
-
-	findCompletedPacks: (project_id, doc_id, callback) ->
-		query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} }
-		db.docHistory.find(query, {pack:false}).sort {v:1}, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?
-			return callback() if not packs?.length
-			last = packs.pop() # discard the last pack, if it's still in progress
-			packs.push(last) if last.finalised # it's finalised so we push it back to archive it
-			callback(null, packs)
-
-	findPacks: (project_id, doc_id, callback) ->
-		query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} }
-		db.docHistory.find(query, {pack:false}).sort {v:1}, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?
-			return callback() if not packs?.length
-			callback(null, packs)
-
-	findUnindexedPacks: (project_id, doc_id, callback) ->
-		PackManager.getIndexWithKeys doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			PackManager.findCompletedPacks project_id, doc_id, (err, historyPacks) ->
-				return callback(err) if err?
-				return callback() if not historyPacks?
-				# select only the new packs not already in the index
-				newPacks = (pack for pack in historyPacks when not indexResult?[pack._id]?)
-				newPacks = (_.omit(pack, 'doc_id', 'project_id', 'n', 'sz', 'last_checked', 'finalised') for pack in newPacks)
-				if newPacks.length
-					logger.log {project_id, doc_id, n: newPacks.length}, "found new packs"
-				callback(null, newPacks)
-
-	insertPacksIntoIndexWithLock: (project_id, doc_id, newPacks, callback) ->
-		LockManager.runWithLock(
+			}
+		};
+		if (lastUpdate.expiresAt && temporary) {
+			update.$set.expiresAt = new Date(Date.now() + (7 * DAYS));
+		}
+		logger.log({project_id, doc_id, lastUpdate, newUpdates}, "appending updates to existing pack");
+		Metrics.inc(`append-pack-${temporary ? "temporary" : "permanent"}`);
+		return db.docHistory.findAndModify({query, update,	new:true, fields:{meta:1,v_end:1}}, callback);
+	},
+
+	// Retrieve all changes for a document
+
+	getOpsByVersionRange(project_id, doc_id, fromVersion, toVersion, callback) {
+		if (callback == null) { callback = function(error, updates) {}; }
+		return PackManager.loadPacksByVersionRange(project_id, doc_id, fromVersion, toVersion, function(error) {
+			const query = {doc_id:ObjectId(doc_id.toString())};
+			if (toVersion != null) { query.v = {$lte:toVersion}; }
+			if (fromVersion != null) { query.v_end = {$gte:fromVersion}; }
+			//console.log "query:", query
+			return db.docHistory.find(query).sort({v:-1}, function(err, result) {
+				if (err != null) { return callback(err); }
+				//console.log "getOpsByVersionRange:", err, result
+				const updates = [];
+				const opInRange = function(op, from, to) {
+					if ((fromVersion != null) && (op.v < fromVersion)) { return false; }
+					if ((toVersion != null) && (op.v > toVersion)) { return false; }
+					return true;
+				};
+				for (let docHistory of Array.from(result)) {
+					//console.log 'adding', docHistory.pack
+					for (let op of Array.from(docHistory.pack.reverse())) {
+						if (opInRange(op, fromVersion, toVersion)) {
+							op.project_id = docHistory.project_id;
+							op.doc_id = docHistory.doc_id;
+							//console.log "added op", op.v, fromVersion, toVersion
+							updates.push(op);
+						}
+					}
+				}
+				return callback(null, updates);
+			});
+		});
+	},
+
+	loadPacksByVersionRange(project_id, doc_id, fromVersion, toVersion, callback) {
+		return PackManager.getIndex(doc_id, function(err, indexResult) {
+			let pack;
+			if (err != null) { return callback(err); }
+			const indexPacks = (indexResult != null ? indexResult.packs : undefined) || [];
+			const packInRange = function(pack, from, to) {
+				if ((fromVersion != null) && (pack.v_end < fromVersion)) { return false; }
+				if ((toVersion != null) && (pack.v > toVersion)) { return false; }
+				return true;
+			};
+			const neededIds = ((() => {
+				const result = [];
+				for (pack of Array.from(indexPacks)) { 					if (packInRange(pack, fromVersion, toVersion)) {
+						result.push(pack._id);
+					}
+				}
+				return result;
+			})());
+			if (neededIds.length) {
+				return PackManager.fetchPacksIfNeeded(project_id, doc_id, neededIds, callback);
+			} else {
+				return callback();
+			}
+		});
+	},
+
+	fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
+		let id;
+		return db.docHistory.find({_id: {$in: ((() => {
+			const result = [];
+			for (id of Array.from(pack_ids)) { 				result.push(ObjectId(id));
+			}
+			return result;
+		})())}}, {_id:1}, function(err, loadedPacks) {
+			if (err != null) { return callback(err); }
+			const allPackIds = ((() => {
+				const result1 = [];
+				for (id of Array.from(pack_ids)) { 					result1.push(id.toString());
+				}
+				return result1;
+			})());
+			const loadedPackIds = (Array.from(loadedPacks).map((pack) => pack._id.toString()));
+			const packIdsToFetch = _.difference(allPackIds, loadedPackIds);
+			logger.log({project_id, doc_id, loadedPackIds, allPackIds, packIdsToFetch}, "analysed packs");
+			if (packIdsToFetch.length === 0) { return callback(); }
+			return async.eachLimit(packIdsToFetch, 4, (pack_id, cb) => MongoAWS.unArchivePack(project_id, doc_id, pack_id, cb)
+			, function(err) {
+				if (err != null) { return callback(err); }
+				logger.log({project_id, doc_id}, "done unarchiving");
+				return callback();
+			});
+		});
+	},
+
+	// Retrieve all changes across a project
+
+	makeProjectIterator(project_id, before, callback) {
+		// get all the docHistory Entries
+		return db.docHistory.find({project_id: ObjectId(project_id)},{pack:false}).sort({"meta.end_ts":-1}, function(err, packs) {
+			let pack;
+			if (err != null) { return callback(err); }
+			const allPacks = [];
+			const seenIds = {};
+			for (pack of Array.from(packs)) {
+				allPacks.push(pack);
+				seenIds[pack._id] = true;
+			}
+			return db.docHistoryIndex.find({project_id: ObjectId(project_id)}, function(err, indexes) {
+				if (err != null) { return callback(err); }
+				for (let index of Array.from(indexes)) {
+					for (pack of Array.from(index.packs)) {
+						if (!seenIds[pack._id]) {
+							pack.project_id = index.project_id;
+							pack.doc_id = index._id;
+							pack.fromIndex = true;
+							allPacks.push(pack);
+							seenIds[pack._id] = true;
+						}
+					}
+				}
+				return callback(null, new ProjectIterator(allPacks, before, PackManager.getPackById));
+			});
+		});
+	},
+
+	getPackById(project_id, doc_id, pack_id, callback) {
+		return db.docHistory.findOne({_id: pack_id}, function(err, pack) {
+			if (err != null) { return callback(err); }
+			if ((pack == null)) {
+				return MongoAWS.unArchivePack(project_id, doc_id, pack_id, callback);
+			} else if ((pack.expiresAt != null) && (pack.temporary === false)) {
+				// we only need to touch the TTL when listing the changes in the project
+				// because diffs on individual documents are always done after that
+				return PackManager.increaseTTL(pack, callback);
+				// only do this for cached packs, not temporary ones to avoid older packs
+				// being kept longer than newer ones (which messes up the last update version)
+			} else {
+				return callback(null, pack);
+			}
+		});
+	},
+
+	increaseTTL(pack, callback) {
+		if (pack.expiresAt < new Date(Date.now() + (6 * DAYS))) {
+			// update cache expiry since we are using this pack
+			return db.docHistory.findAndModify({
+				query: {_id: pack._id},
+				update: {$set: {expiresAt: new Date(Date.now() + (7 * DAYS))}}
+			}, err => callback(err, pack));
+		} else {
+			return callback(null, pack);
+		}
+	},
+
+	// Manage docHistoryIndex collection
+
+	getIndex(doc_id, callback) {
+		return db.docHistoryIndex.findOne({_id:ObjectId(doc_id.toString())}, callback);
+	},
+
+	getPackFromIndex(doc_id, pack_id, callback) {
+		return db.docHistoryIndex.findOne({_id:ObjectId(doc_id.toString()), "packs._id": pack_id}, {"packs.$":1}, callback);
+	},
+
+	getLastPackFromIndex(doc_id, callback) {
+		return db.docHistoryIndex.findOne({_id: ObjectId(doc_id.toString())}, {packs:{$slice:-1}}, function(err, indexPack) {
+			if (err != null) { return callback(err); }
+			if ((indexPack == null)) { return callback(); }
+			return callback(null,indexPack[0]);
+		});
+	},
+
+	getIndexWithKeys(doc_id, callback) {
+		return PackManager.getIndex(doc_id, function(err, index) {
+			if (err != null) { return callback(err); }
+			if ((index == null)) { return callback(); }
+			for (let pack of Array.from((index != null ? index.packs : undefined) || [])) {
+				index[pack._id] = pack;
+			}
+			return callback(null, index);
+		});
+	},
+
+	initialiseIndex(project_id, doc_id, callback) {
+		return PackManager.findCompletedPacks(project_id, doc_id, function(err, packs) {
+			//console.log 'err', err, 'packs', packs, packs?.length
+			if (err != null) { return callback(err); }
+			if ((packs == null)) { return callback(); }
+			return PackManager.insertPacksIntoIndexWithLock(project_id, doc_id, packs, callback);
+		});
+	},
+
+	updateIndex(project_id, doc_id, callback) {
+		// find all packs prior to current pack
+		return PackManager.findUnindexedPacks(project_id, doc_id, function(err, newPacks) {
+			if (err != null) { return callback(err); }
+			if ((newPacks == null) || (newPacks.length === 0)) { return callback(); }
+			return PackManager.insertPacksIntoIndexWithLock(project_id, doc_id, newPacks, function(err) {
+				if (err != null) { return callback(err); }
+				logger.log({project_id, doc_id, newPacks}, "added new packs to index");
+				return callback();
+			});
+		});
+	},
+
+	findCompletedPacks(project_id, doc_id, callback) {
+		const query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} };
+		return db.docHistory.find(query, {pack:false}).sort({v:1}, function(err, packs) {
+			if (err != null) { return callback(err); }
+			if ((packs == null)) { return callback(); }
+			if (!(packs != null ? packs.length : undefined)) { return callback(); }
+			const last = packs.pop(); // discard the last pack, if it's still in progress
+			if (last.finalised) { packs.push(last); } // it's finalised so we push it back to archive it
+			return callback(null, packs);
+		});
+	},
+
+	findPacks(project_id, doc_id, callback) {
+		const query = { doc_id: ObjectId(doc_id.toString()), expiresAt: {$exists:false} };
+		return db.docHistory.find(query, {pack:false}).sort({v:1}, function(err, packs) {
+			if (err != null) { return callback(err); }
+			if ((packs == null)) { return callback(); }
+			if (!(packs != null ? packs.length : undefined)) { return callback(); }
+			return callback(null, packs);
+		});
+	},
+
+	findUnindexedPacks(project_id, doc_id, callback) {
+		return PackManager.getIndexWithKeys(doc_id, function(err, indexResult) {
+			if (err != null) { return callback(err); }
+			return PackManager.findCompletedPacks(project_id, doc_id, function(err, historyPacks) {
+				let pack;
+				if (err != null) { return callback(err); }
+				if ((historyPacks == null)) { return callback(); }
+				// select only the new packs not already in the index
+				let newPacks = ((() => {
+					const result = [];
+					for (pack of Array.from(historyPacks)) { 						if (((indexResult != null ? indexResult[pack._id] : undefined) == null)) {
+							result.push(pack);
+						}
+					}
+					return result;
+				})());
+				newPacks = ((() => {
+					const result1 = [];
+					for (pack of Array.from(newPacks)) { 						result1.push(_.omit(pack, 'doc_id', 'project_id', 'n', 'sz', 'last_checked', 'finalised'));
+					}
+					return result1;
+				})());
+				if (newPacks.length) {
+					logger.log({project_id, doc_id, n: newPacks.length}, "found new packs");
+				}
+				return callback(null, newPacks);
+			});
+		});
+	},
+
+	insertPacksIntoIndexWithLock(project_id, doc_id, newPacks, callback) {
+		return LockManager.runWithLock(
 			keys.historyIndexLock({doc_id}),
-			(releaseLock) ->
-				PackManager._insertPacksIntoIndex project_id, doc_id, newPacks, releaseLock
+			releaseLock => PackManager._insertPacksIntoIndex(project_id, doc_id, newPacks, releaseLock),
 			callback
-		)
-
-	_insertPacksIntoIndex: (project_id, doc_id, newPacks, callback) ->
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString())}
-			update:
-				$setOnInsert: project_id: ObjectId(project_id.toString())
-				$push:
+		);
+	},
+
+	_insertPacksIntoIndex(project_id, doc_id, newPacks, callback) {
+		return db.docHistoryIndex.findAndModify({
+			query: {_id:ObjectId(doc_id.toString())},
+			update: {
+				$setOnInsert: { project_id: ObjectId(project_id.toString())
+			},
+				$push: {
 					packs: {$each: newPacks, $sort: {v: 1}}
+				}
+			},
 			upsert: true
-		}, callback
-
-	# Archiving packs to S3
-
-	archivePack: (project_id, doc_id, pack_id, callback) ->
-		clearFlagOnError = (err, cb) ->
-			if err? # clear the inS3 flag on error
-				PackManager.clearPackAsArchiveInProgress project_id, doc_id, pack_id, (err2) ->
-					return cb(err2) if err2?
-					return cb(err)
-			else
-				cb()
-		async.series [
-			(cb) ->
-				PackManager.checkArchiveNotInProgress project_id, doc_id, pack_id, cb
-			(cb) ->
-				PackManager.markPackAsArchiveInProgress project_id, doc_id, pack_id, cb
-			(cb) ->
-				MongoAWS.archivePack project_id, doc_id, pack_id, (err) ->
-					clearFlagOnError(err, cb)
-			(cb) ->
-				PackManager.checkArchivedPack project_id, doc_id, pack_id, (err) ->
-					clearFlagOnError(err, cb)
-			(cb) ->
-				PackManager.markPackAsArchived project_id, doc_id, pack_id, cb
-			(cb) ->
-				PackManager.setTTLOnArchivedPack project_id, doc_id, pack_id, callback
-		], callback
-
-
-	checkArchivedPack: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findOne {_id: pack_id}, (err, pack) ->
-			return callback(err) if err?
-			return callback new Error("pack not found") if not pack?
-			MongoAWS.readArchivedPack project_id, doc_id, pack_id, (err, result) ->
-				delete result.last_checked
-				delete pack.last_checked
-				# need to compare ids as ObjectIds with .equals()
-				for key in ['_id', 'project_id', 'doc_id']
-					result[key] = pack[key] if result[key].equals(pack[key])
-				for op, i in result.pack
-					op._id = pack.pack[i]._id if op._id? and op._id.equals(pack.pack[i]._id)
-				if _.isEqual pack, result
-					callback()
-				else
-					logger.err {pack, result, jsondiff: JSON.stringify(pack) is JSON.stringify(result)}, "difference when comparing packs"
-					callback new Error("pack retrieved from s3 does not match pack in mongo")
-	# Extra methods to test archive/unarchive for a doc_id
-
-	pushOldPacks: (project_id, doc_id, callback) ->
-		PackManager.findPacks project_id, doc_id, (err, packs) ->
-			return callback(err) if err?
-			return callback() if not packs?.length
-			PackManager.processOldPack project_id, doc_id, packs[0]._id, callback
-
-	pullOldPacks: (project_id, doc_id, callback) ->
-		PackManager.loadPacksByVersionRange project_id, doc_id, null, null, callback
-
-
-	# Processing old packs via worker
-
-	processOldPack: (project_id, doc_id, pack_id, callback) ->
-		markAsChecked = (err) ->
-			PackManager.markPackAsChecked project_id, doc_id, pack_id, (err2) ->
-				return callback(err2) if err2?
-				callback(err)
-		logger.log {project_id, doc_id}, "processing old packs"
-		db.docHistory.findOne {_id:pack_id}, (err, pack) ->
-			return markAsChecked(err) if err?
-			return markAsChecked() if not pack?
-			return callback() if pack.expiresAt? # return directly
-			PackManager.finaliseIfNeeded project_id, doc_id, pack._id, pack, (err) ->
-				return markAsChecked(err) if err?
-				PackManager.updateIndexIfNeeded project_id, doc_id, (err) ->
-					return markAsChecked(err) if err?
-					PackManager.findUnarchivedPacks project_id, doc_id, (err, unarchivedPacks) ->
-						return markAsChecked(err) if err?
-						if not unarchivedPacks?.length
-							logger.log {project_id, doc_id}, "no packs need archiving"
-							return markAsChecked()
-						async.eachSeries unarchivedPacks, (pack, cb) ->
-							PackManager.archivePack project_id, doc_id, pack._id, cb
-						, (err) ->
-							return markAsChecked(err) if err?
-							logger.log  {project_id, doc_id}, "done processing"
-							markAsChecked()
-
-	finaliseIfNeeded: (project_id, doc_id, pack_id, pack, callback) ->
-		sz = pack.sz / (1024 * 1024) # in fractions of a megabyte
-		n = pack.n / 1024 # in fraction of 1024 ops
-		age = (Date.now() - pack.meta.end_ts) / DAYS
-		if age < 30 # always keep if less than 1 month old
-			logger.log {project_id, doc_id, pack_id, age}, "less than 30 days old"
-			return callback()
-		# compute an archiving threshold which decreases for each month of age
-		archive_threshold = 30 / age
-		if sz > archive_threshold or n > archive_threshold or age > 90
-			logger.log {project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "meets archive threshold"
-			PackManager.markPackAsFinalisedWithLock project_id, doc_id, pack_id, callback
-		else
-			logger.log {project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "does not meet archive threshold"
-			callback()
-
-	markPackAsFinalisedWithLock: (project_id, doc_id, pack_id, callback) ->
-		LockManager.runWithLock(
+		}, callback);
+	},
+
+	// Archiving packs to S3
+
+	archivePack(project_id, doc_id, pack_id, callback) {
+		const clearFlagOnError = function(err, cb) {
+			if (err != null) { // clear the inS3 flag on error
+				return PackManager.clearPackAsArchiveInProgress(project_id, doc_id, pack_id, function(err2) {
+					if (err2 != null) { return cb(err2); }
+					return cb(err);
+				});
+			} else {
+				return cb();
+			}
+		};
+		return async.series([
+			cb => PackManager.checkArchiveNotInProgress(project_id, doc_id, pack_id, cb),
+			cb => PackManager.markPackAsArchiveInProgress(project_id, doc_id, pack_id, cb),
+			cb =>
+				MongoAWS.archivePack(project_id, doc_id, pack_id, err => clearFlagOnError(err, cb))
+			,
+			cb =>
+				PackManager.checkArchivedPack(project_id, doc_id, pack_id, err => clearFlagOnError(err, cb))
+			,
+			cb => PackManager.markPackAsArchived(project_id, doc_id, pack_id, cb),
+			cb => PackManager.setTTLOnArchivedPack(project_id, doc_id, pack_id, callback)
+		], callback);
+	},
+
+
+	checkArchivedPack(project_id, doc_id, pack_id, callback) {
+		return db.docHistory.findOne({_id: pack_id}, function(err, pack) {
+			if (err != null) { return callback(err); }
+			if ((pack == null)) { return callback(new Error("pack not found")); }
+			return MongoAWS.readArchivedPack(project_id, doc_id, pack_id, function(err, result) {
+				delete result.last_checked;
+				delete pack.last_checked;
+				// need to compare ids as ObjectIds with .equals()
+				for (let key of ['_id', 'project_id', 'doc_id']) {
+					if (result[key].equals(pack[key])) { result[key] = pack[key]; }
+				}
+				for (let i = 0; i < result.pack.length; i++) {
+					const op = result.pack[i];
+					if ((op._id != null) && op._id.equals(pack.pack[i]._id)) { op._id = pack.pack[i]._id; }
+				}
+				if (_.isEqual(pack, result)) {
+					return callback();
+				} else {
+					logger.err({pack, result, jsondiff: JSON.stringify(pack) === JSON.stringify(result)}, "difference when comparing packs");
+					return callback(new Error("pack retrieved from s3 does not match pack in mongo"));
+				}
+			});
+		});
+	},
+	// Extra methods to test archive/unarchive for a doc_id
+
+	pushOldPacks(project_id, doc_id, callback) {
+		return PackManager.findPacks(project_id, doc_id, function(err, packs) {
+			if (err != null) { return callback(err); }
+			if (!(packs != null ? packs.length : undefined)) { return callback(); }
+			return PackManager.processOldPack(project_id, doc_id, packs[0]._id, callback);
+		});
+	},
+
+	pullOldPacks(project_id, doc_id, callback) {
+		return PackManager.loadPacksByVersionRange(project_id, doc_id, null, null, callback);
+	},
+
+
+	// Processing old packs via worker
+
+	processOldPack(project_id, doc_id, pack_id, callback) {
+		const markAsChecked = err =>
+			PackManager.markPackAsChecked(project_id, doc_id, pack_id, function(err2) {
+				if (err2 != null) { return callback(err2); }
+				return callback(err);
+			})
+		;
+		logger.log({project_id, doc_id}, "processing old packs");
+		return db.docHistory.findOne({_id:pack_id}, function(err, pack) {
+			if (err != null) { return markAsChecked(err); }
+			if ((pack == null)) { return markAsChecked(); }
+			if (pack.expiresAt != null) { return callback(); } // return directly
+			return PackManager.finaliseIfNeeded(project_id, doc_id, pack._id, pack, function(err) {
+				if (err != null) { return markAsChecked(err); }
+				return PackManager.updateIndexIfNeeded(project_id, doc_id, function(err) {
+					if (err != null) { return markAsChecked(err); }
+					return PackManager.findUnarchivedPacks(project_id, doc_id, function(err, unarchivedPacks) {
+						if (err != null) { return markAsChecked(err); }
+						if (!(unarchivedPacks != null ? unarchivedPacks.length : undefined)) {
+							logger.log({project_id, doc_id}, "no packs need archiving");
+							return markAsChecked();
+						}
+						return async.eachSeries(unarchivedPacks, (pack, cb) => PackManager.archivePack(project_id, doc_id, pack._id, cb)
+						, function(err) {
+							if (err != null) { return markAsChecked(err); }
+							logger.log({project_id, doc_id}, "done processing");
+							return markAsChecked();
+						});
+					});
+				});
+			});
+		});
+	},
+
+	finaliseIfNeeded(project_id, doc_id, pack_id, pack, callback) {
+		const sz = pack.sz / (1024 * 1024); // in fractions of a megabyte
+		const n = pack.n / 1024; // in fraction of 1024 ops
+		const age = (Date.now() - pack.meta.end_ts) / DAYS;
+		if (age < 30) { // always keep if less than 1 month old
+			logger.log({project_id, doc_id, pack_id, age}, "less than 30 days old");
+			return callback();
+		}
+		// compute an archiving threshold which decreases for each month of age
+		const archive_threshold = 30 / age;
+		if ((sz > archive_threshold) || (n > archive_threshold) || (age > 90)) {
+			logger.log({project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "meets archive threshold");
+			return PackManager.markPackAsFinalisedWithLock(project_id, doc_id, pack_id, callback);
+		} else {
+			logger.log({project_id, doc_id, pack_id, age, archive_threshold, sz, n}, "does not meet archive threshold");
+			return callback();
+		}
+	},
+
+	markPackAsFinalisedWithLock(project_id, doc_id, pack_id, callback) {
+		return LockManager.runWithLock(
 			keys.historyLock({doc_id}),
-			(releaseLock) ->
-				PackManager._markPackAsFinalised project_id, doc_id, pack_id, releaseLock
+			releaseLock => PackManager._markPackAsFinalised(project_id, doc_id, pack_id, releaseLock),
 			callback
-		)
+		);
+	},
 
-	_markPackAsFinalised: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as finalised"
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
+	_markPackAsFinalised(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id, pack_id}, "marking pack as finalised");
+		return db.docHistory.findAndModify({
+			query: {_id: pack_id},
 			update: {$set: {finalised: true}}
-		}, callback
-
-
-	updateIndexIfNeeded: (project_id, doc_id, callback) ->
-		logger.log {project_id, doc_id}, "archiving old packs"
-		PackManager.getIndexWithKeys doc_id, (err, index) ->
-			return callback(err) if err?
-			if not index?
-				PackManager.initialiseIndex project_id, doc_id, callback
-			else
-				PackManager.updateIndex project_id, doc_id, callback
-
-	markPackAsChecked: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as checked"
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
+		}, callback);
+	},
+
+
+	updateIndexIfNeeded(project_id, doc_id, callback) {
+		logger.log({project_id, doc_id}, "archiving old packs");
+		return PackManager.getIndexWithKeys(doc_id, function(err, index) {
+			if (err != null) { return callback(err); }
+			if ((index == null)) {
+				return PackManager.initialiseIndex(project_id, doc_id, callback);
+			} else {
+				return PackManager.updateIndex(project_id, doc_id, callback);
+			}
+		});
+	},
+
+	markPackAsChecked(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id, pack_id}, "marking pack as checked");
+		return db.docHistory.findAndModify({
+			query: {_id: pack_id},
 			update: {$currentDate: {"last_checked":true}}
-		}, callback
-
-	findUnarchivedPacks: (project_id, doc_id, callback) ->
-		PackManager.getIndex doc_id, (err, indexResult) ->
-			return callback(err) if err?
-			indexPacks = indexResult?.packs or []
-			unArchivedPacks = (pack for pack in indexPacks when not pack.inS3?)
-			if unArchivedPacks.length
-				logger.log {project_id, doc_id, n: unArchivedPacks.length}, "find unarchived packs"
-			callback(null, unArchivedPacks)
-
-	# Archive locking flags
-
-	checkArchiveNotInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "checking if archive in progress"
-		PackManager.getPackFromIndex doc_id, pack_id, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("pack not found in index") if not result?
-			if result.inS3
-				return callback new Error("pack archiving already done")
-			else if result.inS3?
-				return callback new Error("pack archiving already in progress")
-			else
-				return callback()
-
-	markPackAsArchiveInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id}, "marking pack as archive in progress status"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()),  packs: {$elemMatch: {"_id": pack_id, inS3: {$exists:false}}}}
-			fields:  { "packs.$": 1 }
+		}, callback);
+	},
+
+	findUnarchivedPacks(project_id, doc_id, callback) {
+		return PackManager.getIndex(doc_id, function(err, indexResult) {
+			if (err != null) { return callback(err); }
+			const indexPacks = (indexResult != null ? indexResult.packs : undefined) || [];
+			const unArchivedPacks = ((() => {
+				const result = [];
+				for (let pack of Array.from(indexPacks)) { 					if ((pack.inS3 == null)) {
+						result.push(pack);
+					}
+				}
+				return result;
+			})());
+			if (unArchivedPacks.length) {
+				logger.log({project_id, doc_id, n: unArchivedPacks.length}, "find unarchived packs");
+			}
+			return callback(null, unArchivedPacks);
+		});
+	},
+
+	// Archive locking flags
+
+	checkArchiveNotInProgress(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id, pack_id}, "checking if archive in progress");
+		return PackManager.getPackFromIndex(doc_id, pack_id, function(err, result) {
+			if (err != null) { return callback(err); }
+			if ((result == null)) { return callback(new Error("pack not found in index")); }
+			if (result.inS3) {
+				return callback(new Error("pack archiving already done"));
+			} else if (result.inS3 != null) {
+				return callback(new Error("pack archiving already in progress"));
+			} else {
+				return callback();
+			}
+		});
+	},
+
+	markPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id}, "marking pack as archive in progress status");
+		return db.docHistoryIndex.findAndModify({
+			query: {_id:ObjectId(doc_id.toString()),  packs: {$elemMatch: {"_id": pack_id, inS3: {$exists:false}}}},
+			fields:  { "packs.$": 1 },
 			update: {$set: {"packs.$.inS3":false}}
-		}, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("archive is already in progress") if not result?
-			logger.log {project_id, doc_id, pack_id}, "marked as archive in progress"
-			callback()
-
-	clearPackAsArchiveInProgress: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "clearing as archive in progress"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}}
-			fields:  { "packs.$": 1 }
+		}, function(err, result) {
+			if (err != null) { return callback(err); }
+			if ((result == null)) { return callback(new Error("archive is already in progress")); }
+			logger.log({project_id, doc_id, pack_id}, "marked as archive in progress");
+			return callback();
+		});
+	},
+
+	clearPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id, pack_id}, "clearing as archive in progress");
+		return db.docHistoryIndex.findAndModify({
+			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}},
+			fields:  { "packs.$": 1 },
 			update: {$unset: {"packs.$.inS3":true}}
-		}, callback
-
-	markPackAsArchived: (project_id, doc_id, pack_id, callback) ->
-		logger.log {project_id, doc_id, pack_id}, "marking pack as archived"
-		db.docHistoryIndex.findAndModify {
-			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}}
-			fields: { "packs.$": 1 }
+		}, callback);
+	},
+
+	markPackAsArchived(project_id, doc_id, pack_id, callback) {
+		logger.log({project_id, doc_id, pack_id}, "marking pack as archived");
+		return db.docHistoryIndex.findAndModify({
+			query: {_id:ObjectId(doc_id.toString()), "packs" : {$elemMatch: {"_id": pack_id, inS3: false}}},
+			fields: { "packs.$": 1 },
 			update: {$set: {"packs.$.inS3":true}}
-		}, (err, result) ->
-			return callback(err) if err?
-			return callback new Error("archive is not marked as progress") if not result?
-			logger.log {project_id, doc_id, pack_id}, "marked as archived"
-			callback()
-
-	setTTLOnArchivedPack: (project_id, doc_id, pack_id, callback) ->
-		db.docHistory.findAndModify {
-			query: {_id: pack_id}
-			update: {$set: {expiresAt: new Date(Date.now() + 1*DAYS)}}
-		}, (err) ->
-			logger.log {project_id, doc_id, pack_id}, "set expiry on pack"
-			callback()
-
-
-#	_getOneDayInFutureWithRandomDelay: ->
-#		thirtyMins = 1000 * 60 * 30
-#		randomThirtyMinMax = Math.ceil(Math.random() * thirtyMins)
-#		return new Date(Date.now() + randomThirtyMinMax + 1*DAYS)
+		}, function(err, result) {
+			if (err != null) { return callback(err); }
+			if ((result == null)) { return callback(new Error("archive is not marked as progress")); }
+			logger.log({project_id, doc_id, pack_id}, "marked as archived");
+			return callback();
+		});
+	},
+
+	setTTLOnArchivedPack(project_id, doc_id, pack_id, callback) {
+		return db.docHistory.findAndModify({
+			query: {_id: pack_id},
+			update: {$set: {expiresAt: new Date(Date.now() + (1*DAYS))}}
+		}, function(err) {
+			logger.log({project_id, doc_id, pack_id}, "set expiry on pack");
+			return callback();
+		});
+	}
+});
+
+
+//	_getOneDayInFutureWithRandomDelay: ->
+//		thirtyMins = 1000 * 60 * 30
+//		randomThirtyMinMax = Math.ceil(Math.random() * thirtyMins)
+//		return new Date(Date.now() + randomThirtyMinMax + 1*DAYS)

services/track-changes/app/coffee/PackWorker.js

@@ -1,139 +1,183 @@
-Settings = require "settings-sharelatex"
-async = require "async"
-_ = require "underscore"
-{db, ObjectId, BSON} = require "./mongojs"
-fs = require "fs"
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("track-changes")
-logger = require "logger-sharelatex"
-logger.initialize("track-changes-packworker")
-if Settings.sentry?.dsn?
-	logger.initializeErrorReporting(Settings.sentry.dsn)
-
-DAYS = 24 * 3600 * 1000
-
-LockManager = require "./LockManager"
-PackManager = require "./PackManager"
-
-# this worker script is forked by the main process to look for
-# document histories which can be archived
-
-source = process.argv[2]
-DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
-TIMEOUT = Number(process.argv[4]) || 30*60*1000
-COUNT = 0  # number processed
-TOTAL = 0  # total number to process
-
-if !source.match(/^[0-9]+$/)
-	file = fs.readFileSync source
-	result = for line in file.toString().split('\n')
-		[project_id, doc_id] = line.split(' ')
-		{doc_id, project_id}
-	pending = _.filter result, (row) -> row?.doc_id?.match(/^[a-f0-9]{24}$/)
-else
-	LIMIT = Number(process.argv[2]) || 1000
-
-shutDownRequested = false
-shutDownTimer = setTimeout () ->
-	logger.log "pack timed out, requesting shutdown"
-	# start the shutdown on the next pack
-	shutDownRequested = true
-	# do a hard shutdown after a further 5 minutes
-	hardTimeout = setTimeout () ->
-		logger.error "HARD TIMEOUT in pack archive worker"
-		process.exit()
-	, 5*60*1000
-	hardTimeout.unref()
-, TIMEOUT
-
-logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}"
-
-# work around for https://github.com/mafintosh/mongojs/issues/224
-db.close =  (callback) ->
-	this._getServer (err, server) ->
-		return callback(err) if err?
-		server = if server.destroy? then server else server.topology
-		server.destroy(true, true)
-		callback()
-
-finish = () ->
-	if shutDownTimer?
-		logger.log 'cancelling timeout'
-		clearTimeout shutDownTimer
-	logger.log 'closing db'
-	db.close () ->
-		logger.log 'closing LockManager Redis Connection'
-		LockManager.close () ->
-			logger.log {processedCount: COUNT, allCount: TOTAL}, 'ready to exit from pack archive worker'
-			hardTimeout = setTimeout () ->
-				logger.error 'hard exit from pack archive worker'
-				process.exit(1)
-			, 5*1000
-			hardTimeout.unref()
-
-process.on 'exit', (code) ->
-	logger.log {code}, 'pack archive worker exited'
-
-processUpdates = (pending) ->
-	async.eachSeries pending,	(result, callback) ->
-		{_id, project_id, doc_id} = result
-		COUNT++
-		logger.log {project_id, doc_id}, "processing #{COUNT}/#{TOTAL}"
-		if not project_id? or not doc_id?
-			logger.log {project_id, doc_id}, "skipping pack, missing project/doc id"
-			return callback()
-		handler = (err, result) ->
-			if err? and err.code is "InternalError" and err.retryable
-				logger.warn {err, result}, "ignoring S3 error in pack archive worker"
-				# Ignore any s3 errors due to random problems
-				err = null
-			if err?
-				logger.error {err, result}, "error in pack archive worker"
-				return callback(err)
-			if shutDownRequested
-				logger.warn "shutting down pack archive worker"
-				return callback(new Error("shutdown"))
-			setTimeout () ->
-				callback(err, result)
-			, DOCUMENT_PACK_DELAY
-		if not _id?
-			PackManager.pushOldPacks project_id, doc_id, handler
-		else
-			PackManager.processOldPack project_id, doc_id, _id, handler
-	, (err, results) ->
-		if err? and err.message != "shutdown"
-			logger.error {err}, 'error in pack archive worker processUpdates'
-		finish()
-
-# find the packs which can be archived
-
-ObjectIdFromDate =  (date) ->
-	id = Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
-	return ObjectId(id)
-
-# new approach, two passes
-# find packs to be marked as finalised:true, those which have a newer pack present
-# then only consider finalised:true packs for archiving
-
-if pending?
-	logger.log "got #{pending.length} entries from #{source}"
-	processUpdates pending
-else
-	oneWeekAgo = new Date(Date.now() - 7 * DAYS)
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let LIMIT, pending;
+let project_id, doc_id;
+const Settings = require("settings-sharelatex");
+const async = require("async");
+const _ = require("underscore");
+const {db, ObjectId, BSON} = require("./mongojs");
+const fs = require("fs");
+const Metrics = require("metrics-sharelatex");
+Metrics.initialize("track-changes");
+const logger = require("logger-sharelatex");
+logger.initialize("track-changes-packworker");
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
+	logger.initializeErrorReporting(Settings.sentry.dsn);
+}
+
+const DAYS = 24 * 3600 * 1000;
+
+const LockManager = require("./LockManager");
+const PackManager = require("./PackManager");
+
+// this worker script is forked by the main process to look for
+// document histories which can be archived
+
+const source = process.argv[2];
+const DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000;
+const TIMEOUT = Number(process.argv[4]) || (30*60*1000);
+let COUNT = 0;  // number processed
+let TOTAL = 0;  // total number to process
+
+if (!source.match(/^[0-9]+$/)) {
+	const file = fs.readFileSync(source);
+	const result = (() => {
+		const result1 = [];
+		for (let line of Array.from(file.toString().split('\n'))) {
+			[project_id, doc_id] = Array.from(line.split(' '));
+			result1.push({doc_id, project_id});
+		}
+		return result1;
+	})();
+	pending = _.filter(result, row => __guard__(row != null ? row.doc_id : undefined, x => x.match(/^[a-f0-9]{24}$/)));
+} else {
+	LIMIT = Number(process.argv[2]) || 1000;
+}
+
+let shutDownRequested = false;
+const shutDownTimer = setTimeout(function() {
+	logger.log("pack timed out, requesting shutdown");
+	// start the shutdown on the next pack
+	shutDownRequested = true;
+	// do a hard shutdown after a further 5 minutes
+	const hardTimeout = setTimeout(function() {
+		logger.error("HARD TIMEOUT in pack archive worker");
+		return process.exit();
+	}
+	, 5*60*1000);
+	return hardTimeout.unref();
+}
+, TIMEOUT);
+
+logger.log(`checking for updates, limit=${LIMIT}, delay=${DOCUMENT_PACK_DELAY}, timeout=${TIMEOUT}`);
+
+// work around for https://github.com/mafintosh/mongojs/issues/224
+db.close =  function(callback) {
+	return this._getServer(function(err, server) {
+		if (err != null) { return callback(err); }
+		server = (server.destroy != null) ? server : server.topology;
+		server.destroy(true, true);
+		return callback();
+	});
+};
+
+const finish = function() {
+	if (shutDownTimer != null) {
+		logger.log('cancelling timeout');
+		clearTimeout(shutDownTimer);
+	}
+	logger.log('closing db');
+	return db.close(function() {
+		logger.log('closing LockManager Redis Connection');
+		return LockManager.close(function() {
+			logger.log({processedCount: COUNT, allCount: TOTAL}, 'ready to exit from pack archive worker');
+			const hardTimeout = setTimeout(function() {
+				logger.error('hard exit from pack archive worker');
+				return process.exit(1);
+			}
+			, 5*1000);
+			return hardTimeout.unref();
+		});
+	});
+};
+
+process.on('exit', code => logger.log({code}, 'pack archive worker exited'));
+
+const processUpdates = pending =>
+	async.eachSeries(pending,	function(result, callback) {
+		let _id;
+		({_id, project_id, doc_id} = result);
+		COUNT++;
+		logger.log({project_id, doc_id}, `processing ${COUNT}/${TOTAL}`);
+		if ((project_id == null) || (doc_id == null)) {
+			logger.log({project_id, doc_id}, "skipping pack, missing project/doc id");
+			return callback();
+		}
+		const handler = function(err, result) {
+			if ((err != null) && (err.code === "InternalError") && err.retryable) {
+				logger.warn({err, result}, "ignoring S3 error in pack archive worker");
+				// Ignore any s3 errors due to random problems
+				err = null;
+			}
+			if (err != null) {
+				logger.error({err, result}, "error in pack archive worker");
+				return callback(err);
+			}
+			if (shutDownRequested) {
+				logger.warn("shutting down pack archive worker");
+				return callback(new Error("shutdown"));
+			}
+			return setTimeout(() => callback(err, result)
+			, DOCUMENT_PACK_DELAY);
+		};
+		if ((_id == null)) {
+			return PackManager.pushOldPacks(project_id, doc_id, handler);
+		} else {
+			return PackManager.processOldPack(project_id, doc_id, _id, handler);
+		}
+	}
+	, function(err, results) {
+		if ((err != null) && (err.message !== "shutdown")) {
+			logger.error({err}, 'error in pack archive worker processUpdates');
+		}
+		return finish();
+	})
+;
+
+// find the packs which can be archived
+
+const ObjectIdFromDate =  function(date) {
+	const id = Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
+	return ObjectId(id);
+};
+
+// new approach, two passes
+// find packs to be marked as finalised:true, those which have a newer pack present
+// then only consider finalised:true packs for archiving
+
+if (pending != null) {
+	logger.log(`got ${pending.length} entries from ${source}`);
+	processUpdates(pending);
+} else {
+	const oneWeekAgo = new Date(Date.now() - (7 * DAYS));
 	db.docHistory.find({
-		expiresAt: {$exists: false}
-		project_id: {$exists: true}
-		v_end: {$exists: true}
-		_id: {$lt: ObjectIdFromDate(oneWeekAgo)}
+		expiresAt: {$exists: false},
+		project_id: {$exists: true},
+		v_end: {$exists: true},
+		_id: {$lt: ObjectIdFromDate(oneWeekAgo)},
 		last_checked: {$lt: oneWeekAgo}
 	}, {_id:1, doc_id:1, project_id:1}).sort({
 		last_checked:1
-	}).limit LIMIT, (err, results) ->
-		if err?
-			logger.log {err}, 'error checking for updates'
-			finish()
-			return
-		pending = _.uniq results, false, (result) -> result.doc_id.toString()
-		TOTAL = pending.length
-		logger.log "found #{TOTAL} documents to archive"
-		processUpdates pending
+	}).limit(LIMIT, function(err, results) {
+		if (err != null) {
+			logger.log({err}, 'error checking for updates');
+			finish();
+			return;
+		}
+		pending = _.uniq(results, false, result => result.doc_id.toString());
+		TOTAL = pending.length;
+		logger.log(`found ${TOTAL} documents to archive`);
+		return processUpdates(pending);
+	});
+}
+
+function __guard__(value, transform) {
+  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
+}

services/track-changes/app/coffee/ProjectIterator.js

@@ -1,62 +1,84 @@
-Heap = require "heap"
-
-module.exports = ProjectIterator =
-
-	class ProjectIterator
-		constructor: (packs, @before, @getPackByIdFn) ->
-			byEndTs = (a,b) -> (b.meta.end_ts - a.meta.end_ts) || (a.fromIndex - b.fromIndex)
-			@packs = packs.slice().sort byEndTs
-			@queue = new Heap(byEndTs)
-
-		next: (callback) ->
-			#  what's up next
-			#console.log ">>> top item", iterator.packs[0]
-			iterator = this
-			before = @before
-			queue = iterator.queue
-			opsToReturn = []
-			nextPack = iterator.packs[0]
-			lowWaterMark = nextPack?.meta.end_ts || 0
-			nextItem = queue.peek()
-
-			#console.log "queue empty?", queue.empty()
-			#console.log "nextItem", nextItem
-			#console.log "nextItem.meta.end_ts", nextItem?.meta.end_ts
-			#console.log "lowWaterMark", lowWaterMark
-
-			while before? and nextPack?.meta.start_ts > before
-				# discard pack that is outside range
-				iterator.packs.shift()
-				nextPack = iterator.packs[0]
-				lowWaterMark = nextPack?.meta.end_ts || 0
-
-			if (queue.empty() or nextItem?.meta.end_ts <= lowWaterMark) and nextPack?
-				# retrieve the next pack and populate the queue
-				return @getPackByIdFn nextPack.project_id, nextPack.doc_id, nextPack._id, (err, pack) ->
-					return callback(err) if err?
-					iterator.packs.shift() # have now retrieved this pack, remove it
-					#console.log "got pack", pack
-					for op in pack.pack when (not before? or op.meta.end_ts < before)
-						#console.log "adding op", op
-						op.doc_id = nextPack.doc_id
-						op.project_id = nextPack.project_id
-						queue.push op
-					# now try again
-					return iterator.next(callback)
-
-			#console.log "nextItem", nextItem, "lowWaterMark", lowWaterMark
-			while nextItem? and (nextItem?.meta.end_ts > lowWaterMark)
-				opsToReturn.push nextItem
-				queue.pop()
-				nextItem = queue.peek()
-
-			#console.log "queue empty?", queue.empty()
-			#console.log "nextPack", nextPack?
-
-			if queue.empty() and not nextPack? # got everything
-				iterator._done = true
-
-			callback(null, opsToReturn)
-
-		done: () ->
-			return @_done
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let ProjectIterator;
+const Heap = require("heap");
+
+module.exports = (ProjectIterator =
+
+	(ProjectIterator = class ProjectIterator {
+		constructor(packs, before, getPackByIdFn) {
+			this.before = before;
+			this.getPackByIdFn = getPackByIdFn;
+			const byEndTs = (a,b) => (b.meta.end_ts - a.meta.end_ts) || (a.fromIndex - b.fromIndex);
+			this.packs = packs.slice().sort(byEndTs);
+			this.queue = new Heap(byEndTs);
+		}
+
+		next(callback) {
+			//  what's up next
+			//console.log ">>> top item", iterator.packs[0]
+			const iterator = this;
+			const { before } = this;
+			const { queue } = iterator;
+			const opsToReturn = [];
+			let nextPack = iterator.packs[0];
+			let lowWaterMark = (nextPack != null ? nextPack.meta.end_ts : undefined) || 0;
+			let nextItem = queue.peek();
+
+			//console.log "queue empty?", queue.empty()
+			//console.log "nextItem", nextItem
+			//console.log "nextItem.meta.end_ts", nextItem?.meta.end_ts
+			//console.log "lowWaterMark", lowWaterMark
+
+			while ((before != null) && ((nextPack != null ? nextPack.meta.start_ts : undefined) > before)) {
+				// discard pack that is outside range
+				iterator.packs.shift();
+				nextPack = iterator.packs[0];
+				lowWaterMark = (nextPack != null ? nextPack.meta.end_ts : undefined) || 0;
+			}
+
+			if ((queue.empty() || ((nextItem != null ? nextItem.meta.end_ts : undefined) <= lowWaterMark)) && (nextPack != null)) {
+				// retrieve the next pack and populate the queue
+				return this.getPackByIdFn(nextPack.project_id, nextPack.doc_id, nextPack._id, function(err, pack) {
+					if (err != null) { return callback(err); }
+					iterator.packs.shift(); // have now retrieved this pack, remove it
+					//console.log "got pack", pack
+					for (let op of Array.from(pack.pack)) {
+						//console.log "adding op", op
+						if ((before == null) || (op.meta.end_ts < before)) {
+							op.doc_id = nextPack.doc_id;
+							op.project_id = nextPack.project_id;
+							queue.push(op);
+						}
+					}
+					// now try again
+					return iterator.next(callback);
+				});
+			}
+
+			//console.log "nextItem", nextItem, "lowWaterMark", lowWaterMark
+			while ((nextItem != null) && ((nextItem != null ? nextItem.meta.end_ts : undefined) > lowWaterMark)) {
+				opsToReturn.push(nextItem);
+				queue.pop();
+				nextItem = queue.peek();
+			}
+
+			//console.log "queue empty?", queue.empty()
+			//console.log "nextPack", nextPack?
+
+			if (queue.empty() && (nextPack == null)) { // got everything
+				iterator._done = true;
+			}
+
+			return callback(null, opsToReturn);
+		}
+
+		done() {
+			return this._done;
+		}
+	}));

services/track-changes/app/coffee/RedisManager.js

@@ -1,80 +1,121 @@
-Settings = require "settings-sharelatex"
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.history)
-Keys = Settings.redis.history.key_schema
-async = require "async"
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RedisManager;
+const Settings = require("settings-sharelatex");
+const redis = require("redis-sharelatex");
+const rclient = redis.createClient(Settings.redis.history);
+const Keys = Settings.redis.history.key_schema;
+const async = require("async");
 
-module.exports = RedisManager =
+module.exports = (RedisManager = {
 
-	getOldestDocUpdates: (doc_id, batchSize, callback = (error, jsonUpdates) ->) ->
-		key = Keys.uncompressedHistoryOps({doc_id})
-		rclient.lrange key, 0, batchSize - 1, callback
+	getOldestDocUpdates(doc_id, batchSize, callback) {
+		if (callback == null) { callback = function(error, jsonUpdates) {}; }
+		const key = Keys.uncompressedHistoryOps({doc_id});
+		return rclient.lrange(key, 0, batchSize - 1, callback);
+	},
 
-	expandDocUpdates: (jsonUpdates, callback = (error, rawUpdates) ->) ->
-		try
-			rawUpdates = ( JSON.parse(update) for update in jsonUpdates or [] )
-		catch e
-			return callback(e)
-		callback null, rawUpdates
+	expandDocUpdates(jsonUpdates, callback) {
+		let rawUpdates;
+		if (callback == null) { callback = function(error, rawUpdates) {}; }
+		try {
+			rawUpdates = ( Array.from(jsonUpdates || []).map((update) => JSON.parse(update)) );
+		} catch (e) {
+			return callback(e);
+		}
+		return callback(null, rawUpdates);
+	},
 
-	deleteAppliedDocUpdates: (project_id, doc_id, docUpdates, callback = (error) ->) ->
-		multi = rclient.multi()
-		# Delete all the updates which have been applied (exact match)
-		for update in docUpdates or []
-			multi.lrem Keys.uncompressedHistoryOps({doc_id}), 1, update
-		multi.exec (error, results) ->
-			return callback(error) if error?
-			# It's ok to delete the doc_id from the set here. Even though the list
-			# of updates may not be empty, we will continue to process it until it is.
-			rclient.srem Keys.docsWithHistoryOps({project_id}), doc_id, (error) ->
-				return callback(error) if error?
-				callback null
+	deleteAppliedDocUpdates(project_id, doc_id, docUpdates, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		const multi = rclient.multi();
+		// Delete all the updates which have been applied (exact match)
+		for (let update of Array.from(docUpdates || [])) {
+			multi.lrem(Keys.uncompressedHistoryOps({doc_id}), 1, update);
+		}
+		return multi.exec(function(error, results) {
+			if (error != null) { return callback(error); }
+			// It's ok to delete the doc_id from the set here. Even though the list
+			// of updates may not be empty, we will continue to process it until it is.
+			return rclient.srem(Keys.docsWithHistoryOps({project_id}), doc_id, function(error) {
+				if (error != null) { return callback(error); }
+				return callback(null);
+			});
+		});
+	},
 
-	getDocIdsWithHistoryOps: (project_id, callback = (error, doc_ids) ->) ->
-		rclient.smembers Keys.docsWithHistoryOps({project_id}), callback
+	getDocIdsWithHistoryOps(project_id, callback) {
+		if (callback == null) { callback = function(error, doc_ids) {}; }
+		return rclient.smembers(Keys.docsWithHistoryOps({project_id}), callback);
+	},
 
-	# iterate over keys asynchronously using redis scan (non-blocking)
-	# handle all the cluster nodes or single redis server
-	_getKeys: (pattern, callback) ->
-		nodes = rclient.nodes?('master') || [ rclient ];
-		doKeyLookupForNode = (node, cb) ->
-			RedisManager._getKeysFromNode node, pattern, cb
-		async.concatSeries nodes, doKeyLookupForNode, callback
+	// iterate over keys asynchronously using redis scan (non-blocking)
+	// handle all the cluster nodes or single redis server
+	_getKeys(pattern, callback) {
+		const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ];
+		const doKeyLookupForNode = (node, cb) => RedisManager._getKeysFromNode(node, pattern, cb);
+		return async.concatSeries(nodes, doKeyLookupForNode, callback);
+	},
 
-	_getKeysFromNode: (node, pattern, callback) ->
-		cursor = 0  # redis iterator
-		keySet = {} # use hash to avoid duplicate results
-		# scan over all keys looking for pattern
-		doIteration = (cb) ->
-			node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) ->
-				return callback(error) if error?
-				[cursor, keys] = reply
-				for key in keys
-					keySet[key] = true
-				if cursor == '0'  # note redis returns string result not numeric
-					return callback(null, Object.keys(keySet))
-				else
-					doIteration()
-		doIteration()
+	_getKeysFromNode(node, pattern, callback) {
+		let cursor = 0;  // redis iterator
+		const keySet = {}; // use hash to avoid duplicate results
+		// scan over all keys looking for pattern
+		var doIteration = cb =>
+			node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) {
+				let keys;
+				if (error != null) { return callback(error); }
+				[cursor, keys] = Array.from(reply);
+				for (let key of Array.from(keys)) {
+					keySet[key] = true;
+				}
+				if (cursor === '0') {  // note redis returns string result not numeric
+					return callback(null, Object.keys(keySet));
+				} else {
+					return doIteration();
+				}
+			})
+		;
+		return doIteration();
+	},
 
-	# extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b
-	# or DocsWithHistoryOps:{57fd0b1f53a8396d22b2c24b} (for redis cluster)
-	_extractIds: (keyList) ->
-		ids = for key in keyList
-			m = key.match(/:\{?([0-9a-f]{24})\}?/) # extract object id
-			m[1]
-		return ids
+	// extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b
+	// or DocsWithHistoryOps:{57fd0b1f53a8396d22b2c24b} (for redis cluster)
+	_extractIds(keyList) {
+		const ids = (() => {
+			const result = [];
+			for (let key of Array.from(keyList)) {
+				const m = key.match(/:\{?([0-9a-f]{24})\}?/); // extract object id
+				result.push(m[1]);
+			}
+			return result;
+		})();
+		return ids;
+	},
 
-	getProjectIdsWithHistoryOps: (callback = (error, project_ids) ->) ->
-		RedisManager._getKeys Keys.docsWithHistoryOps({project_id:"*"}), (error, project_keys) ->
-			return callback(error) if error?
-			project_ids = RedisManager._extractIds project_keys
-			callback(error, project_ids)
+	getProjectIdsWithHistoryOps(callback) {
+		if (callback == null) { callback = function(error, project_ids) {}; }
+		return RedisManager._getKeys(Keys.docsWithHistoryOps({project_id:"*"}), function(error, project_keys) {
+			if (error != null) { return callback(error); }
+			const project_ids = RedisManager._extractIds(project_keys);
+			return callback(error, project_ids);
+		});
+	},
 
-	getAllDocIdsWithHistoryOps: (callback = (error, doc_ids) ->) ->
-		# return all the docids, to find dangling history entries after
-		# everything is flushed.
-		RedisManager._getKeys Keys.uncompressedHistoryOps({doc_id:"*"}), (error, doc_keys) ->
-			return callback(error) if error?
-			doc_ids = RedisManager._extractIds doc_keys
-			callback(error, doc_ids)
+	getAllDocIdsWithHistoryOps(callback) {
+		// return all the docids, to find dangling history entries after
+		// everything is flushed.
+		if (callback == null) { callback = function(error, doc_ids) {}; }
+		return RedisManager._getKeys(Keys.uncompressedHistoryOps({doc_id:"*"}), function(error, doc_keys) {
+			if (error != null) { return callback(error); }
+			const doc_ids = RedisManager._extractIds(doc_keys);
+			return callback(error, doc_ids);
+		});
+	}
+});

services/track-changes/app/coffee/RestoreManager.js

@@ -1,12 +1,24 @@
-DocumentUpdaterManager = require "./DocumentUpdaterManager"
-DiffManager = require "./DiffManager"
-logger = require "logger-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RestoreManager;
+const DocumentUpdaterManager = require("./DocumentUpdaterManager");
+const DiffManager = require("./DiffManager");
+const logger = require("logger-sharelatex");
 
-module.exports = RestoreManager =
-	restoreToBeforeVersion: (project_id, doc_id, version, user_id, callback = (error) ->) ->
-		logger.log project_id: project_id, doc_id: doc_id, version: version, user_id: user_id, "restoring document"
-		DiffManager.getDocumentBeforeVersion project_id, doc_id, version, (error, content) ->
-			return callback(error) if error?
-			DocumentUpdaterManager.setDocument project_id, doc_id, content, user_id, (error) ->
-				return callback(error) if error?
-				callback()
+module.exports = (RestoreManager = {
+	restoreToBeforeVersion(project_id, doc_id, version, user_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		logger.log({project_id, doc_id, version, user_id}, "restoring document");
+		return DiffManager.getDocumentBeforeVersion(project_id, doc_id, version, function(error, content) {
+			if (error != null) { return callback(error); }
+			return DocumentUpdaterManager.setDocument(project_id, doc_id, content, user_id, function(error) {
+				if (error != null) { return callback(error); }
+				return callback();
+			});
+		});
+	}
+});

services/track-changes/app/coffee/UpdateCompressor.js

@@ -1,218 +1,278 @@
-strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..]
-strRemove = (s1, pos, length) -> s1[...pos] + s1[(pos + length)..]
-
-diff_match_patch = require("../lib/diff_match_patch").diff_match_patch
-dmp = new diff_match_patch()
-
-module.exports = UpdateCompressor =
-	NOOP: "noop"
-
-	# Updates come from the doc updater in format
-	# {
-	# 	op:   [ { ... op1 ... }, { ... op2 ... } ]
-	# 	meta: { ts: ..., user_id: ... }
-	# }
-	# but it's easier to work with on op per update, so convert these updates to
-	# our compressed format
-	# [{
-	# 	op: op1
-	# 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
-	# }, {
-	# 	op: op2
-	# 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
-	# }]
-	convertToSingleOpUpdates: (updates) ->
-		splitUpdates = []
-		for update in updates
-			# Reject any non-insert or delete ops, i.e. comments
-			ops = update.op.filter (o) -> o.i? or o.d?
-			if ops.length == 0
-				splitUpdates.push
-					op: UpdateCompressor.NOOP
-					meta:
-						start_ts: update.meta.start_ts or update.meta.ts
-						end_ts:   update.meta.end_ts   or update.meta.ts
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let oneMinute, twoMegabytes, UpdateCompressor;
+const strInject = (s1, pos, s2) => s1.slice(0, pos) + s2 + s1.slice(pos);
+const strRemove = (s1, pos, length) => s1.slice(0, pos) + s1.slice((pos + length));
+
+const { diff_match_patch } = require("../lib/diff_match_patch");
+const dmp = new diff_match_patch();
+
+module.exports = (UpdateCompressor = {
+	NOOP: "noop",
+
+	// Updates come from the doc updater in format
+	// {
+	// 	op:   [ { ... op1 ... }, { ... op2 ... } ]
+	// 	meta: { ts: ..., user_id: ... }
+	// }
+	// but it's easier to work with on op per update, so convert these updates to
+	// our compressed format
+	// [{
+	// 	op: op1
+	// 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
+	// }, {
+	// 	op: op2
+	// 	meta: { start_ts: ... , end_ts: ..., user_id: ... }
+	// }]
+	convertToSingleOpUpdates(updates) {
+		const splitUpdates = [];
+		for (let update of Array.from(updates)) {
+			// Reject any non-insert or delete ops, i.e. comments
+			const ops = update.op.filter(o => (o.i != null) || (o.d != null));
+			if (ops.length === 0) {
+				splitUpdates.push({
+					op: UpdateCompressor.NOOP,
+					meta: {
+						start_ts: update.meta.start_ts || update.meta.ts,
+						end_ts:   update.meta.end_ts   || update.meta.ts,
 						user_id:  update.meta.user_id
+					},
 					v: update.v
-			else
-				for op in ops
-					splitUpdates.push
-						op: op
-						meta:
-							start_ts: update.meta.start_ts or update.meta.ts
-							end_ts:   update.meta.end_ts   or update.meta.ts
+				});
+			} else {
+				for (let op of Array.from(ops)) {
+					splitUpdates.push({
+						op,
+						meta: {
+							start_ts: update.meta.start_ts || update.meta.ts,
+							end_ts:   update.meta.end_ts   || update.meta.ts,
 							user_id:  update.meta.user_id
+						},
 						v: update.v
-		return splitUpdates
-
-	concatUpdatesWithSameVersion: (updates) ->
-		concattedUpdates = []
-		for update in updates
-			lastUpdate = concattedUpdates[concattedUpdates.length - 1]
-			if lastUpdate? and lastUpdate.v == update.v
-				lastUpdate.op.push update.op unless update.op == UpdateCompressor.NOOP
-			else
-				nextUpdate =
-					op:   []
-					meta: update.meta
+					});
+				}
+			}
+		}
+		return splitUpdates;
+	},
+
+	concatUpdatesWithSameVersion(updates) {
+		const concattedUpdates = [];
+		for (let update of Array.from(updates)) {
+			const lastUpdate = concattedUpdates[concattedUpdates.length - 1];
+			if ((lastUpdate != null) && (lastUpdate.v === update.v)) {
+				if (update.op !== UpdateCompressor.NOOP) { lastUpdate.op.push(update.op); }
+			} else {
+				const nextUpdate = {
+					op:   [],
+					meta: update.meta,
 					v:    update.v
-				nextUpdate.op.push update.op unless update.op == UpdateCompressor.NOOP
-				concattedUpdates.push nextUpdate
-		return concattedUpdates
-
-	compressRawUpdates: (lastPreviousUpdate, rawUpdates) ->
-		if lastPreviousUpdate?.op?.length > 1
-			# if the last previous update was an array op, don't compress onto it.
-			# The avoids cases where array length changes but version number doesn't
-			return [lastPreviousUpdate].concat UpdateCompressor.compressRawUpdates(null,rawUpdates)
-		if lastPreviousUpdate?
-			rawUpdates = [lastPreviousUpdate].concat(rawUpdates)
-		updates = UpdateCompressor.convertToSingleOpUpdates(rawUpdates)
-		updates = UpdateCompressor.compressUpdates(updates)
-		return UpdateCompressor.concatUpdatesWithSameVersion(updates)
-
-	compressUpdates: (updates) ->
-		return [] if updates.length == 0
-
-		compressedUpdates = [updates.shift()]
-		for update in updates
-			lastCompressedUpdate = compressedUpdates.pop()
-			if lastCompressedUpdate?
-				compressedUpdates = compressedUpdates.concat UpdateCompressor._concatTwoUpdates lastCompressedUpdate, update
-			else
-				compressedUpdates.push update
-
-		return compressedUpdates
-
-	MAX_TIME_BETWEEN_UPDATES: oneMinute = 60 * 1000
-	MAX_UPDATE_SIZE: twoMegabytes = 2* 1024 * 1024
-
-	_concatTwoUpdates: (firstUpdate, secondUpdate) ->
-		firstUpdate =
-			op: firstUpdate.op
-			meta:
-				user_id:  firstUpdate.meta.user_id or null
-				start_ts: firstUpdate.meta.start_ts or firstUpdate.meta.ts
-				end_ts:   firstUpdate.meta.end_ts   or firstUpdate.meta.ts
+				};
+				if (update.op !== UpdateCompressor.NOOP) { nextUpdate.op.push(update.op); }
+				concattedUpdates.push(nextUpdate);
+			}
+		}
+		return concattedUpdates;
+	},
+
+	compressRawUpdates(lastPreviousUpdate, rawUpdates) {
+		if (__guard__(lastPreviousUpdate != null ? lastPreviousUpdate.op : undefined, x => x.length) > 1) {
+			// if the last previous update was an array op, don't compress onto it.
+			// The avoids cases where array length changes but version number doesn't
+			return [lastPreviousUpdate].concat(UpdateCompressor.compressRawUpdates(null,rawUpdates));
+		}
+		if (lastPreviousUpdate != null) {
+			rawUpdates = [lastPreviousUpdate].concat(rawUpdates);
+		}
+		let updates = UpdateCompressor.convertToSingleOpUpdates(rawUpdates);
+		updates = UpdateCompressor.compressUpdates(updates);
+		return UpdateCompressor.concatUpdatesWithSameVersion(updates);
+	},
+
+	compressUpdates(updates) {
+		if (updates.length === 0) { return []; }
+
+		let compressedUpdates = [updates.shift()];
+		for (let update of Array.from(updates)) {
+			const lastCompressedUpdate = compressedUpdates.pop();
+			if (lastCompressedUpdate != null) {
+				compressedUpdates = compressedUpdates.concat(UpdateCompressor._concatTwoUpdates(lastCompressedUpdate, update));
+			} else {
+				compressedUpdates.push(update);
+			}
+		}
+
+		return compressedUpdates;
+	},
+
+	MAX_TIME_BETWEEN_UPDATES: (oneMinute = 60 * 1000),
+	MAX_UPDATE_SIZE: (twoMegabytes = 2* 1024 * 1024),
+
+	_concatTwoUpdates(firstUpdate, secondUpdate) {
+		let offset;
+		firstUpdate = {
+			op: firstUpdate.op,
+			meta: {
+				user_id:  firstUpdate.meta.user_id || null,
+				start_ts: firstUpdate.meta.start_ts || firstUpdate.meta.ts,
+				end_ts:   firstUpdate.meta.end_ts   || firstUpdate.meta.ts
+			},
 			v: firstUpdate.v
-		secondUpdate =
-			op: secondUpdate.op
-			meta:
-				user_id:  secondUpdate.meta.user_id or null
-				start_ts: secondUpdate.meta.start_ts or secondUpdate.meta.ts
-				end_ts:   secondUpdate.meta.end_ts   or secondUpdate.meta.ts
+		};
+		secondUpdate = {
+			op: secondUpdate.op,
+			meta: {
+				user_id:  secondUpdate.meta.user_id || null,
+				start_ts: secondUpdate.meta.start_ts || secondUpdate.meta.ts,
+				end_ts:   secondUpdate.meta.end_ts   || secondUpdate.meta.ts
+			},
 			v: secondUpdate.v
+		};
 
-		if firstUpdate.meta.user_id != secondUpdate.meta.user_id
-			return [firstUpdate, secondUpdate]
+		if (firstUpdate.meta.user_id !== secondUpdate.meta.user_id) {
+			return [firstUpdate, secondUpdate];
+		}
 
-		if secondUpdate.meta.start_ts - firstUpdate.meta.end_ts > UpdateCompressor.MAX_TIME_BETWEEN_UPDATES
-			return [firstUpdate, secondUpdate]
+		if ((secondUpdate.meta.start_ts - firstUpdate.meta.end_ts) > UpdateCompressor.MAX_TIME_BETWEEN_UPDATES) {
+			return [firstUpdate, secondUpdate];
+		}
 
-		firstOp = firstUpdate.op
-		secondOp = secondUpdate.op
+		const firstOp = firstUpdate.op;
+		const secondOp = secondUpdate.op;
 
-		firstSize = firstOp.i?.length or firstOp.d?.length
-		secondSize = secondOp.i?.length or secondOp.d?.length
+		const firstSize = (firstOp.i != null ? firstOp.i.length : undefined) || (firstOp.d != null ? firstOp.d.length : undefined);
+		const secondSize = (secondOp.i != null ? secondOp.i.length : undefined) || (secondOp.d != null ? secondOp.d.length : undefined);
 
-		# Two inserts
-		if firstOp.i? and secondOp.i? and firstOp.p <= secondOp.p <= (firstOp.p + firstOp.i.length) and firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
-			return [
-				meta:
-					start_ts: firstUpdate.meta.start_ts
-					end_ts:   secondUpdate.meta.end_ts
+		// Two inserts
+		if ((firstOp.i != null) && (secondOp.i != null) && (firstOp.p <= secondOp.p && secondOp.p <= (firstOp.p + firstOp.i.length)) && ((firstSize + secondSize) < UpdateCompressor.MAX_UPDATE_SIZE)) {
+			return [{
+				meta: {
+					start_ts: firstUpdate.meta.start_ts,
+					end_ts:   secondUpdate.meta.end_ts,
 					user_id:  firstUpdate.meta.user_id
-				op:
-					p: firstOp.p
+				},
+				op: {
+					p: firstOp.p,
 					i: strInject(firstOp.i, secondOp.p - firstOp.p, secondOp.i)
+				},
 				v: secondUpdate.v
-			]
-		# Two deletes
-		else if firstOp.d? and secondOp.d? and secondOp.p <= firstOp.p <= (secondOp.p + secondOp.d.length) and firstSize + secondSize < UpdateCompressor.MAX_UPDATE_SIZE
-			return [
-				meta:
-					start_ts: firstUpdate.meta.start_ts
-					end_ts:   secondUpdate.meta.end_ts
+			}
+			];
+		// Two deletes
+		} else if ((firstOp.d != null) && (secondOp.d != null) && (secondOp.p <= firstOp.p && firstOp.p <= (secondOp.p + secondOp.d.length)) && ((firstSize + secondSize) < UpdateCompressor.MAX_UPDATE_SIZE)) {
+			return [{
+				meta: {
+					start_ts: firstUpdate.meta.start_ts,
+					end_ts:   secondUpdate.meta.end_ts,
 					user_id:  firstUpdate.meta.user_id
-				op:
-					p: secondOp.p
+				},
+				op: {
+					p: secondOp.p,
 					d: strInject(secondOp.d, firstOp.p - secondOp.p, firstOp.d)
+				},
 				v: secondUpdate.v
-			]
-		# An insert and then a delete
-		else if firstOp.i? and secondOp.d? and firstOp.p <= secondOp.p <= (firstOp.p + firstOp.i.length)
-			offset = secondOp.p - firstOp.p
-			insertedText = firstOp.i.slice(offset, offset + secondOp.d.length)
-			# Only trim the insert when the delete is fully contained within in it
-			if insertedText == secondOp.d
-				insert = strRemove(firstOp.i, offset, secondOp.d.length)
-				return [
-					meta:
-						start_ts: firstUpdate.meta.start_ts
-						end_ts:   secondUpdate.meta.end_ts
+			}
+			];
+		// An insert and then a delete
+		} else if ((firstOp.i != null) && (secondOp.d != null) && (firstOp.p <= secondOp.p && secondOp.p <= (firstOp.p + firstOp.i.length))) {
+			offset = secondOp.p - firstOp.p;
+			const insertedText = firstOp.i.slice(offset, offset + secondOp.d.length);
+			// Only trim the insert when the delete is fully contained within in it
+			if (insertedText === secondOp.d) {
+				const insert = strRemove(firstOp.i, offset, secondOp.d.length);
+				return [{
+					meta: {
+						start_ts: firstUpdate.meta.start_ts,
+						end_ts:   secondUpdate.meta.end_ts,
 						user_id:  firstUpdate.meta.user_id
-					op:
-						p: firstOp.p
+					},
+					op: {
+						p: firstOp.p,
 						i: insert
+					},
 					v: secondUpdate.v
-				]
-			else
-				# This will only happen if the delete extends outside the insert
-				return [firstUpdate, secondUpdate]
-
-		# A delete then an insert at the same place, likely a copy-paste of a chunk of content
-		else if firstOp.d? and secondOp.i? and firstOp.p == secondOp.p
-			offset = firstOp.p
-			diff_ops = @diffAsShareJsOps(firstOp.d, secondOp.i)
-			if diff_ops.length == 0
-				return [{ # Noop
-					meta:
-						start_ts: firstUpdate.meta.start_ts
-						end_ts:   secondUpdate.meta.end_ts
+				}
+				];
+			} else {
+				// This will only happen if the delete extends outside the insert
+				return [firstUpdate, secondUpdate];
+			}
+
+		// A delete then an insert at the same place, likely a copy-paste of a chunk of content
+		} else if ((firstOp.d != null) && (secondOp.i != null) && (firstOp.p === secondOp.p)) {
+			offset = firstOp.p;
+			const diff_ops = this.diffAsShareJsOps(firstOp.d, secondOp.i);
+			if (diff_ops.length === 0) {
+				return [{ // Noop
+					meta: {
+						start_ts: firstUpdate.meta.start_ts,
+						end_ts:   secondUpdate.meta.end_ts,
 						user_id:  firstUpdate.meta.user_id
-					op:
-						p: firstOp.p
+					},
+					op: {
+						p: firstOp.p,
 						i: ""
+					},
 					v: secondUpdate.v
-				}]
-			else
-				return diff_ops.map (op) ->
-					op.p += offset
+				}];
+			} else {
+				return diff_ops.map(function(op) {
+					op.p += offset;
 					return {
-						meta:
-							start_ts: firstUpdate.meta.start_ts
-							end_ts:   secondUpdate.meta.end_ts
+						meta: {
+							start_ts: firstUpdate.meta.start_ts,
+							end_ts:   secondUpdate.meta.end_ts,
 							user_id:  firstUpdate.meta.user_id
-						op: op
+						},
+						op,
 						v: secondUpdate.v
-					}
-
-		else
-			return [firstUpdate, secondUpdate]
-
-	ADDED: 1
-	REMOVED: -1
-	UNCHANGED: 0
-	diffAsShareJsOps: (before, after, callback = (error, ops) ->) ->
-		diffs = dmp.diff_main(before, after)
-		dmp.diff_cleanupSemantic(diffs)
-
-		ops = []
-		position = 0
-		for diff in diffs
-			type = diff[0]
-			content = diff[1]
-			if type == @ADDED
-				ops.push
-					i: content
+					};});
+			}
+
+		} else {
+			return [firstUpdate, secondUpdate];
+		}
+	},
+
+	ADDED: 1,
+	REMOVED: -1,
+	UNCHANGED: 0,
+	diffAsShareJsOps(before, after, callback) {
+		if (callback == null) { callback = function(error, ops) {}; }
+		const diffs = dmp.diff_main(before, after);
+		dmp.diff_cleanupSemantic(diffs);
+
+		const ops = [];
+		let position = 0;
+		for (let diff of Array.from(diffs)) {
+			const type = diff[0];
+			const content = diff[1];
+			if (type === this.ADDED) {
+				ops.push({
+					i: content,
 					p: position
-				position += content.length
-			else if type == @REMOVED
-				ops.push
-					d: content
+				});
+				position += content.length;
+			} else if (type === this.REMOVED) {
+				ops.push({
+					d: content,
 					p: position
-			else if type == @UNCHANGED
-				position += content.length
-			else
-				throw "Unknown type"
-		return ops
+				});
+			} else if (type === this.UNCHANGED) {
+				position += content.length;
+			} else {
+				throw "Unknown type";
+			}
+		}
+		return ops;
+	}
+});
+
+function __guard__(value, transform) {
+  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
+}

services/track-changes/app/coffee/UpdateTrimmer.js

@@ -1,23 +1,44 @@
-MongoManager = require "./MongoManager"
-WebApiManager = require "./WebApiManager"
-logger = require "logger-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let UpdateTrimmer;
+const MongoManager = require("./MongoManager");
+const WebApiManager = require("./WebApiManager");
+const logger = require("logger-sharelatex");
 
-module.exports = UpdateTrimmer =
-	shouldTrimUpdates: (project_id, callback = (error, shouldTrim) ->) ->
-		MongoManager.getProjectMetaData project_id, (error, metadata) ->
-			return callback(error) if error?
-			if metadata?.preserveHistory
-				return callback null, false
-			else
-				WebApiManager.getProjectDetails project_id, (error, details) ->
-					return callback(error) if error?
-					logger.log project_id: project_id, details: details, "got details"
-					if details?.features?.versioning
-						MongoManager.setProjectMetaData project_id, preserveHistory: true, (error) ->
-							return callback(error) if error?
-							MongoManager.upgradeHistory project_id, (error) ->
-								return callback(error) if error?
-								callback null, false
-					else
-						callback null, true
+module.exports = (UpdateTrimmer = {
+	shouldTrimUpdates(project_id, callback) {
+		if (callback == null) { callback = function(error, shouldTrim) {}; }
+		return MongoManager.getProjectMetaData(project_id, function(error, metadata) {
+			if (error != null) { return callback(error); }
+			if (metadata != null ? metadata.preserveHistory : undefined) {
+				return callback(null, false);
+			} else {
+				return WebApiManager.getProjectDetails(project_id, function(error, details) {
+					if (error != null) { return callback(error); }
+					logger.log({project_id, details}, "got details");
+					if (__guard__(details != null ? details.features : undefined, x => x.versioning)) {
+						return MongoManager.setProjectMetaData(project_id, {preserveHistory: true}, function(error) {
+							if (error != null) { return callback(error); }
+							return MongoManager.upgradeHistory(project_id, function(error) {
+								if (error != null) { return callback(error); }
+								return callback(null, false);
+							});
+						});
+					} else {
+						return callback(null, true);
+					}
+				});
+			}
+		});
+	}
+});
 
+
+function __guard__(value, transform) {
+  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
+}

services/track-changes/app/coffee/UpdatesManager.js

@@ -1,344 +1,494 @@
-MongoManager = require "./MongoManager"
-PackManager = require "./PackManager"
-RedisManager = require "./RedisManager"
-UpdateCompressor = require "./UpdateCompressor"
-LockManager = require "./LockManager"
-WebApiManager = require "./WebApiManager"
-UpdateTrimmer = require "./UpdateTrimmer"
-logger = require "logger-sharelatex"
-async = require "async"
-_ = require "underscore"
-Settings = require "settings-sharelatex"
-keys = Settings.redis.lock.key_schema
-
-module.exports = UpdatesManager =
-	compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) ->
-		length = rawUpdates.length
-		if length == 0
-			return callback()
-
-		# check that ops are in the correct order
-		for op, i in rawUpdates when i > 0
-			thisVersion = op?.v
-			prevVersion = rawUpdates[i-1]?.v
-			if not (prevVersion < thisVersion)
-				logger.error project_id: project_id, doc_id: doc_id, rawUpdates:rawUpdates, temporary: temporary, thisVersion:thisVersion, prevVersion:prevVersion, "op versions out of order"
-
-		# FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
-		# CORRECTION:  we do use it to log the time in case of error
-		MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
-			# lastCompressedUpdate is the most recent update in Mongo, and
-			# lastVersion is its sharejs version number.
-			#
-			# The peekLastCompressedUpdate method may pass the update back
-			# as 'null' (for example if the previous compressed update has
-			# been archived).  In this case it can still pass back the
-			# lastVersion from the update to allow us to check consistency.
-			return callback(error) if error?
-
-			# Ensure that raw updates start where lastVersion left off
-			if lastVersion?
-				discardedUpdates = []
-				rawUpdates = rawUpdates.slice(0)
-				while rawUpdates[0]? and rawUpdates[0].v <= lastVersion
-					discardedUpdates.push rawUpdates.shift()
-				if discardedUpdates.length
-					logger.error project_id: project_id, doc_id: doc_id, discardedUpdates: discardedUpdates, temporary: temporary, lastVersion: lastVersion, "discarded updates already present"
-
-				if rawUpdates[0]? and rawUpdates[0].v != lastVersion + 1
-					ts = lastCompressedUpdate?.meta?.end_ts
-					last_timestamp = if ts? then new Date(ts) else 'unknown time'
-					error = new Error("Tried to apply raw op at version #{rawUpdates[0].v} to last compressed update with version #{lastVersion} from #{last_timestamp}")
-					logger.error err: error, doc_id: doc_id, project_id: project_id, prev_end_ts: ts, temporary: temporary, lastCompressedUpdate: lastCompressedUpdate, "inconsistent doc versions"
-					if Settings.trackchanges?.continueOnError and rawUpdates[0].v > lastVersion + 1
-						# we have lost some ops - continue to write into the database, we can't recover at this point
-						lastCompressedUpdate = null
-					else
-						return callback error
-
-			if rawUpdates.length == 0
-				return callback()
-
-			# some old large ops in redis need to be rejected, they predate
-			# the size limit that now prevents them going through the system
-			REJECT_LARGE_OP_SIZE = 4 * 1024 * 1024
-			for rawUpdate in rawUpdates
-				opSizes = ((op.i?.length || op.d?.length) for op in rawUpdate?.op or [])
-				size = _.max opSizes
-				if size > REJECT_LARGE_OP_SIZE
-					error = new Error("dropped op exceeding maximum allowed size of #{REJECT_LARGE_OP_SIZE}")
-					logger.error err: error, doc_id: doc_id, project_id: project_id, size: size, rawUpdate: rawUpdate, "dropped op - too big"
-					rawUpdate.op = []
-
-			compressedUpdates = UpdateCompressor.compressRawUpdates null, rawUpdates
-			PackManager.insertCompressedUpdates project_id, doc_id, lastCompressedUpdate, compressedUpdates, temporary, (error, result) ->
-				return callback(error) if error?
-				logger.log {project_id, doc_id, orig_v: lastCompressedUpdate?.v, new_v: result.v}, "inserted updates into pack"	if result?
-				callback()
-
-	# Check whether the updates are temporary (per-project property)
-	_prepareProjectForUpdates: (project_id, callback = (error, temporary) ->) ->
-		UpdateTrimmer.shouldTrimUpdates project_id, (error, temporary) ->
-			return callback(error) if error?
-			callback(null, temporary)
-
-	# Check for project id on document history (per-document property)
-	_prepareDocForUpdates: (project_id, doc_id, callback = (error) ->) ->
-		MongoManager.backportProjectId project_id, doc_id, (error) ->
-			return callback(error) if error?
-			callback(null)
-
-	# Apply updates for specific project/doc after preparing at project and doc level
-	REDIS_READ_BATCH_SIZE: 100
-	processUncompressedUpdates: (project_id, doc_id, temporary, callback = (error) ->) ->
-		# get the updates as strings from redis (so we can delete them after they are applied)
-		RedisManager.getOldestDocUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, docUpdates) ->
-			return callback(error) if error?
-			length = docUpdates.length
-			# parse the redis strings into ShareJs updates
-			RedisManager.expandDocUpdates docUpdates, (error, rawUpdates) ->
-				if error?
-					logger.err project_id: project_id, doc_id: doc_id, docUpdates: docUpdates, "failed to parse docUpdates"
-					return callback(error)
-				logger.log project_id: project_id, doc_id: doc_id, rawUpdates: rawUpdates, "retrieved raw updates from redis"
-				UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, temporary, (error) ->
-					return callback(error) if error?
-					logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates"
-					# delete the applied updates from redis
-					RedisManager.deleteAppliedDocUpdates project_id, doc_id, docUpdates, (error) ->
-						return callback(error) if error?
-						if length == UpdatesManager.REDIS_READ_BATCH_SIZE
-							# There might be more updates
-							logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates"
-							setTimeout () ->
-								UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, callback
-							, 0
-						else
-							logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed"
-							callback()
-
-	# Process updates for a doc when we flush it individually
-	processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
-		UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
-			return callback(error) if error?
-			UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, callback
-
-
-	# Process updates for a doc when the whole project is flushed (internal method)
-	_processUncompressedUpdatesForDocWithLock: (project_id, doc_id, temporary, callback = (error) ->) ->
-		UpdatesManager._prepareDocForUpdates project_id, doc_id, (error) ->
-			return callback(error) if error?
-			LockManager.runWithLock(
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let fiveMinutes, UpdatesManager;
+const MongoManager = require("./MongoManager");
+const PackManager = require("./PackManager");
+const RedisManager = require("./RedisManager");
+const UpdateCompressor = require("./UpdateCompressor");
+const LockManager = require("./LockManager");
+const WebApiManager = require("./WebApiManager");
+const UpdateTrimmer = require("./UpdateTrimmer");
+const logger = require("logger-sharelatex");
+const async = require("async");
+const _ = require("underscore");
+const Settings = require("settings-sharelatex");
+const keys = Settings.redis.lock.key_schema;
+
+module.exports = (UpdatesManager = {
+	compressAndSaveRawUpdates(project_id, doc_id, rawUpdates, temporary, callback) {
+		let i;
+		if (callback == null) { callback = function(error) {}; }
+		const { length } = rawUpdates;
+		if (length === 0) {
+			return callback();
+		}
+
+		// check that ops are in the correct order
+		for (i = 0; i < rawUpdates.length; i++) {
+			const op = rawUpdates[i];
+			if (i > 0) {
+				const thisVersion = op != null ? op.v : undefined;
+				const prevVersion = __guard__(rawUpdates[i-1], x => x.v);
+				if (!(prevVersion < thisVersion)) {
+					logger.error({project_id, doc_id, rawUpdates, temporary, thisVersion, prevVersion}, "op versions out of order");
+				}
+			}
+		}
+
+		// FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
+		// CORRECTION:  we do use it to log the time in case of error
+		return MongoManager.peekLastCompressedUpdate(doc_id, function(error, lastCompressedUpdate, lastVersion) {
+			// lastCompressedUpdate is the most recent update in Mongo, and
+			// lastVersion is its sharejs version number.
+			//
+			// The peekLastCompressedUpdate method may pass the update back
+			// as 'null' (for example if the previous compressed update has
+			// been archived).  In this case it can still pass back the
+			// lastVersion from the update to allow us to check consistency.
+			let op;
+			if (error != null) { return callback(error); }
+
+			// Ensure that raw updates start where lastVersion left off
+			if (lastVersion != null) {
+				const discardedUpdates = [];
+				rawUpdates = rawUpdates.slice(0);
+				while ((rawUpdates[0] != null) && (rawUpdates[0].v <= lastVersion)) {
+					discardedUpdates.push(rawUpdates.shift());
+				}
+				if (discardedUpdates.length) {
+					logger.error({project_id, doc_id, discardedUpdates, temporary, lastVersion}, "discarded updates already present");
+				}
+
+				if ((rawUpdates[0] != null) && (rawUpdates[0].v !== (lastVersion + 1))) {
+					const ts = __guard__(lastCompressedUpdate != null ? lastCompressedUpdate.meta : undefined, x1 => x1.end_ts);
+					const last_timestamp = (ts != null) ? new Date(ts) : 'unknown time';
+					error = new Error(`Tried to apply raw op at version ${rawUpdates[0].v} to last compressed update with version ${lastVersion} from ${last_timestamp}`);
+					logger.error({err: error, doc_id, project_id, prev_end_ts: ts, temporary, lastCompressedUpdate}, "inconsistent doc versions");
+					if ((Settings.trackchanges != null ? Settings.trackchanges.continueOnError : undefined) && (rawUpdates[0].v > (lastVersion + 1))) {
+						// we have lost some ops - continue to write into the database, we can't recover at this point
+						lastCompressedUpdate = null;
+					} else {
+						return callback(error);
+					}
+				}
+			}
+
+			if (rawUpdates.length === 0) {
+				return callback();
+			}
+
+			// some old large ops in redis need to be rejected, they predate
+			// the size limit that now prevents them going through the system
+			const REJECT_LARGE_OP_SIZE = 4 * 1024 * 1024;
+			for (var rawUpdate of Array.from(rawUpdates)) {
+				const opSizes = ((() => {
+					const result = [];
+					for (op of Array.from((rawUpdate != null ? rawUpdate.op : undefined) || [])) { 						result.push(((op.i != null ? op.i.length : undefined) || (op.d != null ? op.d.length : undefined)));
+					}
+					return result;
+				})());
+				const size = _.max(opSizes);
+				if (size > REJECT_LARGE_OP_SIZE) {
+					error = new Error(`dropped op exceeding maximum allowed size of ${REJECT_LARGE_OP_SIZE}`);
+					logger.error({err: error, doc_id, project_id, size, rawUpdate}, "dropped op - too big");
+					rawUpdate.op = [];
+				}
+			}
+
+			const compressedUpdates = UpdateCompressor.compressRawUpdates(null, rawUpdates);
+			return PackManager.insertCompressedUpdates(project_id, doc_id, lastCompressedUpdate, compressedUpdates, temporary, function(error, result) {
+				if (error != null) { return callback(error); }
+				if (result != null) { logger.log({project_id, doc_id, orig_v: (lastCompressedUpdate != null ? lastCompressedUpdate.v : undefined), new_v: result.v}, "inserted updates into pack"); }
+				return callback();
+			});
+		});
+	},
+
+	// Check whether the updates are temporary (per-project property)
+	_prepareProjectForUpdates(project_id, callback) {
+		if (callback == null) { callback = function(error, temporary) {}; }
+		return UpdateTrimmer.shouldTrimUpdates(project_id, function(error, temporary) {
+			if (error != null) { return callback(error); }
+			return callback(null, temporary);
+		});
+	},
+
+	// Check for project id on document history (per-document property)
+	_prepareDocForUpdates(project_id, doc_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return MongoManager.backportProjectId(project_id, doc_id, function(error) {
+			if (error != null) { return callback(error); }
+			return callback(null);
+		});
+	},
+
+	// Apply updates for specific project/doc after preparing at project and doc level
+	REDIS_READ_BATCH_SIZE: 100,
+	processUncompressedUpdates(project_id, doc_id, temporary, callback) {
+		// get the updates as strings from redis (so we can delete them after they are applied)
+		if (callback == null) { callback = function(error) {}; }
+		return RedisManager.getOldestDocUpdates(doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, function(error, docUpdates) {
+			if (error != null) { return callback(error); }
+			const { length } = docUpdates;
+			// parse the redis strings into ShareJs updates
+			return RedisManager.expandDocUpdates(docUpdates, function(error, rawUpdates) {
+				if (error != null) {
+					logger.err({project_id, doc_id, docUpdates}, "failed to parse docUpdates");
+					return callback(error);
+				}
+				logger.log({project_id, doc_id, rawUpdates}, "retrieved raw updates from redis");
+				return UpdatesManager.compressAndSaveRawUpdates(project_id, doc_id, rawUpdates, temporary, function(error) {
+					if (error != null) { return callback(error); }
+					logger.log({project_id, doc_id}, "compressed and saved doc updates");
+					// delete the applied updates from redis
+					return RedisManager.deleteAppliedDocUpdates(project_id, doc_id, docUpdates, function(error) {
+						if (error != null) { return callback(error); }
+						if (length === UpdatesManager.REDIS_READ_BATCH_SIZE) {
+							// There might be more updates
+							logger.log({project_id, doc_id}, "continuing processing updates");
+							return setTimeout(() => UpdatesManager.processUncompressedUpdates(project_id, doc_id, temporary, callback)
+							, 0);
+						} else {
+							logger.log({project_id, doc_id}, "all raw updates processed");
+							return callback();
+						}
+					});
+				});
+			});
+		});
+	},
+
+	// Process updates for a doc when we flush it individually
+	processUncompressedUpdatesWithLock(project_id, doc_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return UpdatesManager._prepareProjectForUpdates(project_id, function(error, temporary) {
+			if (error != null) { return callback(error); }
+			return UpdatesManager._processUncompressedUpdatesForDocWithLock(project_id, doc_id, temporary, callback);
+		});
+	},
+
+
+	// Process updates for a doc when the whole project is flushed (internal method)
+	_processUncompressedUpdatesForDocWithLock(project_id, doc_id, temporary, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return UpdatesManager._prepareDocForUpdates(project_id, doc_id, function(error) {
+			if (error != null) { return callback(error); }
+			return LockManager.runWithLock(
 				keys.historyLock({doc_id}),
-				(releaseLock) ->
-					UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, releaseLock
+				releaseLock => UpdatesManager.processUncompressedUpdates(project_id, doc_id, temporary, releaseLock),
 				callback
-			)
-
-	# Process all updates for a project, only check project-level information once
-	processUncompressedUpdatesForProject: (project_id, callback = (error) ->) ->
-		RedisManager.getDocIdsWithHistoryOps project_id, (error, doc_ids) ->
-			return callback(error) if error?
-			UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
-				jobs = []
-				for doc_id in doc_ids
-					do (doc_id) ->
-						jobs.push (cb) ->
-							UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, cb
-				async.parallelLimit jobs, 5, callback
-
-	# flush all outstanding changes
-	flushAll: (limit, callback = (error, result) ->) ->
-		RedisManager.getProjectIdsWithHistoryOps (error, project_ids) ->
-			return callback(error) if error?
-			logger.log {count: project_ids?.length, project_ids: project_ids}, "found projects"
-			jobs = []
-			project_ids = _.shuffle project_ids # randomise to avoid hitting same projects each time
-			selectedProjects = if limit < 0 then project_ids else project_ids[0...limit]
-			for project_id in selectedProjects
-				do (project_id) ->
-					jobs.push (cb) ->
-						UpdatesManager.processUncompressedUpdatesForProject project_id, (err) ->
-							return cb(null, {failed: err?, project_id: project_id})
-			async.series jobs, (error, result) ->
-				return callback(error) if error?
-				failedProjects = (x.project_id for x in result when x.failed)
-				succeededProjects = (x.project_id for x in result when not x.failed)
-				callback(null, {failed: failedProjects, succeeded: succeededProjects, all: project_ids})
-
-	getDanglingUpdates: (callback = (error, doc_ids) ->) ->
-		RedisManager.getAllDocIdsWithHistoryOps (error, all_doc_ids) ->
-			return callback(error) if error?
-			RedisManager.getProjectIdsWithHistoryOps (error, all_project_ids) ->
-				return callback(error) if error?
-				# function to get doc_ids for each project
-				task = (cb) -> async.concatSeries all_project_ids, RedisManager.getDocIdsWithHistoryOps, cb
-				# find the dangling doc ids
-				task (error, project_doc_ids) ->
-					dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids)
-					logger.log {all_doc_ids: all_doc_ids, all_project_ids: all_project_ids, project_doc_ids: project_doc_ids, dangling_doc_ids: dangling_doc_ids}, "checking for dangling doc ids"
-					callback(null, dangling_doc_ids)
-
-	getDocUpdates: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
-		UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
-			return callback(error) if error?
-			#console.log "options", options
-			PackManager.getOpsByVersionRange project_id, doc_id, options.from, options.to, (error, updates) ->
-				return callback(error) if error?
-				callback null, updates
-
-	getDocUpdatesWithUserInfo: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
-		UpdatesManager.getDocUpdates project_id, doc_id, options, (error, updates) ->
-			return callback(error) if error?
-			UpdatesManager.fillUserInfo updates, (error, updates) ->
-				return callback(error) if error?
-				callback null, updates
-
-	getSummarizedProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
-		options.min_count ||= 25
-		summarizedUpdates = []
-		before = options.before
-		nextBeforeTimestamp = null
-		UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
-			return callback(error) if error?
-			PackManager.makeProjectIterator project_id, before, (err, iterator) ->
-				return callback(err) if err?
-				# repeatedly get updates and pass them through the summariser to get an final output with user info
-				async.whilst () ->
-					#console.log "checking iterator.done", iterator.done()
-					return summarizedUpdates.length < options.min_count and not iterator.done()
-				, (cb) ->
-					iterator.next (err, partialUpdates) ->
-						return callback(err) if err?
-						#logger.log {partialUpdates}, 'got partialUpdates'
-						return cb() if partialUpdates.length is 0 ## FIXME should try to avoid this happening
-						nextBeforeTimestamp = partialUpdates[partialUpdates.length - 1].meta.end_ts
-						# add the updates to the summary list
-						summarizedUpdates = UpdatesManager._summarizeUpdates partialUpdates, summarizedUpdates
-						cb()
-				, () ->
-					# finally done all updates
-					#console.log 'summarized Updates', summarizedUpdates
-					UpdatesManager.fillSummarizedUserInfo summarizedUpdates, (err, results) ->
-						return callback(err) if err?
-						callback null, results,	if not iterator.done() then nextBeforeTimestamp else undefined
-
-	fetchUserInfo: (users, callback = (error, fetchedUserInfo) ->) ->
-		jobs = []
-		fetchedUserInfo = {}
-		for user_id of users
-			do (user_id) ->
-				jobs.push (callback) ->
-					WebApiManager.getUserInfo user_id, (error, userInfo) ->
-						return callback(error) if error?
-						fetchedUserInfo[user_id] = userInfo
-						callback()
-
-		async.series jobs, (err) ->
-			return callback(err) if err?
-			callback(null, fetchedUserInfo)
-
-	fillUserInfo: (updates, callback = (error, updates) ->) ->
-		users = {}
-		for update in updates
-			user_id = update.meta.user_id
-			if UpdatesManager._validUserId(user_id)
-				users[user_id] = true
-
-		UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
-			return callback(error) if error?
-			for update in updates
-				user_id = update.meta.user_id
-				delete update.meta.user_id
-				if UpdatesManager._validUserId(user_id)
-					update.meta.user = fetchedUserInfo[user_id]
-			callback null, updates
-
-	fillSummarizedUserInfo: (updates, callback = (error, updates) ->) ->
-		users = {}
-		for update in updates
-			user_ids = update.meta.user_ids or []
-			for user_id in user_ids
-				if UpdatesManager._validUserId(user_id)
-					users[user_id] = true
-
-		UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
-			return callback(error) if error?
-			for update in updates
-				user_ids = update.meta.user_ids or []
-				update.meta.users = []
-				delete update.meta.user_ids
-				for user_id in user_ids
-					if UpdatesManager._validUserId(user_id)
-						update.meta.users.push fetchedUserInfo[user_id]
-					else
-						update.meta.users.push null
-			callback null, updates
-
-	_validUserId: (user_id) ->
-		if !user_id?
-			return false
-		else
-			return !!user_id.match(/^[a-f0-9]{24}$/)
-
-	TIME_BETWEEN_DISTINCT_UPDATES: fiveMinutes = 5 * 60 * 1000
-	SPLIT_ON_DELETE_SIZE: 16 # characters
-	_summarizeUpdates: (updates, existingSummarizedUpdates = []) ->
-		summarizedUpdates = existingSummarizedUpdates.slice()
-		previousUpdateWasBigDelete = false
-		for update in updates
-			earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
-			shouldConcat = false
-
-			# If a user inserts some text, then deletes a big chunk including that text,
-			# the update we show might concat the insert and delete, and there will be no sign
-			# of that insert having happened, or be able to restore to it (restoring after a big delete is common).
-			# So, we split the summary on 'big' deletes. However, we've stepping backwards in time with
-			# most recent changes considered first, so if this update is a big delete, we want to start
-			# a new summarized update next timge, hence we monitor the previous update.
-			if previousUpdateWasBigDelete
-				shouldConcat = false
-			else if earliestUpdate and earliestUpdate.meta.end_ts - update.meta.start_ts < @TIME_BETWEEN_DISTINCT_UPDATES
-				# We're going backwards in time through the updates, so only combine if this update starts less than 5 minutes before
-				# the end of current summarized block, so no block spans more than 5 minutes.
-				shouldConcat = true
-
-			isBigDelete = false
-			for op in update.op or []
-				if op.d? and op.d.length > @SPLIT_ON_DELETE_SIZE
-					isBigDelete = true
-
-			previousUpdateWasBigDelete = isBigDelete
-
-			if shouldConcat
-				# check if the user in this update is already present in the earliest update,
-				# if not, add them to the users list of the earliest update
-				earliestUpdate.meta.user_ids = _.union earliestUpdate.meta.user_ids, [update.meta.user_id]
-
-				doc_id = update.doc_id.toString()
-				doc = earliestUpdate.docs[doc_id]
-				if doc?
-					doc.fromV = Math.min(doc.fromV, update.v)
-					doc.toV = Math.max(doc.toV, update.v)
-				else
-					earliestUpdate.docs[doc_id] =
-						fromV: update.v
+			);
+		});
+	},
+
+	// Process all updates for a project, only check project-level information once
+	processUncompressedUpdatesForProject(project_id, callback) {
+		if (callback == null) { callback = function(error) {}; }
+		return RedisManager.getDocIdsWithHistoryOps(project_id, function(error, doc_ids) {
+			if (error != null) { return callback(error); }
+			return UpdatesManager._prepareProjectForUpdates(project_id, function(error, temporary) {
+				const jobs = [];
+				for (let doc_id of Array.from(doc_ids)) {
+					(doc_id =>
+						jobs.push(cb => UpdatesManager._processUncompressedUpdatesForDocWithLock(project_id, doc_id, temporary, cb))
+					)(doc_id);
+				}
+				return async.parallelLimit(jobs, 5, callback);
+			});
+		});
+	},
+
+	// flush all outstanding changes
+	flushAll(limit, callback) {
+		if (callback == null) { callback = function(error, result) {}; }
+		return RedisManager.getProjectIdsWithHistoryOps(function(error, project_ids) {
+			let project_id;
+			if (error != null) { return callback(error); }
+			logger.log({count: (project_ids != null ? project_ids.length : undefined), project_ids}, "found projects");
+			const jobs = [];
+			project_ids = _.shuffle(project_ids); // randomise to avoid hitting same projects each time
+			const selectedProjects = limit < 0 ? project_ids : project_ids.slice(0, limit);
+			for (project_id of Array.from(selectedProjects)) {
+				(project_id =>
+					jobs.push(cb =>
+						UpdatesManager.processUncompressedUpdatesForProject(project_id, err => cb(null, {failed: (err != null), project_id}))
+					)
+				)(project_id);
+			}
+			return async.series(jobs, function(error, result) {
+				let x;
+				if (error != null) { return callback(error); }
+				const failedProjects = ((() => {
+					const result1 = [];
+					for (x of Array.from(result)) { 						if (x.failed) {
+							result1.push(x.project_id);
+						}
+					}
+					return result1;
+				})());
+				const succeededProjects = ((() => {
+					const result2 = [];
+					for (x of Array.from(result)) { 						if (!x.failed) {
+							result2.push(x.project_id);
+						}
+					}
+					return result2;
+				})());
+				return callback(null, {failed: failedProjects, succeeded: succeededProjects, all: project_ids});
+			});
+		});
+	},
+
+	getDanglingUpdates(callback) {
+		if (callback == null) { callback = function(error, doc_ids) {}; }
+		return RedisManager.getAllDocIdsWithHistoryOps(function(error, all_doc_ids) {
+			if (error != null) { return callback(error); }
+			return RedisManager.getProjectIdsWithHistoryOps(function(error, all_project_ids) {
+				if (error != null) { return callback(error); }
+				// function to get doc_ids for each project
+				const task = cb => async.concatSeries(all_project_ids, RedisManager.getDocIdsWithHistoryOps, cb);
+				// find the dangling doc ids
+				return task(function(error, project_doc_ids) {
+					const dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids);
+					logger.log({all_doc_ids, all_project_ids, project_doc_ids, dangling_doc_ids}, "checking for dangling doc ids");
+					return callback(null, dangling_doc_ids);
+				});
+			});
+		});
+	},
+
+	getDocUpdates(project_id, doc_id, options, callback) {
+		if (options == null) { options = {}; }
+		if (callback == null) { callback = function(error, updates) {}; }
+		return UpdatesManager.processUncompressedUpdatesWithLock(project_id, doc_id, function(error) {
+			if (error != null) { return callback(error); }
+			//console.log "options", options
+			return PackManager.getOpsByVersionRange(project_id, doc_id, options.from, options.to, function(error, updates) {
+				if (error != null) { return callback(error); }
+				return callback(null, updates);
+			});
+		});
+	},
+
+	getDocUpdatesWithUserInfo(project_id, doc_id, options, callback) {
+		if (options == null) { options = {}; }
+		if (callback == null) { callback = function(error, updates) {}; }
+		return UpdatesManager.getDocUpdates(project_id, doc_id, options, function(error, updates) {
+			if (error != null) { return callback(error); }
+			return UpdatesManager.fillUserInfo(updates, function(error, updates) {
+				if (error != null) { return callback(error); }
+				return callback(null, updates);
+			});
+		});
+	},
+
+	getSummarizedProjectUpdates(project_id, options, callback) {
+		if (options == null) { options = {}; }
+		if (callback == null) { callback = function(error, updates) {}; }
+		if (!options.min_count) { options.min_count = 25; }
+		let summarizedUpdates = [];
+		const { before } = options;
+		let nextBeforeTimestamp = null;
+		return UpdatesManager.processUncompressedUpdatesForProject(project_id, function(error) {
+			if (error != null) { return callback(error); }
+			return PackManager.makeProjectIterator(project_id, before, function(err, iterator) {
+				if (err != null) { return callback(err); }
+				// repeatedly get updates and pass them through the summariser to get an final output with user info
+				return async.whilst(() =>
+					//console.log "checking iterator.done", iterator.done()
+					(summarizedUpdates.length < options.min_count) && !iterator.done()
+				
+				, cb =>
+					iterator.next(function(err, partialUpdates) {
+						if (err != null) { return callback(err); }
+						//logger.log {partialUpdates}, 'got partialUpdates'
+						if (partialUpdates.length === 0) { return cb(); } //# FIXME should try to avoid this happening
+						nextBeforeTimestamp = partialUpdates[partialUpdates.length - 1].meta.end_ts;
+						// add the updates to the summary list
+						summarizedUpdates = UpdatesManager._summarizeUpdates(partialUpdates, summarizedUpdates);
+						return cb();
+					})
+				
+				, () =>
+					// finally done all updates
+					//console.log 'summarized Updates', summarizedUpdates
+					UpdatesManager.fillSummarizedUserInfo(summarizedUpdates, function(err, results) {
+						if (err != null) { return callback(err); }
+						return callback(null, results,	!iterator.done() ? nextBeforeTimestamp : undefined);
+					})
+				);
+			});
+		});
+	},
+
+	fetchUserInfo(users, callback) {
+		if (callback == null) { callback = function(error, fetchedUserInfo) {}; }
+		const jobs = [];
+		const fetchedUserInfo = {};
+		for (let user_id in users) {
+			(user_id =>
+				jobs.push(callback =>
+					WebApiManager.getUserInfo(user_id, function(error, userInfo) {
+						if (error != null) { return callback(error); }
+						fetchedUserInfo[user_id] = userInfo;
+						return callback();
+					})
+				)
+			)(user_id);
+		}
+
+		return async.series(jobs, function(err) {
+			if (err != null) { return callback(err); }
+			return callback(null, fetchedUserInfo);
+		});
+	},
+
+	fillUserInfo(updates, callback) {
+		let update, user_id;
+		if (callback == null) { callback = function(error, updates) {}; }
+		const users = {};
+		for (update of Array.from(updates)) {
+			({ user_id } = update.meta);
+			if (UpdatesManager._validUserId(user_id)) {
+				users[user_id] = true;
+			}
+		}
+
+		return UpdatesManager.fetchUserInfo(users, function(error, fetchedUserInfo) {
+			if (error != null) { return callback(error); }
+			for (update of Array.from(updates)) {
+				({ user_id } = update.meta);
+				delete update.meta.user_id;
+				if (UpdatesManager._validUserId(user_id)) {
+					update.meta.user = fetchedUserInfo[user_id];
+				}
+			}
+			return callback(null, updates);
+		});
+	},
+
+	fillSummarizedUserInfo(updates, callback) {
+		let update, user_id, user_ids;
+		if (callback == null) { callback = function(error, updates) {}; }
+		const users = {};
+		for (update of Array.from(updates)) {
+			user_ids = update.meta.user_ids || [];
+			for (user_id of Array.from(user_ids)) {
+				if (UpdatesManager._validUserId(user_id)) {
+					users[user_id] = true;
+				}
+			}
+		}
+
+		return UpdatesManager.fetchUserInfo(users, function(error, fetchedUserInfo) {
+			if (error != null) { return callback(error); }
+			for (update of Array.from(updates)) {
+				user_ids = update.meta.user_ids || [];
+				update.meta.users = [];
+				delete update.meta.user_ids;
+				for (user_id of Array.from(user_ids)) {
+					if (UpdatesManager._validUserId(user_id)) {
+						update.meta.users.push(fetchedUserInfo[user_id]);
+					} else {
+						update.meta.users.push(null);
+					}
+				}
+			}
+			return callback(null, updates);
+		});
+	},
+
+	_validUserId(user_id) {
+		if ((user_id == null)) {
+			return false;
+		} else {
+			return !!user_id.match(/^[a-f0-9]{24}$/);
+		}
+	},
+
+	TIME_BETWEEN_DISTINCT_UPDATES: (fiveMinutes = 5 * 60 * 1000),
+	SPLIT_ON_DELETE_SIZE: 16, // characters
+	_summarizeUpdates(updates, existingSummarizedUpdates) {
+		if (existingSummarizedUpdates == null) { existingSummarizedUpdates = []; }
+		const summarizedUpdates = existingSummarizedUpdates.slice();
+		let previousUpdateWasBigDelete = false;
+		for (let update of Array.from(updates)) {
+			var doc_id;
+			const earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1];
+			let shouldConcat = false;
+
+			// If a user inserts some text, then deletes a big chunk including that text,
+			// the update we show might concat the insert and delete, and there will be no sign
+			// of that insert having happened, or be able to restore to it (restoring after a big delete is common).
+			// So, we split the summary on 'big' deletes. However, we've stepping backwards in time with
+			// most recent changes considered first, so if this update is a big delete, we want to start
+			// a new summarized update next timge, hence we monitor the previous update.
+			if (previousUpdateWasBigDelete) {
+				shouldConcat = false;
+			} else if (earliestUpdate && ((earliestUpdate.meta.end_ts - update.meta.start_ts) < this.TIME_BETWEEN_DISTINCT_UPDATES)) {
+				// We're going backwards in time through the updates, so only combine if this update starts less than 5 minutes before
+				// the end of current summarized block, so no block spans more than 5 minutes.
+				shouldConcat = true;
+			}
+
+			let isBigDelete = false;
+			for (let op of Array.from(update.op || [])) {
+				if ((op.d != null) && (op.d.length > this.SPLIT_ON_DELETE_SIZE)) {
+					isBigDelete = true;
+				}
+			}
+
+			previousUpdateWasBigDelete = isBigDelete;
+
+			if (shouldConcat) {
+				// check if the user in this update is already present in the earliest update,
+				// if not, add them to the users list of the earliest update
+				earliestUpdate.meta.user_ids = _.union(earliestUpdate.meta.user_ids, [update.meta.user_id]);
+
+				doc_id = update.doc_id.toString();
+				const doc = earliestUpdate.docs[doc_id];
+				if (doc != null) {
+					doc.fromV = Math.min(doc.fromV, update.v);
+					doc.toV = Math.max(doc.toV, update.v);
+				} else {
+					earliestUpdate.docs[doc_id] = {
+						fromV: update.v,
 						toV: update.v
-
-				earliestUpdate.meta.start_ts = Math.min(earliestUpdate.meta.start_ts, update.meta.start_ts)
-				earliestUpdate.meta.end_ts   = Math.max(earliestUpdate.meta.end_ts, update.meta.end_ts)
-			else
-				newUpdate =
-					meta:
-						user_ids: []
-						start_ts: update.meta.start_ts
+					};
+				}
+
+				earliestUpdate.meta.start_ts = Math.min(earliestUpdate.meta.start_ts, update.meta.start_ts);
+				earliestUpdate.meta.end_ts   = Math.max(earliestUpdate.meta.end_ts, update.meta.end_ts);
+			} else {
+				const newUpdate = {
+					meta: {
+						user_ids: [],
+						start_ts: update.meta.start_ts,
 						end_ts: update.meta.end_ts
+					},
 					docs: {}
+				};
 
-				newUpdate.docs[update.doc_id.toString()] =
-					fromV: update.v
+				newUpdate.docs[update.doc_id.toString()] = {
+					fromV: update.v,
 					toV: update.v
-				newUpdate.meta.user_ids.push update.meta.user_id
-				summarizedUpdates.push newUpdate
-
-		return summarizedUpdates
+				};
+				newUpdate.meta.user_ids.push(update.meta.user_id);
+				summarizedUpdates.push(newUpdate);
+			}
+		}
+
+		return summarizedUpdates;
+	}
+});
+
+function __guard__(value, transform) {
+  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
+}

services/track-changes/app/coffee/WebApiManager.js

@@ -1,69 +1,99 @@
-request = require "requestretry"  # allow retry on error https://github.com/FGRibreau/node-request-retry
-logger = require "logger-sharelatex"
-Settings = require "settings-sharelatex"
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let WebApiManager;
+const request = require("requestretry");  // allow retry on error https://github.com/FGRibreau/node-request-retry
+const logger = require("logger-sharelatex");
+const Settings = require("settings-sharelatex");
 
-# Don't let HTTP calls hang for a long time
-MAX_HTTP_REQUEST_LENGTH = 15000 # 15 seconds
+// Don't let HTTP calls hang for a long time
+const MAX_HTTP_REQUEST_LENGTH = 15000; // 15 seconds
 
-# DEPRECATED! This method of getting user details via track-changes is deprecated
-# in the way we lay out our services.
-# Instead, web should be responsible for collecting the raw data (user_ids) and
-# filling it out with calls to other services. All API calls should create a
-# tree-like structure as much as possible, with web as the root.
-module.exports = WebApiManager =
-	sendRequest: (url, callback = (error, body) ->) ->
-		request.get {
-			url: "#{Settings.apis.web.url}#{url}"
-			timeout: MAX_HTTP_REQUEST_LENGTH
-			maxAttempts: 2 # for node-request-retry
-			auth:
-				user: Settings.apis.web.user
-				pass: Settings.apis.web.pass
+// DEPRECATED! This method of getting user details via track-changes is deprecated
+// in the way we lay out our services.
+// Instead, web should be responsible for collecting the raw data (user_ids) and
+// filling it out with calls to other services. All API calls should create a
+// tree-like structure as much as possible, with web as the root.
+module.exports = (WebApiManager = {
+	sendRequest(url, callback) {
+		if (callback == null) { callback = function(error, body) {}; }
+		return request.get({
+			url: `${Settings.apis.web.url}${url}`,
+			timeout: MAX_HTTP_REQUEST_LENGTH,
+			maxAttempts: 2, // for node-request-retry
+			auth: {
+				user: Settings.apis.web.user,
+				pass: Settings.apis.web.pass,
 				sendImmediately: true
-		}, (error, res, body)->
-			if error?
-				return callback(error)
-			if res.statusCode == 404
-				logger.log url: url, "got 404 from web api"
-				return callback null, null
-			if res.statusCode >= 200 and res.statusCode < 300
-				return callback null, body
-			else
-				error = new Error("web returned a non-success status code: #{res.statusCode} (attempts: #{res.attempts})")
-				callback error
+			}
+		}, function(error, res, body){
+			if (error != null) {
+				return callback(error);
+			}
+			if (res.statusCode === 404) {
+				logger.log({url}, "got 404 from web api");
+				return callback(null, null);
+			}
+			if ((res.statusCode >= 200) && (res.statusCode < 300)) {
+				return callback(null, body);
+			} else {
+				error = new Error(`web returned a non-success status code: ${res.statusCode} (attempts: ${res.attempts})`);
+				return callback(error);
+			}
+		});
+	},
 
-	getUserInfo: (user_id, callback = (error, userInfo) ->) ->
-		url = "/user/#{user_id}/personal_info"
-		logger.log user_id: user_id, "getting user info from web"
-		WebApiManager.sendRequest url, (error, body) ->
-			if error?
-				logger.error err: error, user_id: user_id, url: url, "error accessing web"
-				return callback error
+	getUserInfo(user_id, callback) {
+		if (callback == null) { callback = function(error, userInfo) {}; }
+		const url = `/user/${user_id}/personal_info`;
+		logger.log({user_id}, "getting user info from web");
+		return WebApiManager.sendRequest(url, function(error, body) {
+			let user;
+			if (error != null) {
+				logger.error({err: error, user_id, url}, "error accessing web");
+				return callback(error);
+			}
 
-			if body == null
-				logger.error user_id: user_id, url: url, "no user found"
-				return callback null, null
-			try
-				user = JSON.parse(body)
-			catch error
-				return callback(error)
-			callback null, {
-				id: user.id
-				email: user.email
-				first_name: user.first_name
-				last_name: user.last_name
+			if (body === null) {
+				logger.error({user_id, url}, "no user found");
+				return callback(null, null);
+			}
+			try {
+				user = JSON.parse(body);
+			} catch (error1) {
+				error = error1;
+				return callback(error);
 			}
+			return callback(null, {
+				id: user.id,
+				email: user.email,
+				first_name: user.first_name,
+				last_name: user.last_name
+			});
+	});
+	},
 
-	getProjectDetails: (project_id, callback = (error, details) ->) ->
-		url = "/project/#{project_id}/details"
-		logger.log project_id: project_id, "getting project details from web"
-		WebApiManager.sendRequest url, (error, body) ->
-			if error?
-				logger.error err: error, project_id: project_id, url: url, "error accessing web"
-				return callback error
+	getProjectDetails(project_id, callback) {
+		if (callback == null) { callback = function(error, details) {}; }
+		const url = `/project/${project_id}/details`;
+		logger.log({project_id}, "getting project details from web");
+		return WebApiManager.sendRequest(url, function(error, body) {
+			let project;
+			if (error != null) {
+				logger.error({err: error, project_id, url}, "error accessing web");
+				return callback(error);
+			}
 
-			try
-				project = JSON.parse(body)
-			catch error
-				return callback(error)
-			callback null, project
+			try {
+				project = JSON.parse(body);
+			} catch (error1) {
+				error = error1;
+				return callback(error);
+			}
+			return callback(null, project);
+		});
+	}
+});

services/track-changes/app/coffee/mongojs.js

@@ -1,9 +1,10 @@
-Settings = require "settings-sharelatex"
-mongojs = require "mongojs"
-bson = require "bson"
-db = mongojs(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryIndex"])
-module.exports =
-	db: db
-	ObjectId: mongojs.ObjectId
+const Settings = require("settings-sharelatex");
+const mongojs = require("mongojs");
+const bson = require("bson");
+const db = mongojs(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryIndex"]);
+module.exports = {
+	db,
+	ObjectId: mongojs.ObjectId,
 	BSON: new bson.BSONPure()
+};