Initial open sourcing

services/document-updater/.gitignore

@@ -0,0 +1,46 @@
+compileFolder
+
+Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store?
+ehthumbs.db
+Icon?
+Thumbs.db
+
+/node_modules/*
+
+app.js
+app/js/*
+
+test/unit/js/*
+test/acceptance/js/*
+
+**.swp

services/document-updater/Gruntfile.coffee

@@ -0,0 +1,111 @@
+module.exports = (grunt) ->
+	grunt.loadNpmTasks 'grunt-contrib-coffee'
+	grunt.loadNpmTasks 'grunt-contrib-clean'
+	grunt.loadNpmTasks 'grunt-mocha-test'
+	grunt.loadNpmTasks 'grunt-available-tasks'
+	grunt.loadNpmTasks 'grunt-execute'
+	grunt.loadNpmTasks 'grunt-bunyan'
+	
+	grunt.initConfig
+		execute:
+			app:
+				src: "app.js"
+
+		bunyan:
+			strict: false
+
+		coffee:
+			app_dir: 
+				expand: true,
+				flatten: false,
+				cwd: 'app/coffee',
+				src: ['**/*.coffee'],
+				dest: 'app/js/',
+				ext: '.js'
+
+			app: 
+				src: 'app.coffee'
+				dest: 'app.js'
+
+			acceptance_tests:
+				expand: true,
+				flatten: false,
+				cwd: 'test/acceptance/coffee',
+				src: ['**/*.coffee'],
+				dest: 'test/acceptance/js/',
+				ext: '.js'
+
+			unit_tests: 
+				expand: true,
+				flatten: false,
+				cwd: 'test/unit/coffee',
+				src: ['**/*.coffee'],
+				dest: 'test/unit/js/',
+				ext: '.js'
+
+		clean:
+			app: ["app/js"]
+			acceptance_tests: ["test/unit/js"]
+
+		mochaTest:
+			unit:
+				src: ['test/unit/js/**/*.js']
+				options:
+					reporter: grunt.option('reporter') or 'spec'
+					grep: grunt.option("grep")
+			acceptance:
+				src: ['test/acceptance/js/**/*.js']
+				options:
+					reporter: grunt.option('reporter') or 'spec'
+					grep: grunt.option("grep")
+					timeout: 10000
+
+		availabletasks:
+			tasks:
+				options:
+		            filter: 'exclude',
+		            tasks: [
+		            	'coffee'
+		            	'clean'
+		            	'mochaTest'
+		            	'availabletasks'
+		            	'execute'
+		            	'bunyan'
+		           	]
+		            groups:
+		            	"Compile tasks": [
+		            		"compile:server"
+		            		"compile:tests"
+		            		"compile"
+		            		"compile:unit_tests"
+		            		"compile:acceptance_tests"
+		            		"install"
+		            	]
+		            	"Test tasks": [
+		            		"test:unit"
+		            		"test:acceptance"
+		            	]
+		            	"Run tasks": [
+		            		"run"
+		            		"default"
+		            	]
+		            	"Misc": [
+		            		"help"
+		            	]
+
+	grunt.registerTask 'help', 'Display this help list', 'availabletasks'
+
+	grunt.registerTask 'compile:server', 'Compile the server side coffee script', ['clean:app', 'coffee:app', 'coffee:app_dir']
+	grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['coffee:unit_tests']
+	grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
+	grunt.registerTask 'compile:tests', 'Compile all the tests', ['compile:acceptance_tests', 'compile:unit_tests']
+	grunt.registerTask 'compile', 'Compiles everything need to run document-updater-sharelatex', ['compile:server']
+
+	grunt.registerTask 'install', "Compile everything when installing as an npm module", ['compile']
+
+	grunt.registerTask 'test:unit', 'Run the unit tests (use --grep=<regex> for individual tests)', ['compile:unit_tests', 'mochaTest:unit']
+	grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep=<regex> for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance']
+
+	grunt.registerTask 'run', "Compile and run the document-updater-sharelatex server", ['compile', 'bunyan', 'execute']
+	grunt.registerTask 'default', 'run'
+

services/document-updater/app.coffee

@@ -0,0 +1,68 @@
+express = require('express')
+http = require("http")
+Settings = require('settings-sharelatex')
+logger = require('logger-sharelatex')
+logger.initialize("documentupdater")
+RedisManager = require('./app/js/RedisManager.js')
+UpdateManager = require('./app/js/UpdateManager.js')
+Keys = require('./app/js/RedisKeyBuilder')
+redis = require('redis')
+rclient = redis.createClient(Settings.redis.port, Settings.redis.host)
+rclient.auth(Settings.redis.password)
+metrics = require('./app/js/Metrics')
+Errors = require "./app/js/Errors"
+HttpController = require "./app/js/HttpController"
+
+app = express()
+app.configure ->
+	app.use(express.logger(':remote-addr - [:date] - :user-agent ":method :url" :status - :response-time ms'));
+	app.use express.bodyParser()
+	app.use app.router
+
+app.configure 'development', ()->
+	console.log "Development Enviroment"
+	app.use express.errorHandler({ dumpExceptions: true, showStack: true })
+
+app.configure 'production', ()->
+	console.log "Production Enviroment"
+	app.use express.logger()
+	app.use express.errorHandler()
+
+rclient.subscribe("pending-updates")
+rclient.on "message", (channel, doc_key)->
+	[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)
+	UpdateManager.processOutstandingUpdatesWithLock project_id, doc_id, (error) ->
+		logger.error err: error, project_id: project_id, doc_id: doc_id, "error processing update" if error?
+
+UpdateManager.resumeProcessing()
+
+app.use (req, res, next)->
+	metrics.inc "http-request"
+	next()
+
+app.get    '/project/:project_id/doc/:doc_id',       HttpController.getDoc
+app.post   '/project/:project_id/doc/:doc_id',       HttpController.setDoc
+app.post   '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLoaded
+app.delete '/project/:project_id/doc/:doc_id',       HttpController.flushAndDeleteDoc
+app.delete '/project/:project_id',                   HttpController.deleteProject
+app.post   '/project/:project_id/flush',             HttpController.flushProject
+
+app.get '/total', (req, res)->
+	timer = new metrics.Timer("http.allDocList")	
+	RedisManager.getCountOfDocsInMemory (err, count)->
+		timer.done()
+		res.send {total:count}
+	
+app.get '/status', (req, res)->
+	res.send('document updater is alive')
+
+app.use (error, req, res, next) ->
+	logger.error err: error, "request errored"
+	if error instanceof Errors.NotFoundError
+		res.send 404
+	else
+		res.send(500, "Oops, something went wrong")
+
+port = Settings.internal?.documentupdater?.port or Settings.apis?.documentupdater?.port or 3003
+app.listen port, "localhost", ->
+	logger.log("documentupdater-sharelatex server listening on port #{port}")

services/document-updater/app/DocumentUpdater.js

@@ -0,0 +1,181 @@
+(function(exports){
+  Ace = require('aceserverside-sharelatex')
+  Range = Ace.Range
+
+  //look at applyDeltas method
+  exports.applyChange = function(aceDoc, change, callback) {
+    var r = change.range;
+    var range = new Range(r.start.row, r.start.column, r.end.row, r.end.column);
+    if('insertText'==change.action){
+      aceDoc.insert(change.range.start, change.text);
+    }else if('insertLines'==change.action){
+      aceDoc.insertLines(change.range.start.row, change.lines);
+    }else if('removeText'==change.action){
+      aceDoc.remove(range);
+    }else if('removeLines'==change.action){
+      aceDoc.removeLines(range.start.row, range.end.row-1);
+    }
+
+    if(typeof callback === 'function'){
+      callback(null, aceDoc);
+    };
+  }
+
+})(typeof exports === 'undefined'? this['documentUpdater']={}: exports);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

services/document-updater/app/coffee/DiffCodec.coffee

@@ -0,0 +1,31 @@
+diff_match_patch = require("../lib/diff_match_patch").diff_match_patch
+dmp = new diff_match_patch()
+
+module.exports = DiffCodec =
+	ADDED: 1
+	REMOVED: -1
+	UNCHANGED: 0
+
+	diffAsShareJsOp: (before, after, callback = (error, ops) ->) ->
+		diffs = dmp.diff_main(before.join("\n"), after.join("\n"))
+		dmp.diff_cleanupSemantic(diffs)
+
+		ops = []
+		position = 0
+		for diff in diffs
+			type = diff[0]
+			content = diff[1]
+			if type == @ADDED
+				ops.push
+					i: content
+					p: position
+				position += content.length
+			else if type == @REMOVED
+				ops.push
+					d: content
+					p: position
+			else if type == @UNCHANGED
+				position += content.length
+			else
+				throw "Unknown type"
+		callback null, ops

services/document-updater/app/coffee/DocOpsManager.coffee

@@ -0,0 +1,127 @@
+RedisManager = require "./RedisManager"
+mongojs = require("./mongojs")
+db = mongojs.db
+ObjectId = mongojs.ObjectId
+logger = require "logger-sharelatex"
+async = require "async"
+Metrics = require("./Metrics")
+
+module.exports = DocOpsManager =
+	flushDocOpsToMongo: (project_id, doc_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("docOpsManager.flushDocOpsToMongo")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		DocOpsManager.getDocVersionInMongo doc_id, (error, mongoVersion) ->
+			return callback(error) if error?
+			RedisManager.getDocVersion doc_id, (error, redisVersion) ->
+				return callback(error) if error?
+				if !mongoVersion? or !redisVersion? or mongoVersion > redisVersion
+					logger.error doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, "mongo version is ahead of redis"
+					return callback(new Error("inconsistent versions"))
+
+				RedisManager.getPreviousDocOps doc_id, mongoVersion, -1, (error, ops) ->
+					return callback(error) if error?
+					if ops.length != redisVersion - mongoVersion
+						logger.error doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, opsLength: ops.length, "version difference does not match ops length"
+						return callback(new Error("inconsistent versions"))
+					logger.log doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, "flushing doc ops to mongo"
+					DocOpsManager._appendDocOpsInMongo doc_id, ops, redisVersion, (error) ->
+						return callback(error) if error?
+						callback null
+
+	getPreviousDocOps: (project_id, doc_id, start, end, _callback = (error, ops) ->) ->
+		timer = new Metrics.Timer("docOpsManager.getPreviousDocOps")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		DocOpsManager._ensureOpsAreLoaded project_id, doc_id, start, (error) ->
+			return callback(error) if error?
+			RedisManager.getPreviousDocOps doc_id, start, end, (error, ops) ->
+				return callback(error) if error?
+				callback null, ops
+
+	pushDocOp: (project_id, doc_id, op, callback = (error) ->) ->
+		RedisManager.pushDocOp doc_id, op, callback
+
+	_ensureOpsAreLoaded: (project_id, doc_id, backToVersion, callback = (error) ->) ->
+		RedisManager.getDocVersion doc_id, (error, redisVersion) ->
+			return callback(error) if error?
+			RedisManager.getDocOpsLength doc_id, (error, opsLength) ->
+				return callback(error) if error?
+				oldestVersionInRedis = redisVersion - opsLength
+				if oldestVersionInRedis > backToVersion
+					# _getDocOpsFromMongo(<id>, 4, 6, ...) will return the ops in positions 4 and 5, but not 6.
+					logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, "loading old ops from mongo"
+					DocOpsManager._getDocOpsFromMongo doc_id, backToVersion, oldestVersionInRedis, (error, ops) ->
+						logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, ops: ops, "loaded old ops from mongo"
+						return callback(error) if error?
+						RedisManager.prependDocOps doc_id, ops, (error) ->
+							return callback(error) if error?
+							callback null
+				else
+					logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, "ops already in redis"
+					callback()
+
+	getDocVersionInMongo: (doc_id, callback = (error, version) ->) ->
+		t = new Metrics.Timer("mongo-time")
+		db.docOps.find {
+			doc_id: ObjectId(doc_id)
+		}, {
+			version: 1
+		}, (error, docs) ->
+			t.done()
+			return callback(error) if error?
+			if docs.length < 1 or !docs[0].version?
+				return callback null, 0
+			else
+				return callback null, docs[0].version
+
+	APPEND_OPS_BATCH_SIZE: 100
+
+	_appendDocOpsInMongo: (doc_id, docOps, newVersion, callback = (error) ->) ->
+		currentVersion = newVersion - docOps.length
+		batchSize = DocOpsManager.APPEND_OPS_BATCH_SIZE
+		noOfBatches = Math.ceil(docOps.length / batchSize)
+		if noOfBatches <= 0
+			return callback()
+		jobs = []
+		for batchNo in [0..(noOfBatches-1)]
+			do (batchNo) ->
+				jobs.push (callback) ->
+					batch = docOps.slice(batchNo * batchSize, (batchNo + 1) * batchSize)
+					currentVersion += batch.length
+					logger.log doc_id: doc_id, batchNo: batchNo, "appending doc op batch to Mongo"
+					t = new Metrics.Timer("mongo-time")
+					db.docOps.update {
+						doc_id: ObjectId(doc_id)
+					}, {
+						$push: docOps: { $each: batch, $slice: -100 }
+						$set: version: currentVersion
+					}, {
+						upsert: true
+					}, (err)->
+						t.done()
+						callback(err)
+
+		async.series jobs, (error) -> callback(error)
+
+	_getDocOpsFromMongo: (doc_id, start, end, callback = (error, ops) ->) ->
+		DocOpsManager.getDocVersionInMongo doc_id, (error, version) ->
+			return callback(error) if error?
+			offset = - (version - start) # Negative tells mongo to count from the end backwards
+			limit = end - start
+			t = new Metrics.Timer("mongo-time")
+			db.docOps.find {
+				doc_id: ObjectId(doc_id)
+			}, {
+				docOps: $slice: [offset, limit]
+			}, (error, docs) ->
+				t.done()
+				if docs.length < 1 or !docs[0].docOps?
+					return callback null, []
+				else
+					return callback null, docs[0].docOps
+		

services/document-updater/app/coffee/DocumentManager.coffee

@@ -0,0 +1,127 @@
+RedisManager = require "./RedisManager"
+PersistenceManager = require "./PersistenceManager"
+DocOpsManager = require "./DocOpsManager"
+DiffCodec = require "./DiffCodec"
+logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
+
+module.exports = DocumentManager =
+	getDoc: (project_id, doc_id, _callback = (error, lines, version) ->) ->
+		timer = new Metrics.Timer("docManager.getDoc")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		RedisManager.getDoc doc_id, (error, lines, version) ->
+			return callback(error) if error?
+			if !lines? or !version?
+				logger.log project_id: project_id, doc_id: doc_id, "doc not in redis so getting from persistence API"
+				PersistenceManager.getDoc project_id, doc_id, (error, lines) ->
+					return callback(error) if error?
+					DocOpsManager.getDocVersionInMongo doc_id, (error, version) ->
+						return callback(error) if error?
+						logger.log project_id: project_id, doc_id: doc_id, lines: lines, version: version, "got doc from persistence API"
+						RedisManager.putDocInMemory project_id, doc_id, lines, version, (error) ->
+							return callback(error) if error?
+							callback null, lines, version
+			else
+				callback null, lines, version
+
+	getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps) ->) ->
+		timer = new Metrics.Timer("docManager.getDocAndRecentOps")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+		
+		DocumentManager.getDoc project_id, doc_id, (error, lines, version) ->
+			return callback(error) if error?
+			if fromVersion == -1
+				callback null, lines, version, []
+			else
+				DocOpsManager.getPreviousDocOps project_id, doc_id, fromVersion, version, (error, ops) ->
+					return callback(error) if error?
+					callback null, lines, version, ops
+
+	setDoc: (project_id, doc_id, newLines, _callback = (error) ->) ->
+		timer = new Metrics.Timer("docManager.setDoc")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		if !newLines?
+			return callback(new Error("No lines were provided to setDoc"))
+
+		UpdateManager = require "./UpdateManager"
+		DocumentManager.getDoc project_id, doc_id, (error, oldLines, version) ->
+			return callback(error) if error?
+			
+			if oldLines? and oldLines.length > 0 and oldLines[0].text?
+				logger.log doc_id: doc_id, project_id: project_id, oldLines: oldLines, newLines: newLines, "document is JSON so not updating"
+				return callback(null)
+
+			logger.log doc_id: doc_id, project_id: project_id, oldLines: oldLines, newLines: newLines, "setting a document via http"
+			DiffCodec.diffAsShareJsOp oldLines, newLines, (error, op) ->
+				return callback(error) if error?
+				update =
+					doc: doc_id
+					op: op
+					v: version
+					meta:
+						type: "external"
+				UpdateManager.applyUpdates project_id, doc_id, [update], (error) ->
+					return callback(error) if error?
+					DocumentManager.flushDocIfLoaded project_id, doc_id, (error) ->
+						return callback(error) if error?
+						callback null
+		
+
+	flushDocIfLoaded: (project_id, doc_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("docManager.flushDocIfLoaded")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		RedisManager.getDoc doc_id, (error, lines, version) ->
+			return callback(error) if error?
+			if !lines? or !version?
+				logger.log project_id: project_id, doc_id: doc_id, "doc is not loaded so not flushing"
+				callback null
+			else
+				logger.log project_id: project_id, doc_id: doc_id, "flushing doc"
+				PersistenceManager.setDoc project_id, doc_id, lines, (error) ->
+					return callback(error) if error?
+					DocOpsManager.flushDocOpsToMongo project_id, doc_id, (error) ->
+						return callback(error) if error?
+						callback null
+
+	flushAndDeleteDoc: (project_id, doc_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("docManager.flushAndDeleteDoc")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		DocumentManager.flushDocIfLoaded project_id, doc_id, (error) ->
+			return callback(error) if error?
+			RedisManager.removeDocFromMemory project_id, doc_id, (error) ->
+				return callback(error) if error?
+				callback null
+
+	getDocWithLock: (project_id, doc_id, callback = (error, lines, version) ->) ->
+		UpdateManager = require "./UpdateManager"
+		UpdateManager.lockUpdatesAndDo DocumentManager.getDoc, project_id, doc_id, callback
+		
+	getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version) ->) ->
+		UpdateManager = require "./UpdateManager"
+		UpdateManager.lockUpdatesAndDo DocumentManager.getDocAndRecentOps, project_id, doc_id, fromVersion, callback
+		
+	setDocWithLock: (project_id, doc_id, lines, callback = (error) ->) ->
+		UpdateManager = require "./UpdateManager"
+		UpdateManager.lockUpdatesAndDo DocumentManager.setDoc, project_id, doc_id, lines, callback
+		
+	flushDocIfLoadedWithLock: (project_id, doc_id, callback = (error) ->) ->
+		UpdateManager = require "./UpdateManager"
+		UpdateManager.lockUpdatesAndDo DocumentManager.flushDocIfLoaded, project_id, doc_id, callback
+
+	flushAndDeleteDocWithLock: (project_id, doc_id, callback = (error) ->) ->
+		UpdateManager = require "./UpdateManager"
+		UpdateManager.lockUpdatesAndDo DocumentManager.flushAndDeleteDoc, project_id, doc_id, callback

services/document-updater/app/coffee/Errors.coffee

@@ -0,0 +1,10 @@
+NotFoundError = (message) ->
+	error = new Error(message)
+	error.name = "NotFoundError"
+	error.__proto__ = NotFoundError.prototype
+	return error
+NotFoundError.prototype.__proto__ = Error.prototype
+
+module.exports = Errors =
+	NotFoundError: NotFoundError
+

services/document-updater/app/coffee/HttpController.coffee

@@ -0,0 +1,85 @@
+DocumentManager = require "./DocumentManager"
+ProjectManager = require "./ProjectManager"
+Errors = require "./Errors"
+logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
+
+module.exports = HttpController =
+	getDoc: (req, res, next = (error) ->) ->
+		doc_id = req.params.doc_id
+		project_id = req.params.project_id
+		logger.log project_id: project_id, doc_id: doc_id, "getting doc via http"
+		timer = new Metrics.Timer("http.getDoc")
+
+		if req.query?.fromVersion?
+			fromVersion = parseInt(req.query.fromVersion, 10)
+		else
+			fromVersion = -1
+
+		DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, doc_id: doc_id, "got doc via http"
+			if !lines? or !version?
+				return next(new Errors.NotFoundError("document not found"))
+			res.send JSON.stringify
+				id: doc_id
+				lines: lines
+				version: version
+				ops: ops
+
+	setDoc: (req, res, next = (error) ->) ->
+		doc_id = req.params.doc_id
+		project_id = req.params.project_id
+		lines = req.body.lines
+		logger.log project_id: project_id, doc_id: doc_id, lines: lines, "setting doc via http"
+		timer = new Metrics.Timer("http.setDoc")
+		DocumentManager.setDocWithLock project_id, doc_id, lines, (error) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, doc_id: doc_id, "set doc via http"
+			res.send 204 # No Content
+		
+
+	flushDocIfLoaded: (req, res, next = (error) ->) ->
+		doc_id = req.params.doc_id
+		project_id = req.params.project_id
+		logger.log project_id: project_id, doc_id: doc_id, "flushing doc via http"
+		timer = new Metrics.Timer("http.flushDoc")
+		DocumentManager.flushDocIfLoadedWithLock project_id, doc_id, (error) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, doc_id: doc_id, "flushed doc via http"
+			res.send 204 # No Content
+		
+	flushAndDeleteDoc: (req, res, next = (error) ->) ->
+		doc_id = req.params.doc_id
+		project_id = req.params.project_id
+		logger.log project_id: project_id, doc_id: doc_id, "deleting doc via http"
+		timer = new Metrics.Timer("http.deleteDoc")
+		DocumentManager.flushAndDeleteDocWithLock project_id, doc_id, (error) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, doc_id: doc_id, "deleted doc via http"
+			res.send 204 # No Content
+
+	flushProject: (req, res, next = (error) ->) ->
+		project_id = req.params.project_id
+		logger.log project_id: project_id, "flushing project via http"
+		timer = new Metrics.Timer("http.flushProject")
+		ProjectManager.flushProjectWithLocks project_id, (error) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, "flushed project via http"
+			res.send 204 # No Content
+		
+	deleteProject: (req, res, next = (error) ->) ->
+		project_id = req.params.project_id
+		logger.log project_id: project_id, "deleting project via http"
+		timer = new Metrics.Timer("http.deleteProject")
+		ProjectManager.flushAndDeleteProjectWithLocks project_id, (error) ->
+			timer.done()
+			return next(error) if error?
+			logger.log project_id: project_id, "deleted project via http"
+			res.send 204 # No Content
+			

services/document-updater/app/coffee/LockManager.coffee

@@ -0,0 +1,55 @@
+metrics = require('./Metrics')
+Settings = require('settings-sharelatex')
+redis = require('redis')
+redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379}
+rclient = redis.createClient(redisConf.port, redisConf.host)
+rclient.auth(redisConf.password)
+keys = require('./RedisKeyBuilder')
+logger = require "logger-sharelatex"
+
+module.exports = LockManager =
+	LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock
+	MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock
+
+	tryLock : (doc_id, callback = (err, isFree)->)->
+		tenSeconds = 10
+		rclient.set keys.blockingKey(doc_id: doc_id), "locked", "EX", 10, "NX", (err, gotLock)->
+			return callback(err) if err?
+			if gotLock == "OK"
+				metrics.inc "doc-not-blocking"
+				callback err, true
+			else
+				metrics.inc "doc-blocking"
+				logger.log doc_id: doc_id, redis_response: gotLock, "doc is locked"
+				callback err, false
+
+	getLock: (doc_id, callback = (error) ->) ->
+		startTime = Date.now()
+		do attempt = () ->
+			if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
+				return callback(new Error("Timeout"))
+
+			LockManager.tryLock doc_id, (error, gotLock) ->
+				return callback(error) if error?
+				if gotLock
+					callback(null)
+				else
+					setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
+
+	checkLock: (doc_id, callback = (err, isFree)->)->
+		multi = rclient.multi()
+		multi.exists keys.blockingKey(doc_id:doc_id)
+		multi.exec (err, replys)->
+			return callback(err) if err?
+			exists = parseInt replys[0]
+			if exists == 1
+				metrics.inc "doc-blocking"
+				callback err, false
+			else
+				metrics.inc "doc-not-blocking"
+				callback err, true
+
+	releaseLock: (doc_id, callback)->
+		rclient.del keys.blockingKey(doc_id:doc_id), callback
+
+	

services/document-updater/app/coffee/Metrics.coffee

@@ -0,0 +1,23 @@
+StatsD = require('lynx')
+statsd = new StatsD('localhost', 8125, {on_error:->})
+
+buildKey = (key)-> "doc-updater.#{process.env.NODE_ENV}.#{key}"
+
+module.exports =
+	set : (key, value, sampleRate = 1)->
+		statsd.set buildKey(key), value, sampleRate
+
+	inc : (key, sampleRate = 1)->
+		statsd.increment buildKey(key), sampleRate
+
+	Timer : class
+		constructor :(key, sampleRate = 1)->
+			this.start = new Date()
+			this.key = buildKey(key)
+		done:->
+			timeSpan = new Date - this.start
+			statsd.timing(this.key, timeSpan, this.sampleRate)
+
+	gauge : (key, value, sampleRate = 1)->
+		statsd.gauge key, value, sampleRate
+

services/document-updater/app/coffee/PersistenceManager.coffee

@@ -0,0 +1,66 @@
+request = require "request"
+Settings = require "settings-sharelatex"
+Errors = require "./Errors"
+Metrics = require "./Metrics"
+
+module.exports = PersistenceManager =
+	getDoc: (project_id, doc_id, _callback = (error, lines) ->) ->
+		timer = new Metrics.Timer("persistenceManager.getDoc")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		url = "#{Settings.apis.web.url}/project/#{project_id}/doc/#{doc_id}"
+		request {
+			url: url
+			method: "GET"
+			headers:
+				"accept": "application/json"
+			auth:
+				user: Settings.apis.web.user
+				pass: Settings.apis.web.pass
+				sendImmediately: true
+			jar: false
+		}, (error, res, body) ->
+			return callback(error) if error?
+			if res.statusCode >= 200 and res.statusCode < 300
+				try
+					body = JSON.parse body
+				catch e
+					return callback(e)
+				return callback null, body.lines
+			else if res.statusCode == 404
+				return callback(new Errors.NotFoundError("doc not not found: #{url}"))
+			else
+				return callback(new Error("error accessing web API: #{url} #{res.statusCode}"))
+
+	setDoc: (project_id, doc_id, lines, _callback = (error) ->) ->
+		timer = new Metrics.Timer("persistenceManager.setDoc")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		url = "#{Settings.apis.web.url}/project/#{project_id}/doc/#{doc_id}"
+		request {
+			url: url
+			method: "POST"
+			body: JSON.stringify
+				lines: lines
+			headers:
+				"content-type": "application/json"
+			auth:
+				user: Settings.apis.web.user
+				pass: Settings.apis.web.pass
+				sendImmediately: true
+			jar: false
+		}, (error, res, body) ->
+			return callback(error) if error?
+			if res.statusCode >= 200 and res.statusCode < 300
+				return callback null
+			else if res.statusCode == 404
+				return callback(new Errors.NotFoundError("doc not not found: #{url}"))
+			else
+				return callback(new Error("error accessing web API: #{url} #{res.statusCode}"))
+		
+
+			

services/document-updater/app/coffee/ProjectManager.coffee

@@ -0,0 +1,60 @@
+RedisManager = require "./RedisManager"
+DocumentManager = require "./DocumentManager"
+async = require "async"
+logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
+
+module.exports = ProjectManager =
+	flushProjectWithLocks: (project_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("projectManager.flushProjectWithLocks")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		RedisManager.getDocIdsInProject project_id, (error, doc_ids) ->
+			return callback(error) if error?
+			jobs = []
+			errors = []
+			for doc_id in (doc_ids or [])
+				do (doc_id) ->
+					jobs.push (callback) ->
+						DocumentManager.flushDocIfLoadedWithLock project_id, doc_id, (error) ->
+							if error?
+								logger.error err: error, project_id: project_id, doc_id: doc_id, "error flushing doc"
+								errors.push(error)
+							callback()
+
+			logger.log project_id: project_id, doc_ids: doc_ids, "flushing docs"
+			async.series jobs, () ->
+				if errors.length > 0
+					callback new Error("Errors flushing docs. See log for details")
+				else
+					callback(null)
+
+	flushAndDeleteProjectWithLocks: (project_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("projectManager.flushAndDeleteProjectWithLocks")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+
+		RedisManager.getDocIdsInProject project_id, (error, doc_ids) ->
+			return callback(error) if error?
+			jobs = []
+			errors = []
+			for doc_id in (doc_ids or [])
+				do (doc_id) ->
+					jobs.push (callback) ->
+						DocumentManager.flushAndDeleteDocWithLock project_id, doc_id, (error) ->
+							if error?
+								logger.error err: error, project_id: project_id, doc_id: doc_id, "error deleting doc"
+								errors.push(error)
+							callback()
+
+			logger.log project_id: project_id, doc_ids: doc_ids, "deleting docs"
+			async.series jobs, () ->
+				if errors.length > 0
+					callback new Error("Errors deleting docs. See log for details")
+				else
+					callback(null)
+
+		

services/document-updater/app/coffee/RedisKeyBuilder.coffee

@@ -0,0 +1,28 @@
+ALLDOCSKEY = "AllDocIds"
+PROJECTKEY = "ProjectId"
+BLOCKINGKEY = "Blocking"
+CHANGEQUE = "ChangeQue"
+DOCSINPROJECT = "DocsIn"
+PENDINGUPDATESKEY = "PendingUpdates"
+DOCLINES = "doclines"
+DOCOPS = "DocOps"
+DOCVERSION = "DocVersion"
+DOCIDSWITHPENDINGUPDATES = "DocsWithPendingUpdates"
+
+module.exports =
+
+	allDocs : ALLDOCSKEY
+	docLines : (op)-> DOCLINES+":"+op.doc_id
+	docOps : (op)-> DOCOPS+":"+op.doc_id
+	docVersion : (op)-> DOCVERSION+":"+op.doc_id
+	projectKey : (op)-> PROJECTKEY+":"+op.doc_id
+	blockingKey : (op)-> BLOCKINGKEY+":"+op.doc_id
+	changeQue : (op)-> CHANGEQUE+":"+op.project_id
+	docsInProject : (op)-> DOCSINPROJECT+":"+op.project_id
+	pendingUpdates : (op)-> PENDINGUPDATESKEY+":"+op.doc_id
+	docsWithPendingUpdates : DOCIDSWITHPENDINGUPDATES
+	combineProjectIdAndDocId: (project_id, doc_id) -> "#{project_id}:#{doc_id}"
+	splitProjectIdAndDocId: (project_and_doc_id) -> project_and_doc_id.split(":")
+	now : (key)->
+		d = new Date()
+		d.getDate()+":"+(d.getMonth()+1)+":"+d.getFullYear()+":"+key

services/document-updater/app/coffee/RedisManager.coffee

@@ -0,0 +1,184 @@
+Settings = require('settings-sharelatex')
+redis = require('redis')
+redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379}
+rclient = redis.createClient(redisConf.port, redisConf.host)
+rclient.auth(redisConf.password)
+async = require('async')
+_ = require('underscore')
+keys = require('./RedisKeyBuilder')
+logger = require('logger-sharelatex')
+metrics = require('./Metrics')
+
+module.exports =
+	putDocInMemory : (project_id, doc_id, docLines, version, callback)->
+		timer = new metrics.Timer("redis.put-doc")
+		logger.log project_id:project_id, doc_id:doc_id, docLines:docLines, version: version, "putting doc in redis"
+		multi = rclient.multi()
+		multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
+		multi.set keys.projectKey({doc_id:doc_id}), project_id
+		multi.set keys.docVersion(doc_id:doc_id), version
+		multi.del keys.docOps(doc_id:doc_id)
+		multi.sadd keys.allDocs, doc_id
+		multi.sadd keys.docsInProject(project_id:project_id), doc_id
+		multi.exec (err, replys)->
+			timer.done()
+			callback(err)
+
+	removeDocFromMemory : (project_id, doc_id, callback)->
+		logger.log project_id:project_id, doc_id:doc_id, "removing doc from redis"
+		multi = rclient.multi()
+		multi.get keys.docLines(doc_id:doc_id)
+		multi.del keys.docLines(doc_id:doc_id)
+		multi.del keys.projectKey(doc_id:doc_id)
+		multi.del keys.docVersion(doc_id:doc_id)
+		multi.del keys.docOps(doc_id:doc_id)
+		multi.srem keys.docsInProject(project_id:project_id), doc_id
+		multi.srem keys.allDocs, doc_id
+		multi.exec (err, replys)->
+			if err?
+				logger.err project_id:project_id, doc_id:doc_id, err:err, "error removing doc from redis"
+				callback(err, null)
+			else
+				docLines = replys[0]
+				logger.log project_id:project_id, doc_id:doc_id, docLines:docLines, "removed doc from redis"
+				callback()
+
+	getDoc : (doc_id, callback = (error, lines, version) ->)->
+		timer = new metrics.Timer("redis.get-doc")
+		multi = rclient.multi()
+		linesKey = keys.docLines(doc_id:doc_id)
+		multi.get linesKey
+		multi.get keys.docVersion(doc_id:doc_id)
+		multi.exec (error, result)->
+			timer.done()
+			return callback(error) if error?
+			try
+				docLines = JSON.parse result[0]
+			catch e
+				return callback(e)
+			version = parseInt(result[1] or 0, 10)
+			callback null, docLines, version
+
+	getDocVersion: (doc_id, callback = (error, version) ->) ->
+		rclient.get keys.docVersion(doc_id: doc_id), (error, version) ->
+			return callback(error) if error?
+			version = parseInt(version, 10)
+			callback null, version
+
+	getCountOfDocsInMemory : (callback)->
+		rclient.smembers keys.allDocs, (err, members)->
+			len = members.length
+			callback null, len
+
+	setDocument : (doc_id, docLines, version, callback = (error) ->)->
+		multi = rclient.multi()
+		multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
+		multi.set keys.docVersion(doc_id:doc_id), version
+		multi.incr keys.now("docsets")
+		multi.exec (error, replys) -> callback(error)
+
+	getPendingUpdatesForDoc : (doc_id, callback)->
+		multi = rclient.multi()
+		multi.lrange keys.pendingUpdates(doc_id:doc_id), 0 , -1
+		multi.del keys.pendingUpdates(doc_id:doc_id)
+		multi.exec (error, replys) ->
+			jsonUpdates = replys[0]
+			updates = []
+			for jsonUpdate in jsonUpdates
+				try
+					update = JSON.parse jsonUpdate
+				catch e
+					return callback e
+				updates.push update
+			callback error, updates
+
+	getUpdatesLength: (doc_id, callback)->
+		rclient.llen keys.pendingUpdates(doc_id:doc_id), callback
+
+	getDocsWithPendingUpdates: (callback = (error, docs) ->) ->
+		rclient.smembers keys.docsWithPendingUpdates, (error, doc_keys) ->
+			return callback(error) if error?
+			docs = doc_keys.map (doc_key) ->
+				[project_id, doc_id] = keys.splitProjectIdAndDocId(doc_key)
+				return {
+					doc_id: doc_id
+					project_id: project_id
+				}
+			callback null, docs
+
+	clearDocFromPendingUpdatesSet: (project_id, doc_id, callback = (error) ->) ->
+		doc_key = keys.combineProjectIdAndDocId(project_id, doc_id)
+		rclient.srem keys.docsWithPendingUpdates, doc_key, callback
+
+	getPreviousDocOps: (doc_id, start, end, callback = (error, jsonOps) ->) ->
+		# TODO: parse the ops and return them as objects, not JSON
+		rclient.llen keys.docOps(doc_id: doc_id), (error, length) ->
+			return callback(error) if error?
+			rclient.get keys.docVersion(doc_id: doc_id), (error, version) ->
+				return callback(error) if error?
+				version = parseInt(version, 10)
+				first_version_in_redis = version - length
+
+				if start < first_version_in_redis or end > version
+					error = new Error("doc ops range is not loaded in redis")
+					logger.error err: error, length: length, version: version, start: start, end: end, "inconsistent version or length"
+					return callback(error)
+
+				start = start - first_version_in_redis
+				if end > -1
+					end = end - first_version_in_redis
+
+				if isNaN(start) or isNaN(end)
+					error = new Error("inconsistent version or lengths")
+					logger.error err: error, length: length, version: version, start: start, end: end, "inconsistent version or length"
+					return callback(error)
+
+				rclient.lrange keys.docOps(doc_id: doc_id), start, end, (error, jsonOps) ->
+					return callback(error) if error?
+					try
+						ops = jsonOps.map (jsonOp) -> JSON.parse jsonOp
+					catch e
+						return callback(e)
+					callback null, ops
+
+	pushDocOp: (doc_id, op, callback = (error, new_version) ->) ->
+		# TODO: take a raw op object and JSONify it here
+		jsonOp = JSON.stringify op
+		rclient.rpush keys.docOps(doc_id: doc_id), jsonOp, (error) ->
+			return callback(error) if error?
+			rclient.incr keys.docVersion(doc_id: doc_id), (error, version) ->
+				return callback(error) if error?
+				version = parseInt(version, 10)
+				callback null, version
+
+	prependDocOps: (doc_id, ops, callback = (error) ->) ->
+		jsonOps = ops.map (op) -> JSON.stringify op
+		rclient.lpush keys.docOps(doc_id: doc_id), jsonOps.reverse(), callback
+
+	getDocOpsLength: (doc_id, callback = (error, length) ->) ->
+		rclient.llen keys.docOps(doc_id: doc_id), callback
+
+	getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) ->
+		rclient.smembers keys.docsInProject(project_id: project_id), callback
+		
+
+getDocumentsProjectId = (doc_id, callback)->
+	rclient.get keys.projectKey({doc_id:doc_id}), (err, project_id)->
+		callback err, {doc_id:doc_id, project_id:project_id}
+
+getAllProjectDocsIds = (project_id, callback)->
+	rclient.SMEMBERS keys.docsInProject(project_id:project_id), (err, doc_ids)->
+		if callback?
+			callback(err, doc_ids)
+
+getDocumentsAndExpire = (doc_ids, callback)->
+	multi = rclient.multi()
+	oneDay = 86400
+	doc_ids.forEach (doc_id)->
+		#		rclient.expire keys.docLines(doc_id:doc_id), oneDay, ->
+	doc_ids.forEach (doc_id)->
+		multi.get keys.docLines(doc_id:doc_id)
+	multi.exec (err, docsLines)->
+		callback err, docsLines
+	
+

services/document-updater/app/coffee/ShareJsDB.coffee

@@ -0,0 +1,58 @@
+Keys = require('./RedisKeyBuilder')
+Settings = require('settings-sharelatex')
+DocumentManager = require "./DocumentManager"
+RedisManager = require "./RedisManager"
+DocOpsManager = require "./DocOpsManager"
+Errors = require "./Errors"
+
+module.exports = ShareJsDB =
+	getOps: (doc_key, start, end, callback) ->
+		if start == end
+			return callback null, []
+
+		# In redis, lrange values are inclusive.
+		if end?
+			end--
+		else
+			end = -1
+
+		[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)
+		DocOpsManager.getPreviousDocOps project_id, doc_id, start, end, (error, ops) ->
+			return callback error if error?
+			callback null, ops
+	
+	writeOp: (doc_key, opData, callback) ->
+		[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)
+		DocOpsManager.pushDocOp project_id, doc_id, {op:opData.op, meta:opData.meta}, (error, version) ->
+			return callback error if error?
+
+			if version == opData.v + 1
+				callback()
+			else
+				# The document has been corrupted by the change. For now, throw an exception.
+				# Later, rebuild the snapshot.
+				callback "Version mismatch in db.append. '#{doc_id}' is corrupted."
+
+	getSnapshot: (doc_key, callback) ->
+		[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)
+		DocumentManager.getDoc project_id, doc_id, (error, lines, version) ->
+			return callback(error) if error?
+			if !lines? or !version?
+				return callback(new Errors.NotFoundError("document not found: #{doc_id}"))
+
+			if lines.length > 0 and lines[0].text?
+				type = "json"
+				snapshot = lines: lines
+			else
+				type = "text"
+				snapshot = lines.join("\n")
+			callback null,
+				snapshot: snapshot
+				v: parseInt(version, 10)
+				type: type
+
+	# To be able to remove a doc from the ShareJS memory
+	# we need to called Model::delete, which calls this 
+	# method on the database. However, we will handle removing
+	# it from Redis ourselves
+	delete: (docName, dbMeta, callback) -> callback()

services/document-updater/app/coffee/ShareJsUpdateManager.coffee

@@ -0,0 +1,68 @@
+ShareJsModel = require "./sharejs/server/model"
+ShareJsDB = require "./ShareJsDB"
+async = require "async"
+logger = require "logger-sharelatex"
+Settings = require('settings-sharelatex')
+Keys = require "./RedisKeyBuilder"
+{EventEmitter} = require "events"
+util = require "util"
+
+redis = require('redis')
+redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379}
+rclient = redis.createClient(redisConf.port, redisConf.host)
+rclient.auth(redisConf.password)
+
+ShareJsModel:: = {}
+util.inherits ShareJsModel, EventEmitter
+
+module.exports = ShareJsUpdateManager =
+	getNewShareJsModel: () -> new ShareJsModel(ShareJsDB)
+
+	applyUpdates: (project_id, doc_id, updates, callback = (error, updatedDocLines) ->) ->
+		logger.log project_id: project_id, doc_id: doc_id, updates: updates, "applying sharejs updates"
+		jobs = []
+
+		# We could use a global model for all docs, but we're hitting issues with the
+		# internal state of ShareJS not being accessible for clearing caches, and
+		# getting stuck due to queued callbacks (line 260 of sharejs/server/model.coffee)
+		# This adds a small but hopefully acceptable overhead (~12ms per 1000 updates on
+		# my 2009 MBP).
+		model = @getNewShareJsModel()
+		@_listenForOps(model)
+		doc_key = Keys.combineProjectIdAndDocId(project_id, doc_id)
+		for update in updates
+			do (update) =>
+				jobs.push (callback) =>
+					model.applyOp doc_key, update, callback
+
+		async.series jobs, (error) =>
+			logger.log project_id: project_id, doc_id: doc_id, error: error, "applied updates"
+			if error?
+				@_sendError(project_id, doc_id, error)
+				return callback(error)
+			model.getSnapshot doc_key, (error, data) =>
+				if error?
+					@_sendError(project_id, doc_id, error)
+					return callback(error)
+				if typeof data.snapshot == "string"
+					docLines = data.snapshot.split("\n")
+				else
+					docLines = data.snapshot.lines
+				callback(null, docLines, data.v)
+
+	_listenForOps: (model) ->
+		model.on "applyOp", (doc_key, opData) ->
+			[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)
+			data = JSON.stringify
+				project_id: project_id
+				doc_id: doc_id
+				op: opData
+			rclient.publish "applied-ops", data
+
+	_sendError: (project_id, doc_id, error) ->
+		data = JSON.stringify
+			project_id: project_id
+			doc_id: doc_id
+			error: error.message || error
+		rclient.publish "applied-ops", data
+		

services/document-updater/app/coffee/UpdateManager.coffee

@@ -0,0 +1,79 @@
+LockManager = require "./LockManager"
+RedisManager = require "./RedisManager"
+ShareJsUpdateManager = require "./ShareJsUpdateManager"
+Settings = require('settings-sharelatex')
+async = require("async")
+logger = require('logger-sharelatex')
+Metrics = require "./Metrics"
+
+module.exports = UpdateManager =
+	resumeProcessing: (callback = (error) ->) ->
+		RedisManager.getDocsWithPendingUpdates (error, docs) =>
+			return callback(error) if error?
+			jobs = for doc in (docs or [])
+				do (doc) =>
+					(callback) => @processOutstandingUpdatesWithLock doc.project_id, doc.doc_id, callback
+
+			async.parallelLimit jobs, 5, callback
+
+	processOutstandingUpdates: (project_id, doc_id, _callback = (error) ->) ->
+		timer = new Metrics.Timer("updateManager.processOutstandingUpdates")
+		callback = (args...) ->
+			timer.done()
+			_callback(args...)
+		
+		UpdateManager.fetchAndApplyUpdates project_id, doc_id, (error) =>
+			return callback(error) if error?
+			RedisManager.clearDocFromPendingUpdatesSet project_id, doc_id, (error) =>
+				return callback(error) if error?
+				callback()
+
+	processOutstandingUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
+		LockManager.tryLock doc_id, (error, gotLock) =>
+			return callback(error) if error?
+			return callback() if !gotLock
+			UpdateManager.processOutstandingUpdates project_id, doc_id, (error) ->
+				return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error?
+				LockManager.releaseLock doc_id, (error) =>
+					return callback(error) if error?
+					UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id, callback
+
+	continueProcessingUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
+		RedisManager.getUpdatesLength doc_id, (error, length) =>
+			return callback(error) if error?
+			if length > 0
+				UpdateManager.processOutstandingUpdatesWithLock project_id, doc_id, callback
+			else
+				callback()
+
+	fetchAndApplyUpdates: (project_id, doc_id, callback = (error) ->) ->
+		RedisManager.getPendingUpdatesForDoc doc_id, (error, updates) =>
+			return callback(error) if error?
+			if updates.length == 0
+				return callback()
+			UpdateManager.applyUpdates project_id, doc_id, updates, callback
+
+	applyUpdates: (project_id, doc_id, updates, callback = (error) ->) ->
+		ShareJsUpdateManager.applyUpdates project_id, doc_id, updates, (error, updatedDocLines, version) ->
+			return callback(error) if error?
+			logger.log doc_id: doc_id, version: version, "updating doc via sharejs"
+			RedisManager.setDocument doc_id, updatedDocLines, version, callback
+
+	lockUpdatesAndDo: (method, project_id, doc_id, args..., callback) ->
+		LockManager.getLock doc_id, (error) ->
+			return callback(error) if error?
+			UpdateManager.processOutstandingUpdates project_id, doc_id, (error) ->
+				return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error?
+				method project_id, doc_id, args..., (error, response_args...) ->
+					return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error?
+					LockManager.releaseLock doc_id, (error) ->
+						return callback(error) if error?
+						callback null, response_args...
+						# We held the lock for a while so updates might have queued up
+						UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id
+
+	_handleErrorInsideLock: (doc_id, original_error, callback = (error) ->) ->
+		LockManager.releaseLock doc_id, (lock_error) ->
+			callback(original_error)
+
+			

services/document-updater/app/coffee/mongojs.coffee

@@ -0,0 +1,7 @@
+Settings = require "settings-sharelatex"
+mongojs = require "mongojs"
+db = mongojs.connect(Settings.mongo.url, ["docOps"])
+module.exports =
+	db: db
+	ObjectId: mongojs.ObjectId
+

services/document-updater/app/coffee/sharejs/README.md

@@ -0,0 +1,48 @@
+This directory contains all the operational transform code. Each file defines a type.
+
+Most of the types in here are for testing or demonstration. The only types which are sent to the webclient
+are `text` and `json`.
+
+
+# An OT type
+
+All OT types have the following fields:
+
+`name`: _(string)_ Name of the type. Should match the filename.
+`create() -> snapshot`: Function which creates and returns a new document snapshot
+
+`apply(snapshot, op) -> snapshot`: A function which creates a new document snapshot with the op applied
+`transform(op1, op2, side) -> op1'`: OT transform function.
+
+Given op1, op2, `apply(s, op2, transform(op1, op2, 'left')) == apply(s, op1, transform(op2, op1, 'right'))`.
+
+Transform and apply must never modify their arguments.
+
+
+Optional properties:
+
+`tp2`: _(bool)_ True if the transform function supports TP2. This allows p2p architectures to work.
+`compose(op1, op2) -> op`: Create and return a new op which has the same effect as op1 + op2.
+`serialize(snapshot) -> JSON object`: Serialize a document to something we can JSON.stringify()
+`deserialize(object) -> snapshot`: Deserialize a JSON object into the document's internal snapshot format
+`prune(op1', op2, side) -> op1`: Inserse transform function. Only required for TP2 types.
+`normalize(op) -> op`: Fix up an op to make it valid. Eg, remove skips of size zero.
+`api`: _(object)_ Set of helper methods which will be mixed in to the client document object for manipulating documents. See below.
+
+
+# Examples
+
+`count` and `simple` are two trivial OT type definitions if you want to take a look. JSON defines
+the ot-for-JSON type (see the wiki for documentation) and all the text types define different text
+implementations. (I still have no idea which one I like the most, and they're fun to write!)
+
+
+# API
+
+Types can also define API functions. These methods are mixed into the client's Doc object when a document is created.
+You can use them to help construct ops programatically (so users don't need to understand how ops are structured).
+
+For example, the three text types defined here (text, text-composable and text-tp2) all provide the text API, supplying
+`.insert()`, `.del()`, `.getLength` and `.getText` methods.
+
+See text-api.coffee for an example.

services/document-updater/app/coffee/sharejs/count.coffee

@@ -0,0 +1,22 @@
+# This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment]
+
+exports.name = 'count'
+exports.create = -> 1
+
+exports.apply = (snapshot, op) ->
+  [v, inc] = op
+  throw new Error "Op #{v} != snapshot #{snapshot}" unless snapshot == v
+  snapshot + inc
+
+# transform op1 by op2. Return transformed version of op1.
+exports.transform = (op1, op2) ->
+  throw new Error "Op1 #{op1[0]} != op2 #{op2[0]}" unless op1[0] == op2[0]
+  [op1[0] + op2[1], op1[1]]
+
+exports.compose = (op1, op2) ->
+  throw new Error "Op1 #{op1} + 1 != op2 #{op2}" unless op1[0] + op1[1] == op2[0]
+  [op1[0], op1[1] + op2[1]]
+
+exports.generateRandomOp = (doc) ->
+  [[doc, 1], doc + 1]
+

services/document-updater/app/coffee/sharejs/helpers.coffee

@@ -0,0 +1,65 @@
+# These methods let you build a transform function from a transformComponent function
+# for OT types like text and JSON in which operations are lists of components
+# and transforming them requires N^2 work.
+
+# Add transform and transformX functions for an OT type which has transformComponent defined.
+# transformComponent(destination array, component, other component, side)
+exports['_bt'] = bootstrapTransform = (type, transformComponent, checkValidOp, append) ->
+  transformComponentX = (left, right, destLeft, destRight) ->
+    transformComponent destLeft, left, right, 'left'
+    transformComponent destRight, right, left, 'right'
+
+  # Transforms rightOp by leftOp. Returns ['rightOp', clientOp']
+  type.transformX = type['transformX'] = transformX = (leftOp, rightOp) ->
+    checkValidOp leftOp
+    checkValidOp rightOp
+
+    newRightOp = []
+
+    for rightComponent in rightOp
+      # Generate newLeftOp by composing leftOp by rightComponent
+      newLeftOp = []
+
+      k = 0
+      while k < leftOp.length
+        nextC = []
+        transformComponentX leftOp[k], rightComponent, newLeftOp, nextC
+        k++
+
+        if nextC.length == 1
+          rightComponent = nextC[0]
+        else if nextC.length == 0
+          append newLeftOp, l for l in leftOp[k..]
+          rightComponent = null
+          break
+        else
+          # Recurse.
+          [l_, r_] = transformX leftOp[k..], nextC
+          append newLeftOp, l for l in l_
+          append newRightOp, r for r in r_
+          rightComponent = null
+          break
+    
+      append newRightOp, rightComponent if rightComponent?
+      leftOp = newLeftOp
+    
+    [leftOp, newRightOp]
+
+  # Transforms op with specified type ('left' or 'right') by otherOp.
+  type.transform = type['transform'] = (op, otherOp, type) ->
+    throw new Error "type must be 'left' or 'right'" unless type == 'left' or type == 'right'
+
+    return op if otherOp.length == 0
+
+    # TODO: Benchmark with and without this line. I _think_ it'll make a big difference...?
+    return transformComponent [], op[0], otherOp[0], type if op.length == 1 and otherOp.length == 1
+
+    if type == 'left'
+      [left, _] = transformX op, otherOp
+      left
+    else
+      [_, right] = transformX otherOp, op
+      right
+
+if typeof WEB is 'undefined'
+  exports.bootstrapTransform = bootstrapTransform

services/document-updater/app/coffee/sharejs/index.coffee

@@ -0,0 +1,15 @@
+
+register = (file) ->
+  type = require file
+  exports[type.name] = type
+  try require "#{file}-api"
+
+# Import all the built-in types.
+register './simple'
+register './count'
+
+register './text'
+register './text-composable'
+register './text-tp2'
+
+register './json'

services/document-updater/app/coffee/sharejs/json-api.coffee

@@ -0,0 +1,180 @@
+# API for JSON OT
+
+json = require './json' if typeof WEB is 'undefined'
+
+if WEB?
+  extendDoc = exports.extendDoc
+  exports.extendDoc = (name, fn) ->
+    SubDoc::[name] = fn
+    extendDoc name, fn
+
+depath = (path) ->
+  if path.length == 1 and path[0].constructor == Array
+    path[0]
+  else path
+
+class SubDoc
+  constructor: (@doc, @path) ->
+  at: (path...) -> @doc.at @path.concat depath path
+  get: -> @doc.getAt @path
+  # for objects and lists
+  set: (value, cb) -> @doc.setAt @path, value, cb
+  # for strings and lists.
+  insert: (pos, value, cb) -> @doc.insertAt @path, pos, value, cb
+  # for strings
+  del: (pos, length, cb) -> @doc.deleteTextAt @path, length, pos, cb
+  # for objects and lists
+  remove: (cb) -> @doc.removeAt @path, cb
+  push: (value, cb) -> @insert @get().length, value, cb
+  move: (from, to, cb) -> @doc.moveAt @path, from, to, cb
+  add: (amount, cb) -> @doc.addAt @path, amount, cb
+  on: (event, cb) -> @doc.addListener @path, event, cb
+  removeListener: (l) -> @doc.removeListener l
+
+  # text API compatibility
+  getLength: -> @get().length
+  getText: -> @get()
+
+traverse = (snapshot, path) ->
+  container = data:snapshot
+  key = 'data'
+  elem = container
+  for p in path
+    elem = elem[key]
+    key = p
+    throw new Error 'bad path' if typeof elem == 'undefined'
+  {elem, key}
+
+pathEquals = (p1, p2) ->
+  return false if p1.length != p2.length
+  for e,i in p1
+    return false if e != p2[i]
+  true
+
+json.api =
+  provides: {json:true}
+
+  at: (path...) -> new SubDoc this, depath path
+
+  get: -> @snapshot
+  set: (value, cb) -> @setAt [], value, cb
+
+  getAt: (path) ->
+    {elem, key} = traverse @snapshot, path
+    return elem[key]
+
+  setAt: (path, value, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = {p:path}
+    if elem.constructor == Array
+      op.li = value
+      op.ld = elem[key] if typeof elem[key] != 'undefined'
+    else if typeof elem == 'object'
+      op.oi = value
+      op.od = elem[key] if typeof elem[key] != 'undefined'
+    else throw new Error 'bad path'
+    @submitOp [op], cb
+
+  removeAt: (path, cb) ->
+    {elem, key} = traverse @snapshot, path
+    throw new Error 'no element at that path' unless typeof elem[key] != 'undefined'
+    op = {p:path}
+    if elem.constructor == Array
+      op.ld = elem[key]
+    else if typeof elem == 'object'
+      op.od = elem[key]
+    else throw new Error 'bad path'
+    @submitOp [op], cb
+
+  insertAt: (path, pos, value, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = {p:path.concat pos}
+    if elem[key].constructor == Array
+      op.li = value
+    else if typeof elem[key] == 'string'
+      op.si = value
+    @submitOp [op], cb
+
+  moveAt: (path, from, to, cb) ->
+    op = [{p:path.concat(from), lm:to}]
+    @submitOp op, cb
+
+  addAt: (path, amount, cb) ->
+    op = [{p:path, na:amount}]
+    @submitOp op, cb
+
+  deleteTextAt: (path, length, pos, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = [{p:path.concat(pos), sd:elem[key][pos...(pos + length)]}]
+    @submitOp op, cb
+
+  addListener: (path, event, cb) ->
+    l = {path, event, cb}
+    @_listeners.push l
+    l
+  removeListener: (l) ->
+    i = @_listeners.indexOf l
+    return false if i < 0
+    @_listeners.splice i, 1
+    return true
+  _register: ->
+    @_listeners = []
+    @on 'change', (op) ->
+      for c in op
+        if c.na != undefined or c.si != undefined or c.sd != undefined
+          # no change to structure
+          continue
+        to_remove = []
+        for l, i in @_listeners
+          # Transform a dummy op by the incoming op to work out what
+          # should happen to the listener.
+          dummy = {p:l.path, na:0}
+          xformed = @type.transformComponent [], dummy, c, 'left'
+          if xformed.length == 0
+            # The op was transformed to noop, so we should delete the listener.
+            to_remove.push i
+          else if xformed.length == 1
+            # The op remained, so grab its new path into the listener.
+            l.path = xformed[0].p
+          else
+            throw new Error "Bad assumption in json-api: xforming an 'si' op will always result in 0 or 1 components."
+        to_remove.sort (a, b) -> b - a
+        for i in to_remove
+          @_listeners.splice i, 1
+    @on 'remoteop', (op) ->
+      for c in op
+        match_path = if c.na == undefined then c.p[...c.p.length-1] else c.p
+        for {path, event, cb} in @_listeners
+          if pathEquals path, match_path
+            switch event
+              when 'insert'
+                if c.li != undefined and c.ld == undefined
+                  cb(c.p[c.p.length-1], c.li)
+                else if c.oi != undefined and c.od == undefined
+                  cb(c.p[c.p.length-1], c.oi)
+                else if c.si != undefined
+                  cb(c.p[c.p.length-1], c.si)
+              when 'delete'
+                if c.li == undefined and c.ld != undefined
+                  cb(c.p[c.p.length-1], c.ld)
+                else if c.oi == undefined and c.od != undefined
+                  cb(c.p[c.p.length-1], c.od)
+                else if c.sd != undefined
+                  cb(c.p[c.p.length-1], c.sd)
+              when 'replace'
+                if c.li != undefined and c.ld != undefined
+                  cb(c.p[c.p.length-1], c.ld, c.li)
+                else if c.oi != undefined and c.od != undefined
+                  cb(c.p[c.p.length-1], c.od, c.oi)
+              when 'move'
+                if c.lm != undefined
+                  cb(c.p[c.p.length-1], c.lm)
+              when 'add'
+                if c.na != undefined
+                  cb(c.na)
+          else if (common = @type.commonPath match_path, path)?
+            if event == 'child op'
+              if match_path.length == path.length == common
+                throw new Error "paths match length and have commonality, but aren't equal?"
+              child_path = c.p[common+1..]
+              cb(child_path, c)

services/document-updater/app/coffee/sharejs/json.coffee

@@ -0,0 +1,441 @@
+# This is the implementation of the JSON OT type.
+#
+# Spec is here: https://github.com/josephg/ShareJS/wiki/JSON-Operations
+
+if WEB?
+  text = exports.types.text
+else
+  text = require './text'
+
+json = {}
+
+json.name = 'json'
+
+json.create = -> null
+
+json.invertComponent = (c) ->
+  c_ = {p: c.p}
+  c_.sd = c.si if c.si != undefined
+  c_.si = c.sd if c.sd != undefined
+  c_.od = c.oi if c.oi != undefined
+  c_.oi = c.od if c.od != undefined
+  c_.ld = c.li if c.li != undefined
+  c_.li = c.ld if c.ld != undefined
+  c_.na = -c.na if c.na != undefined
+  if c.lm != undefined
+    c_.lm = c.p[c.p.length-1]
+    c_.p = c.p[0...c.p.length - 1].concat([c.lm])
+  c_
+
+json.invert = (op) -> json.invertComponent c for c in op.slice().reverse()
+
+json.checkValidOp = (op) ->
+
+isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]'
+json.checkList = (elem) ->
+  throw new Error 'Referenced element not a list' unless isArray(elem)
+
+json.checkObj = (elem) ->
+  throw new Error "Referenced element not an object (it was #{JSON.stringify elem})" unless elem.constructor is Object
+
+json.apply = (snapshot, op) ->
+  json.checkValidOp op
+  op = clone op
+
+  container = {data: clone snapshot}
+
+  try
+    for c, i in op
+      parent = null
+      parentkey = null
+      elem = container
+      key = 'data'
+
+      for p in c.p
+        parent = elem
+        parentkey = key
+        elem = elem[key]
+        key = p
+
+        throw new Error 'Path invalid' unless parent?
+
+      if c.na != undefined
+        # Number add
+        throw new Error 'Referenced element not a number' unless typeof elem[key] is 'number'
+        elem[key] += c.na
+
+      else if c.si != undefined
+        # String insert
+        throw new Error "Referenced element not a string (it was #{JSON.stringify elem})" unless typeof elem is 'string'
+        parent[parentkey] = elem[...key] + c.si + elem[key..]
+      else if c.sd != undefined
+        # String delete
+        throw new Error 'Referenced element not a string' unless typeof elem is 'string'
+        throw new Error 'Deleted string does not match' unless elem[key...key + c.sd.length] == c.sd
+        parent[parentkey] = elem[...key] + elem[key + c.sd.length..]
+
+      else if c.li != undefined && c.ld != undefined
+        # List replace
+        json.checkList elem
+
+        # Should check the list element matches c.ld
+        elem[key] = c.li
+      else if c.li != undefined
+        # List insert
+        json.checkList elem
+
+        elem.splice key, 0, c.li
+      else if c.ld != undefined
+        # List delete
+        json.checkList elem
+
+        # Should check the list element matches c.ld here too.
+        elem.splice key, 1
+      else if c.lm != undefined
+        # List move
+        json.checkList elem
+        if c.lm != key
+          e = elem[key]
+          # Remove it...
+          elem.splice key, 1
+          # And insert it back.
+          elem.splice c.lm, 0, e
+
+      else if c.oi != undefined
+        # Object insert / replace
+        json.checkObj elem
+        
+        # Should check that elem[key] == c.od
+        elem[key] = c.oi
+      else if c.od != undefined
+        # Object delete
+        json.checkObj elem
+
+        # Should check that elem[key] == c.od
+        delete elem[key]
+      else
+        throw new Error 'invalid / missing instruction in op'
+  catch error
+    # TODO: Roll back all already applied changes. Write tests before implementing this code.
+    throw error
+
+  container.data
+
+# Checks if two paths, p1 and p2 match.
+json.pathMatches = (p1, p2, ignoreLast) ->
+  return false unless p1.length == p2.length
+
+  for p, i in p1
+    return false if p != p2[i] and (!ignoreLast or i != p1.length - 1)
+      
+  true
+
+json.append = (dest, c) ->
+  c = clone c
+  if dest.length != 0 and json.pathMatches c.p, (last = dest[dest.length - 1]).p
+    if last.na != undefined and c.na != undefined
+      dest[dest.length - 1] = { p: last.p, na: last.na + c.na }
+    else if last.li != undefined and c.li == undefined and c.ld == last.li
+      # insert immediately followed by delete becomes a noop.
+      if last.ld != undefined
+        # leave the delete part of the replace
+        delete last.li
+      else
+        dest.pop()
+    else if last.od != undefined and last.oi == undefined and
+        c.oi != undefined and c.od == undefined
+      last.oi = c.oi
+    else if c.lm != undefined and c.p[c.p.length-1] == c.lm
+      null # don't do anything
+    else
+      dest.push c
+  else
+    dest.push c
+
+json.compose = (op1, op2) ->
+  json.checkValidOp op1
+  json.checkValidOp op2
+
+  newOp = clone op1
+  json.append newOp, c for c in op2
+
+  newOp
+
+json.normalize = (op) ->
+  newOp = []
+  
+  op = [op] unless isArray op
+
+  for c in op
+    c.p ?= []
+    json.append newOp, c
+  
+  newOp
+
+# hax, copied from test/types/json. Apparently this is still the fastest way to deep clone an object, assuming
+# we have browser support for JSON.
+# http://jsperf.com/cloning-an-object/12
+clone = (o) -> JSON.parse(JSON.stringify o)
+
+json.commonPath = (p1, p2) ->
+  p1 = p1.slice()
+  p2 = p2.slice()
+  p1.unshift('data')
+  p2.unshift('data')
+  p1 = p1[...p1.length-1]
+  p2 = p2[...p2.length-1]
+  return -1 if p2.length == 0
+  i = 0
+  while p1[i] == p2[i] && i < p1.length
+    i++
+    if i == p2.length
+      return i-1
+  return
+
+# transform c so it applies to a document with otherC applied.
+json.transformComponent = (dest, c, otherC, type) ->
+  c = clone c
+  c.p.push(0) if c.na != undefined
+  otherC.p.push(0) if otherC.na != undefined
+
+  common = json.commonPath c.p, otherC.p
+  common2 = json.commonPath otherC.p, c.p
+
+  cplength = c.p.length
+  otherCplength = otherC.p.length
+
+  c.p.pop() if c.na != undefined # hax
+  otherC.p.pop() if otherC.na != undefined
+
+  if otherC.na
+    if common2? && otherCplength >= cplength && otherC.p[common2] == c.p[common2]
+      if c.ld != undefined
+        oc = clone otherC
+        oc.p = oc.p[cplength..]
+        c.ld = json.apply clone(c.ld), [oc]
+      else if c.od != undefined
+        oc = clone otherC
+        oc.p = oc.p[cplength..]
+        c.od = json.apply clone(c.od), [oc]
+    json.append dest, c
+    return dest
+
+  if common2? && otherCplength > cplength && c.p[common2] == otherC.p[common2]
+    # transform based on c
+    if c.ld != undefined
+      oc = clone otherC
+      oc.p = oc.p[cplength..]
+      c.ld = json.apply clone(c.ld), [oc]
+    else if c.od != undefined
+      oc = clone otherC
+      oc.p = oc.p[cplength..]
+      c.od = json.apply clone(c.od), [oc]
+
+
+  if common?
+    commonOperand = cplength == otherCplength
+    # transform based on otherC
+    if otherC.na != undefined
+      # this case is handled above due to icky path hax
+    else if otherC.si != undefined || otherC.sd != undefined
+      # String op vs string op - pass through to text type
+      if c.si != undefined || c.sd != undefined
+        throw new Error("must be a string?") unless commonOperand
+
+        # Convert an op component to a text op component
+        convert = (component) ->
+          newC = p:component.p[component.p.length - 1]
+          if component.si
+            newC.i = component.si
+          else
+            newC.d = component.sd
+          newC
+
+        tc1 = convert c
+        tc2 = convert otherC
+          
+        res = []
+        text._tc res, tc1, tc2, type
+        for tc in res
+          jc = { p: c.p[...common] }
+          jc.p.push(tc.p)
+          jc.si = tc.i if tc.i?
+          jc.sd = tc.d if tc.d?
+          json.append dest, jc
+        return dest
+    else if otherC.li != undefined && otherC.ld != undefined
+      if otherC.p[common] == c.p[common]
+        # noop
+        if !commonOperand
+          # we're below the deleted element, so -> noop
+          return dest
+        else if c.ld != undefined
+          # we're trying to delete the same element, -> noop
+          if c.li != undefined and type == 'left'
+            # we're both replacing one element with another. only one can
+            # survive!
+            c.ld = clone otherC.li
+          else
+            return dest
+    else if otherC.li != undefined
+      if c.li != undefined and c.ld == undefined and commonOperand and c.p[common] == otherC.p[common]
+        # in li vs. li, left wins.
+        if type == 'right'
+          c.p[common]++
+      else if otherC.p[common] <= c.p[common]
+        c.p[common]++
+
+      if c.lm != undefined
+        if commonOperand
+          # otherC edits the same list we edit
+          if otherC.p[common] <= c.lm
+            c.lm++
+          # changing c.from is handled above.
+    else if otherC.ld != undefined
+      if c.lm != undefined
+        if commonOperand
+          if otherC.p[common] == c.p[common]
+            # they deleted the thing we're trying to move
+            return dest
+          # otherC edits the same list we edit
+          p = otherC.p[common]
+          from = c.p[common]
+          to = c.lm
+          if p < to || (p == to && from < to)
+            c.lm--
+
+      if otherC.p[common] < c.p[common]
+        c.p[common]--
+      else if otherC.p[common] == c.p[common]
+        if otherCplength < cplength
+          # we're below the deleted element, so -> noop
+          return dest
+        else if c.ld != undefined
+          if c.li != undefined
+            # we're replacing, they're deleting. we become an insert.
+            delete c.ld
+          else
+            # we're trying to delete the same element, -> noop
+            return dest
+    else if otherC.lm != undefined
+      if c.lm != undefined and cplength == otherCplength
+        # lm vs lm, here we go!
+        from = c.p[common]
+        to = c.lm
+        otherFrom = otherC.p[common]
+        otherTo = otherC.lm
+        if otherFrom != otherTo
+          # if otherFrom == otherTo, we don't need to change our op.
+
+          # where did my thing go?
+          if from == otherFrom
+            # they moved it! tie break.
+            if type == 'left'
+              c.p[common] = otherTo
+              if from == to # ugh
+                c.lm = otherTo
+            else
+              return dest
+          else
+            # they moved around it
+            if from > otherFrom
+              c.p[common]--
+            if from > otherTo
+              c.p[common]++
+            else if from == otherTo
+              if otherFrom > otherTo
+                c.p[common]++
+                if from == to # ugh, again
+                  c.lm++
+
+            # step 2: where am i going to put it?
+            if to > otherFrom
+              c.lm--
+            else if to == otherFrom
+              if to > from
+                c.lm--
+            if to > otherTo
+              c.lm++
+            else if to == otherTo
+              # if we're both moving in the same direction, tie break
+              if (otherTo > otherFrom and to > from) or
+                 (otherTo < otherFrom and to < from)
+                if type == 'right'
+                  c.lm++
+              else
+                if to > from
+                  c.lm++
+                else if to == otherFrom
+                  c.lm--
+      else if c.li != undefined and c.ld == undefined and commonOperand
+        # li
+        from = otherC.p[common]
+        to = otherC.lm
+        p = c.p[common]
+        if p > from
+          c.p[common]--
+        if p > to
+          c.p[common]++
+      else
+        # ld, ld+li, si, sd, na, oi, od, oi+od, any li on an element beneath
+        # the lm
+        #
+        # i.e. things care about where their item is after the move.
+        from = otherC.p[common]
+        to = otherC.lm
+        p = c.p[common]
+        if p == from
+          c.p[common] = to
+        else
+          if p > from
+            c.p[common]--
+          if p > to
+            c.p[common]++
+          else if p == to
+            if from > to
+              c.p[common]++
+    else if otherC.oi != undefined && otherC.od != undefined
+      if c.p[common] == otherC.p[common]
+        if c.oi != undefined and commonOperand
+          # we inserted where someone else replaced
+          if type == 'right'
+            # left wins
+            return dest
+          else
+            # we win, make our op replace what they inserted
+            c.od = otherC.oi
+        else
+          # -> noop if the other component is deleting the same object (or any
+          # parent)
+          return dest
+    else if otherC.oi != undefined
+      if c.oi != undefined and c.p[common] == otherC.p[common]
+        # left wins if we try to insert at the same place
+        if type == 'left'
+          json.append dest, {p:c.p, od:otherC.oi}
+        else
+          return dest
+    else if otherC.od != undefined
+      if c.p[common] == otherC.p[common]
+        return dest if !commonOperand
+        if c.oi != undefined
+          delete c.od
+        else
+          return dest
+  
+  json.append dest, c
+  return dest
+
+if WEB?
+  exports.types ||= {}
+
+  # This is kind of awful - come up with a better way to hook this helper code up.
+  exports._bt(json, json.transformComponent, json.checkValidOp, json.append)
+
+  # [] is used to prevent closure from renaming types.text
+  exports.types.json = json
+else
+  module.exports = json
+
+  require('./helpers').bootstrapTransform(json, json.transformComponent, json.checkValidOp, json.append)
+

services/document-updater/app/coffee/sharejs/model.coffee

@@ -0,0 +1,603 @@
+# The model of all the ops. Responsible for applying & transforming remote deltas
+# and managing the storage layer.
+#
+# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
+
+{EventEmitter} = require 'events'
+
+queue = require './syncqueue'
+types = require '../types'
+
+isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]'
+
+# This constructor creates a new Model object. There will be one model object
+# per server context.
+#
+# The model object is responsible for a lot of things:
+#
+# - It manages the interactions with the database
+# - It maintains (in memory) a set of all active documents
+# - It calls out to the OT functions when necessary
+#
+# The model is an event emitter. It emits the following events:
+#
+# create(docName, data): A document has been created with the specified name & data
+module.exports = Model = (db, options) ->
+  # db can be null if the user doesn't want persistance.
+
+  return new Model(db, options) if !(this instanceof Model)
+
+  model = this
+
+  options ?= {}
+
+  # This is a cache of 'live' documents.
+  #
+  # The cache is a map from docName -> {
+  #   ops:[{op, meta}]
+  #   snapshot
+  #   type
+  #   v
+  #   meta
+  #   eventEmitter
+  #   reapTimer
+  #   committedVersion: v
+  #   snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant
+  #   dbMeta: database specific data
+  #   opQueue: syncQueue for processing ops
+  # }
+  #
+  # The ops list contains the document's last options.numCachedOps ops. (Or all
+  # of them if we're using a memory store).
+  #
+  # Documents are stored in this set so long as the document has been accessed in
+  # the last few seconds (options.reapTime) OR at least one client has the document
+  # open. I don't know if I should keep open (but not being edited) documents live -
+  # maybe if a client has a document open but the document isn't being edited, I should
+  # flush it from the cache.
+  #
+  # In any case, the API to model is designed such that if we want to change that later
+  # it should be pretty easy to do so without any external-to-the-model code changes.
+  docs = {}
+
+  # This is a map from docName -> [callback]. It is used when a document hasn't been
+  # cached and multiple getSnapshot() / getVersion() requests come in. All requests
+  # are added to the callback list and called when db.getSnapshot() returns.
+  #
+  # callback(error, snapshot data)
+  awaitingGetSnapshot = {}
+
+  # The time that documents which no clients have open will stay in the cache.
+  # Should be > 0.
+  options.reapTime ?= 3000
+
+  # The number of operations the cache holds before reusing the space
+  options.numCachedOps ?= 10
+
+  # This option forces documents to be reaped, even when there's no database backend.
+  # This is useful when you don't care about persistance and don't want to gradually
+  # fill memory.
+  #
+  # You might want to set reapTime to a day or something.
+  options.forceReaping ?= false
+
+  # Until I come up with a better strategy, we'll save a copy of the document snapshot
+  # to the database every ~20 submitted ops.
+  options.opsBeforeCommit ?= 20
+
+  # It takes some processing time to transform client ops. The server will punt ops back to the
+  # client to transform if they're too old.
+  options.maximumAge ?= 40
+
+  # **** Cache API methods
+
+  # Its important that all ops are applied in order. This helper method creates the op submission queue
+  # for a single document. This contains the logic for transforming & applying ops.
+  makeOpQueue = (docName, doc) -> queue (opData, callback) ->
+    return callback 'Version missing' unless opData.v >= 0
+    return callback 'Op at future version' if opData.v > doc.v
+
+    # Punt the transforming work back to the client if the op is too old.
+    return callback 'Op too old' if opData.v + options.maximumAge < doc.v
+
+    opData.meta ||= {}
+    opData.meta.ts = Date.now()
+
+    # We'll need to transform the op to the current version of the document. This
+    # calls the callback immediately if opVersion == doc.v.
+    getOps docName, opData.v, doc.v, (error, ops) ->
+      return callback error if error
+
+      unless doc.v - opData.v == ops.length
+        # This should never happen. It indicates that we didn't get all the ops we
+        # asked for. Its important that the submitted op is correctly transformed.
+        console.error "Could not get old ops in model for document #{docName}"
+        console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops"
+        return callback 'Internal error'
+
+      if ops.length > 0
+        try
+          # If there's enough ops, it might be worth spinning this out into a webworker thread.
+          for oldOp in ops
+            # Dup detection works by sending the id(s) the op has been submitted with previously.
+            # If the id matches, we reject it. The client can also detect the op has been submitted
+            # already if it sees its own previous id in the ops it sees when it does catchup.
+            if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource
+              return callback 'Op already submitted'
+
+            opData.op = doc.type.transform opData.op, oldOp.op, 'left'
+            opData.v++
+        catch error
+          console.error error.stack
+          return callback error.message
+
+      try
+        snapshot = doc.type.apply doc.snapshot, opData.op
+      catch error
+        console.error error.stack
+        return callback error.message
+
+      # The op data should be at the current version, and the new document data should be at
+      # the next version.
+      #
+      # This should never happen in practice, but its a nice little check to make sure everything
+      # is hunky-dory.
+      unless opData.v == doc.v
+        # This should never happen.
+        console.error "Version mismatch detected in model. File a ticket - this is a bug."
+        console.error "Expecting #{opData.v} == #{doc.v}"
+        return callback 'Internal error'
+
+      #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta}
+      writeOp = db?.writeOp or (docName, newOpData, callback) -> callback()
+
+      writeOp docName, opData, (error) ->
+        if error
+          # The user should probably know about this.
+          console.warn "Error writing ops to database: #{error}"
+          return callback error
+
+        options.stats?.writeOp?()
+
+        # This is needed when we emit the 'change' event, below.
+        oldSnapshot = doc.snapshot
+
+        # All the heavy lifting is now done. Finally, we'll update the cache with the new data
+        # and (maybe!) save a new document snapshot to the database.
+
+        doc.v = opData.v + 1
+        doc.snapshot = snapshot
+
+        doc.ops.push opData
+        doc.ops.shift() if db and doc.ops.length > options.numCachedOps
+
+        model.emit 'applyOp', docName, opData, snapshot, oldSnapshot
+        doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot
+
+        # The callback is called with the version of the document at which the op was applied.
+        # This is the op.v after transformation, and its doc.v - 1.
+        callback null, opData.v
+    
+        # I need a decent strategy here for deciding whether or not to save the snapshot.
+        #
+        # The 'right' strategy looks something like "Store the snapshot whenever the snapshot
+        # is smaller than the accumulated op data". For now, I'll just store it every 20
+        # ops or something. (Configurable with doc.committedVersion)
+        if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v
+          tryWriteSnapshot docName, (error) ->
+            console.warn "Error writing snapshot #{error}. This is nonfatal" if error
+
+  # Add the data for the given docName to the cache. The named document shouldn't already
+  # exist in the doc set.
+  #
+  # Returns the new doc.
+  add = (docName, error, data, committedVersion, ops, dbMeta) ->
+    callbacks = awaitingGetSnapshot[docName]
+    delete awaitingGetSnapshot[docName]
+
+    if error
+      callback error for callback in callbacks if callbacks
+    else
+      doc = docs[docName] =
+        snapshot: data.snapshot
+        v: data.v
+        type: data.type
+        meta: data.meta
+
+        # Cache of ops
+        ops: ops or []
+
+        eventEmitter: new EventEmitter
+
+        # Timer before the document will be invalidated from the cache (if the document has no
+        # listeners)
+        reapTimer: null
+
+        # Version of the snapshot thats in the database
+        committedVersion: committedVersion ? data.v
+        snapshotWriteLock: false
+        dbMeta: dbMeta
+
+      doc.opQueue = makeOpQueue docName, doc
+      
+      refreshReapingTimeout docName
+      model.emit 'add', docName, data
+      callback null, doc for callback in callbacks if callbacks
+
+    doc
+  
+  # This is a little helper wrapper around db.getOps. It does two things:
+  #
+  # - If there's no database set, it returns an error to the callback
+  # - It adds version numbers to each op returned from the database
+  # (These can be inferred from context so the DB doesn't store them, but its useful to have them).
+  getOpsInternal = (docName, start, end, callback) ->
+    return callback? 'Document does not exist' unless db
+
+    db.getOps docName, start, end, (error, ops) ->
+      return callback? error if error
+
+      v = start
+      op.v = v++ for op in ops
+
+      callback? null, ops
+
+  # Load the named document into the cache. This function is re-entrant.
+  #
+  # The callback is called with (error, doc)
+  load = (docName, callback) ->
+    if docs[docName]
+      # The document is already loaded. Return immediately.
+      options.stats?.cacheHit? 'getSnapshot'
+      return callback null, docs[docName]
+
+    # We're a memory store. If we don't have it, nobody does.
+    return callback 'Document does not exist' unless db
+
+    callbacks = awaitingGetSnapshot[docName]
+
+    # The document is being loaded already. Add ourselves as a callback.
+    return callbacks.push callback if callbacks
+
+    options.stats?.cacheMiss? 'getSnapshot'
+
+    # The document isn't loaded and isn't being loaded. Load it.
+    awaitingGetSnapshot[docName] = [callback]
+    db.getSnapshot docName, (error, data, dbMeta) ->
+      return add docName, error if error
+
+      type = types[data.type]
+      unless type
+        console.warn "Type '#{data.type}' missing"
+        return callback "Type not found"
+      data.type = type
+
+      committedVersion = data.v
+
+      # The server can close without saving the most recent document snapshot.
+      # In this case, there are extra ops which need to be applied before
+      # returning the snapshot.
+      getOpsInternal docName, data.v, null, (error, ops) ->
+        return callback error if error
+
+        if ops.length > 0
+          console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}"
+
+          try
+            for op in ops
+              data.snapshot = type.apply data.snapshot, op.op
+              data.v++
+          catch e
+            # This should never happen - it indicates that whats in the
+            # database is invalid.
+            console.error "Op data invalid for #{docName}: #{e.stack}"
+            return callback 'Op data invalid'
+
+        model.emit 'load', docName, data
+        add docName, error, data, committedVersion, ops, dbMeta
+
+  # This makes sure the cache contains a document. If the doc cache doesn't contain
+  # a document, it is loaded from the database and stored.
+  #
+  # Documents are stored so long as either:
+  # - They have been accessed within the past #{PERIOD}
+  # - At least one client has the document open
+  refreshReapingTimeout = (docName) ->
+    doc = docs[docName]
+    return unless doc
+
+    # I want to let the clients list be updated before this is called.
+    process.nextTick ->
+      # This is an awkward way to find out the number of clients on a document. If this
+      # causes performance issues, add a numClients field to the document.
+      #
+      # The first check is because its possible that between refreshReapingTimeout being called and this
+      # event being fired, someone called delete() on the document and hence the doc is something else now.
+      if doc == docs[docName] and
+          doc.eventEmitter.listeners('op').length == 0 and
+          (db or options.forceReaping) and
+          doc.opQueue.busy is false
+
+        clearTimeout doc.reapTimer
+        doc.reapTimer = reapTimer = setTimeout ->
+            tryWriteSnapshot docName, ->
+              # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're
+              # in the middle of applying an operation, don't reap.
+              delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false
+          , options.reapTime
+
+  tryWriteSnapshot = (docName, callback) ->
+    return callback?() unless db
+
+    doc = docs[docName]
+
+    # The doc is closed
+    return callback?() unless doc
+
+    # The document is already saved.
+    return callback?() if doc.committedVersion is doc.v
+
+    return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock
+
+    doc.snapshotWriteLock = true
+
+    options.stats?.writeSnapshot?()
+
+    writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback()
+
+    data =
+      v: doc.v
+      meta: doc.meta
+      snapshot: doc.snapshot
+      # The database doesn't know about object types.
+      type: doc.type.name
+
+    # Commit snapshot.
+    writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) ->
+      doc.snapshotWriteLock = false
+
+      # We have to use data.v here because the version in the doc could
+      # have been updated between the call to writeSnapshot() and now.
+      doc.committedVersion = data.v
+      doc.dbMeta = dbMeta
+
+      callback? error
+
+  # *** Model interface methods
+
+  # Create a new document.
+  #
+  # data should be {snapshot, type, [meta]}. The version of a new document is 0.
+  @create = (docName, type, meta, callback) ->
+    [meta, callback] = [{}, meta] if typeof meta is 'function'
+
+    return callback? 'Invalid document name' if docName.match /\//
+    return callback? 'Document already exists' if docs[docName]
+
+    type = types[type] if typeof type == 'string'
+    return callback? 'Type not found' unless type
+
+    data =
+      snapshot:type.create()
+      type:type.name
+      meta:meta or {}
+      v:0
+
+    done = (error, dbMeta) ->
+      # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something.
+      return callback? error if error
+
+      # From here on we'll store the object version of the type name.
+      data.type = type
+      add docName, null, data, 0, [], dbMeta
+      model.emit 'create', docName, data
+      callback?()
+
+    if db
+      db.create docName, data, done
+    else
+      done()
+
+  # Perminantly deletes the specified document.
+  # If listeners are attached, they are removed.
+  # 
+  # The callback is called with (error) if there was an error. If error is null / undefined, the
+  # document was deleted.
+  #
+  # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the
+  # deletion. Subsequent op submissions will fail).
+  @delete = (docName, callback) ->
+    doc = docs[docName]
+
+    if doc
+      clearTimeout doc.reapTimer
+      delete docs[docName]
+
+    done = (error) ->
+      model.emit 'delete', docName unless error
+      callback? error
+
+    if db
+      db.delete docName, doc?.dbMeta, done
+    else
+      done (if !doc then 'Document does not exist')
+
+  # This gets all operations from [start...end]. (That is, its not inclusive.)
+  #
+  # end can be null. This means 'get me all ops from start'.
+  #
+  # Each op returned is in the form {op:o, meta:m, v:version}.
+  #
+  # Callback is called with (error, [ops])
+  #
+  # If the document does not exist, getOps doesn't necessarily return an error. This is because
+  # its awkward to figure out whether or not the document exists for things
+  # like the redis database backend. I guess its a bit gross having this inconsistant
+  # with the other DB calls, but its certainly convenient.
+  #
+  # Use getVersion() to determine if a document actually exists, if thats what you're
+  # after.
+  @getOps = getOps = (docName, start, end, callback) ->
+    # getOps will only use the op cache if its there. It won't fill the op cache in.
+    throw new Error 'start must be 0+' unless start >= 0
+
+    [end, callback] = [null, end] if typeof end is 'function'
+
+    ops = docs[docName]?.ops
+
+    if ops
+      version = docs[docName].v
+
+      # Ops contains an array of ops. The last op in the list is the last op applied
+      end ?= version
+      start = Math.min start, end
+
+      return callback null, [] if start == end
+
+      # Base is the version number of the oldest op we have cached
+      base = version - ops.length
+
+      # If the database is null, we'll trim to the ops we do have and hope thats enough.
+      if start >= base or db is null
+        refreshReapingTimeout docName
+        options.stats?.cacheHit 'getOps'
+
+        return callback null, ops[(start - base)...(end - base)]
+
+    options.stats?.cacheMiss 'getOps'
+
+    getOpsInternal docName, start, end, callback
+
+  # Gets the snapshot data for the specified document.
+  # getSnapshot(docName, callback)
+  # Callback is called with (error, {v: <version>, type: <type>, snapshot: <snapshot>, meta: <meta>})
+  @getSnapshot = (docName, callback) ->
+    load docName, (error, doc) ->
+      callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta}
+
+  # Gets the latest version # of the document.
+  # getVersion(docName, callback)
+  # callback is called with (error, version).
+  @getVersion = (docName, callback) ->
+    load docName, (error, doc) -> callback error, doc?.v
+
+  # Apply an op to the specified document.
+  # The callback is passed (error, applied version #)
+  # opData = {op:op, v:v, meta:metadata}
+  # 
+  # Ops are queued before being applied so that the following code applies op C before op B:
+  # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB
+  # model.applyOp 'doc', OPC
+  @applyOp = (docName, opData, callback) ->
+    # All the logic for this is in makeOpQueue, above.
+    load docName, (error, doc) ->
+      return callback error if error
+
+      process.nextTick -> doc.opQueue opData, (error, newVersion) ->
+        refreshReapingTimeout docName
+        callback? error, newVersion
+
+  # TODO: store (some) metadata in DB
+  # TODO: op and meta should be combineable in the op that gets sent
+  @applyMetaOp = (docName, metaOpData, callback) ->
+    {path, value} = metaOpData.meta
+   
+    return callback? "path should be an array" unless isArray path
+
+    load docName, (error, doc) ->
+      if error?
+        callback? error
+      else
+        applied = false
+        switch path[0]
+          when 'shout'
+            doc.eventEmitter.emit 'op', metaOpData
+            applied = true
+
+        model.emit 'applyMetaOp', docName, path, value if applied
+        callback? null, doc.v
+
+  # Listen to all ops from the specified version. If version is in the past, all
+  # ops since that version are sent immediately to the listener.
+  #
+  # The callback is called once the listener is attached, but before any ops have been passed
+  # to the listener.
+  # 
+  # This will _not_ edit the document metadata.
+  #
+  # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour
+  # might change in a future version.
+  #
+  # version is the document version at which the document is opened. It can be left out if you want to open
+  # the document at the most recent version.
+  #
+  # listener is called with (opData) each time an op is applied.
+  #
+  # callback(error, openedVersion)
+  @listen = (docName, version, listener, callback) ->
+    [version, listener, callback] = [null, version, listener] if typeof version is 'function'
+
+    load docName, (error, doc) ->
+      return callback? error if error
+
+      clearTimeout doc.reapTimer
+
+      if version?
+        getOps docName, version, null, (error, data) ->
+          return callback? error if error
+
+          doc.eventEmitter.on 'op', listener
+          callback? null, version
+          for op in data
+            listener op
+
+            # The listener may well remove itself during the catchup phase. If this happens, break early.
+            # This is done in a quite inefficient way. (O(n) where n = #listeners on doc)
+            break unless listener in doc.eventEmitter.listeners 'op'
+
+      else # Version is null / undefined. Just add the listener.
+        doc.eventEmitter.on 'op', listener
+        callback? null, doc.v
+
+  # Remove a listener for a particular document.
+  #
+  # removeListener(docName, listener)
+  #
+  # This is synchronous.
+  @removeListener = (docName, listener) ->
+    # The document should already be loaded.
+    doc = docs[docName]
+    throw new Error 'removeListener called but document not loaded' unless doc
+
+    doc.eventEmitter.removeListener 'op', listener
+    refreshReapingTimeout docName
+
+  # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed -
+  # sharejs will happily replay uncommitted ops when documents are re-opened anyway.
+  @flush = (callback) ->
+    return callback?() unless db
+
+    pendingWrites = 0
+
+    for docName, doc of docs
+      if doc.committedVersion < doc.v
+        pendingWrites++
+        # I'm hoping writeSnapshot will always happen in another thread.
+        tryWriteSnapshot docName, ->
+          process.nextTick ->
+            pendingWrites--
+            callback?() if pendingWrites is 0
+
+    # If nothing was queued, terminate immediately.
+    callback?() if pendingWrites is 0
+
+  # Close the database connection. This is needed so nodejs can shut down cleanly.
+  @closeDb = ->
+    db?.close?()
+    db = null
+
+  return
+
+# Model inherits from EventEmitter.
+Model:: = new EventEmitter
+

services/document-updater/app/coffee/sharejs/server/model.coffee

@@ -0,0 +1,603 @@
+# The model of all the ops. Responsible for applying & transforming remote deltas
+# and managing the storage layer.
+#
+# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
+
+{EventEmitter} = require 'events'
+
+queue = require './syncqueue'
+types = require '../types'
+
+isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]'
+
+# This constructor creates a new Model object. There will be one model object
+# per server context.
+#
+# The model object is responsible for a lot of things:
+#
+# - It manages the interactions with the database
+# - It maintains (in memory) a set of all active documents
+# - It calls out to the OT functions when necessary
+#
+# The model is an event emitter. It emits the following events:
+#
+# create(docName, data): A document has been created with the specified name & data
+module.exports = Model = (db, options) ->
+  # db can be null if the user doesn't want persistance.
+
+  return new Model(db, options) if !(this instanceof Model)
+
+  model = this
+
+  options ?= {}
+
+  # This is a cache of 'live' documents.
+  #
+  # The cache is a map from docName -> {
+  #   ops:[{op, meta}]
+  #   snapshot
+  #   type
+  #   v
+  #   meta
+  #   eventEmitter
+  #   reapTimer
+  #   committedVersion: v
+  #   snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant
+  #   dbMeta: database specific data
+  #   opQueue: syncQueue for processing ops
+  # }
+  #
+  # The ops list contains the document's last options.numCachedOps ops. (Or all
+  # of them if we're using a memory store).
+  #
+  # Documents are stored in this set so long as the document has been accessed in
+  # the last few seconds (options.reapTime) OR at least one client has the document
+  # open. I don't know if I should keep open (but not being edited) documents live -
+  # maybe if a client has a document open but the document isn't being edited, I should
+  # flush it from the cache.
+  #
+  # In any case, the API to model is designed such that if we want to change that later
+  # it should be pretty easy to do so without any external-to-the-model code changes.
+  docs = {}
+
+  # This is a map from docName -> [callback]. It is used when a document hasn't been
+  # cached and multiple getSnapshot() / getVersion() requests come in. All requests
+  # are added to the callback list and called when db.getSnapshot() returns.
+  #
+  # callback(error, snapshot data)
+  awaitingGetSnapshot = {}
+
+  # The time that documents which no clients have open will stay in the cache.
+  # Should be > 0.
+  options.reapTime ?= 3000
+
+  # The number of operations the cache holds before reusing the space
+  options.numCachedOps ?= 10
+
+  # This option forces documents to be reaped, even when there's no database backend.
+  # This is useful when you don't care about persistance and don't want to gradually
+  # fill memory.
+  #
+  # You might want to set reapTime to a day or something.
+  options.forceReaping ?= false
+
+  # Until I come up with a better strategy, we'll save a copy of the document snapshot
+  # to the database every ~20 submitted ops.
+  options.opsBeforeCommit ?= 20
+
+  # It takes some processing time to transform client ops. The server will punt ops back to the
+  # client to transform if they're too old.
+  options.maximumAge ?= 40
+
+  # **** Cache API methods
+
+  # Its important that all ops are applied in order. This helper method creates the op submission queue
+  # for a single document. This contains the logic for transforming & applying ops.
+  makeOpQueue = (docName, doc) -> queue (opData, callback) ->
+    return callback 'Version missing' unless opData.v >= 0
+    return callback 'Op at future version' if opData.v > doc.v
+
+    # Punt the transforming work back to the client if the op is too old.
+    return callback 'Op too old' if opData.v + options.maximumAge < doc.v
+
+    opData.meta ||= {}
+    opData.meta.ts = Date.now()
+
+    # We'll need to transform the op to the current version of the document. This
+    # calls the callback immediately if opVersion == doc.v.
+    getOps docName, opData.v, doc.v, (error, ops) ->
+      return callback error if error
+
+      unless doc.v - opData.v == ops.length
+        # This should never happen. It indicates that we didn't get all the ops we
+        # asked for. Its important that the submitted op is correctly transformed.
+        console.error "Could not get old ops in model for document #{docName}"
+        console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops"
+        return callback 'Internal error'
+
+      if ops.length > 0
+        try
+          # If there's enough ops, it might be worth spinning this out into a webworker thread.
+          for oldOp in ops
+            # Dup detection works by sending the id(s) the op has been submitted with previously.
+            # If the id matches, we reject it. The client can also detect the op has been submitted
+            # already if it sees its own previous id in the ops it sees when it does catchup.
+            if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource
+              return callback 'Op already submitted'
+
+            opData.op = doc.type.transform opData.op, oldOp.op, 'left'
+            opData.v++
+        catch error
+          console.error error.stack
+          return callback error.message
+
+      try
+        snapshot = doc.type.apply doc.snapshot, opData.op
+      catch error
+        console.error error.stack
+        return callback error.message
+
+      # The op data should be at the current version, and the new document data should be at
+      # the next version.
+      #
+      # This should never happen in practice, but its a nice little check to make sure everything
+      # is hunky-dory.
+      unless opData.v == doc.v
+        # This should never happen.
+        console.error "Version mismatch detected in model. File a ticket - this is a bug."
+        console.error "Expecting #{opData.v} == #{doc.v}"
+        return callback 'Internal error'
+
+      #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta}
+      writeOp = db?.writeOp or (docName, newOpData, callback) -> callback()
+
+      writeOp docName, opData, (error) ->
+        if error
+          # The user should probably know about this.
+          console.warn "Error writing ops to database: #{error}"
+          return callback error
+
+        options.stats?.writeOp?()
+
+        # This is needed when we emit the 'change' event, below.
+        oldSnapshot = doc.snapshot
+
+        # All the heavy lifting is now done. Finally, we'll update the cache with the new data
+        # and (maybe!) save a new document snapshot to the database.
+
+        doc.v = opData.v + 1
+        doc.snapshot = snapshot
+
+        doc.ops.push opData
+        doc.ops.shift() if db and doc.ops.length > options.numCachedOps
+
+        model.emit 'applyOp', docName, opData, snapshot, oldSnapshot
+        doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot
+
+        # The callback is called with the version of the document at which the op was applied.
+        # This is the op.v after transformation, and its doc.v - 1.
+        callback null, opData.v
+    
+        # I need a decent strategy here for deciding whether or not to save the snapshot.
+        #
+        # The 'right' strategy looks something like "Store the snapshot whenever the snapshot
+        # is smaller than the accumulated op data". For now, I'll just store it every 20
+        # ops or something. (Configurable with doc.committedVersion)
+        if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v
+          tryWriteSnapshot docName, (error) ->
+            console.warn "Error writing snapshot #{error}. This is nonfatal" if error
+
+  # Add the data for the given docName to the cache. The named document shouldn't already
+  # exist in the doc set.
+  #
+  # Returns the new doc.
+  add = (docName, error, data, committedVersion, ops, dbMeta) ->
+    callbacks = awaitingGetSnapshot[docName]
+    delete awaitingGetSnapshot[docName]
+
+    if error
+      callback error for callback in callbacks if callbacks
+    else
+      doc = docs[docName] =
+        snapshot: data.snapshot
+        v: data.v
+        type: data.type
+        meta: data.meta
+
+        # Cache of ops
+        ops: ops or []
+
+        eventEmitter: new EventEmitter
+
+        # Timer before the document will be invalidated from the cache (if the document has no
+        # listeners)
+        reapTimer: null
+
+        # Version of the snapshot thats in the database
+        committedVersion: committedVersion ? data.v
+        snapshotWriteLock: false
+        dbMeta: dbMeta
+
+      doc.opQueue = makeOpQueue docName, doc
+      
+      refreshReapingTimeout docName
+      model.emit 'add', docName, data
+      callback null, doc for callback in callbacks if callbacks
+
+    doc
+  
+  # This is a little helper wrapper around db.getOps. It does two things:
+  #
+  # - If there's no database set, it returns an error to the callback
+  # - It adds version numbers to each op returned from the database
+  # (These can be inferred from context so the DB doesn't store them, but its useful to have them).
+  getOpsInternal = (docName, start, end, callback) ->
+    return callback? 'Document does not exist' unless db
+
+    db.getOps docName, start, end, (error, ops) ->
+      return callback? error if error
+
+      v = start
+      op.v = v++ for op in ops
+
+      callback? null, ops
+
+  # Load the named document into the cache. This function is re-entrant.
+  #
+  # The callback is called with (error, doc)
+  load = (docName, callback) ->
+    if docs[docName]
+      # The document is already loaded. Return immediately.
+      options.stats?.cacheHit? 'getSnapshot'
+      return callback null, docs[docName]
+
+    # We're a memory store. If we don't have it, nobody does.
+    return callback 'Document does not exist' unless db
+
+    callbacks = awaitingGetSnapshot[docName]
+
+    # The document is being loaded already. Add ourselves as a callback.
+    return callbacks.push callback if callbacks
+
+    options.stats?.cacheMiss? 'getSnapshot'
+
+    # The document isn't loaded and isn't being loaded. Load it.
+    awaitingGetSnapshot[docName] = [callback]
+    db.getSnapshot docName, (error, data, dbMeta) ->
+      return add docName, error if error
+
+      type = types[data.type]
+      unless type
+        console.warn "Type '#{data.type}' missing"
+        return callback "Type not found"
+      data.type = type
+
+      committedVersion = data.v
+
+      # The server can close without saving the most recent document snapshot.
+      # In this case, there are extra ops which need to be applied before
+      # returning the snapshot.
+      getOpsInternal docName, data.v, null, (error, ops) ->
+        return callback error if error
+
+        if ops.length > 0
+          console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}"
+
+          try
+            for op in ops
+              data.snapshot = type.apply data.snapshot, op.op
+              data.v++
+          catch e
+            # This should never happen - it indicates that whats in the
+            # database is invalid.
+            console.error "Op data invalid for #{docName}: #{e.stack}"
+            return callback 'Op data invalid'
+
+        model.emit 'load', docName, data
+        add docName, error, data, committedVersion, ops, dbMeta
+
+  # This makes sure the cache contains a document. If the doc cache doesn't contain
+  # a document, it is loaded from the database and stored.
+  #
+  # Documents are stored so long as either:
+  # - They have been accessed within the past #{PERIOD}
+  # - At least one client has the document open
+  refreshReapingTimeout = (docName) ->
+    doc = docs[docName]
+    return unless doc
+
+    # I want to let the clients list be updated before this is called.
+    process.nextTick ->
+      # This is an awkward way to find out the number of clients on a document. If this
+      # causes performance issues, add a numClients field to the document.
+      #
+      # The first check is because its possible that between refreshReapingTimeout being called and this
+      # event being fired, someone called delete() on the document and hence the doc is something else now.
+      if doc == docs[docName] and
+          doc.eventEmitter.listeners('op').length == 0 and
+          (db or options.forceReaping) and
+          doc.opQueue.busy is false
+
+        clearTimeout doc.reapTimer
+        doc.reapTimer = reapTimer = setTimeout ->
+            tryWriteSnapshot docName, ->
+              # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're
+              # in the middle of applying an operation, don't reap.
+              delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false
+          , options.reapTime
+
+  tryWriteSnapshot = (docName, callback) ->
+    return callback?() unless db
+
+    doc = docs[docName]
+
+    # The doc is closed
+    return callback?() unless doc
+
+    # The document is already saved.
+    return callback?() if doc.committedVersion is doc.v
+
+    return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock
+
+    doc.snapshotWriteLock = true
+
+    options.stats?.writeSnapshot?()
+
+    writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback()
+
+    data =
+      v: doc.v
+      meta: doc.meta
+      snapshot: doc.snapshot
+      # The database doesn't know about object types.
+      type: doc.type.name
+
+    # Commit snapshot.
+    writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) ->
+      doc.snapshotWriteLock = false
+
+      # We have to use data.v here because the version in the doc could
+      # have been updated between the call to writeSnapshot() and now.
+      doc.committedVersion = data.v
+      doc.dbMeta = dbMeta
+
+      callback? error
+
+  # *** Model interface methods
+
+  # Create a new document.
+  #
+  # data should be {snapshot, type, [meta]}. The version of a new document is 0.
+  @create = (docName, type, meta, callback) ->
+    [meta, callback] = [{}, meta] if typeof meta is 'function'
+
+    return callback? 'Invalid document name' if docName.match /\//
+    return callback? 'Document already exists' if docs[docName]
+
+    type = types[type] if typeof type == 'string'
+    return callback? 'Type not found' unless type
+
+    data =
+      snapshot:type.create()
+      type:type.name
+      meta:meta or {}
+      v:0
+
+    done = (error, dbMeta) ->
+      # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something.
+      return callback? error if error
+
+      # From here on we'll store the object version of the type name.
+      data.type = type
+      add docName, null, data, 0, [], dbMeta
+      model.emit 'create', docName, data
+      callback?()
+
+    if db
+      db.create docName, data, done
+    else
+      done()
+
+  # Perminantly deletes the specified document.
+  # If listeners are attached, they are removed.
+  # 
+  # The callback is called with (error) if there was an error. If error is null / undefined, the
+  # document was deleted.
+  #
+  # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the
+  # deletion. Subsequent op submissions will fail).
+  @delete = (docName, callback) ->
+    doc = docs[docName]
+
+    if doc
+      clearTimeout doc.reapTimer
+      delete docs[docName]
+
+    done = (error) ->
+      model.emit 'delete', docName unless error
+      callback? error
+
+    if db
+      db.delete docName, doc?.dbMeta, done
+    else
+      done (if !doc then 'Document does not exist')
+
+  # This gets all operations from [start...end]. (That is, its not inclusive.)
+  #
+  # end can be null. This means 'get me all ops from start'.
+  #
+  # Each op returned is in the form {op:o, meta:m, v:version}.
+  #
+  # Callback is called with (error, [ops])
+  #
+  # If the document does not exist, getOps doesn't necessarily return an error. This is because
+  # its awkward to figure out whether or not the document exists for things
+  # like the redis database backend. I guess its a bit gross having this inconsistant
+  # with the other DB calls, but its certainly convenient.
+  #
+  # Use getVersion() to determine if a document actually exists, if thats what you're
+  # after.
+  @getOps = getOps = (docName, start, end, callback) ->
+    # getOps will only use the op cache if its there. It won't fill the op cache in.
+    throw new Error 'start must be 0+' unless start >= 0
+
+    [end, callback] = [null, end] if typeof end is 'function'
+
+    ops = docs[docName]?.ops
+
+    if ops
+      version = docs[docName].v
+
+      # Ops contains an array of ops. The last op in the list is the last op applied
+      end ?= version
+      start = Math.min start, end
+
+      return callback null, [] if start == end
+
+      # Base is the version number of the oldest op we have cached
+      base = version - ops.length
+
+      # If the database is null, we'll trim to the ops we do have and hope thats enough.
+      if start >= base or db is null
+        refreshReapingTimeout docName
+        options.stats?.cacheHit 'getOps'
+
+        return callback null, ops[(start - base)...(end - base)]
+
+    options.stats?.cacheMiss 'getOps'
+
+    getOpsInternal docName, start, end, callback
+
+  # Gets the snapshot data for the specified document.
+  # getSnapshot(docName, callback)
+  # Callback is called with (error, {v: <version>, type: <type>, snapshot: <snapshot>, meta: <meta>})
+  @getSnapshot = (docName, callback) ->
+    load docName, (error, doc) ->
+      callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta}
+
+  # Gets the latest version # of the document.
+  # getVersion(docName, callback)
+  # callback is called with (error, version).
+  @getVersion = (docName, callback) ->
+    load docName, (error, doc) -> callback error, doc?.v
+
+  # Apply an op to the specified document.
+  # The callback is passed (error, applied version #)
+  # opData = {op:op, v:v, meta:metadata}
+  # 
+  # Ops are queued before being applied so that the following code applies op C before op B:
+  # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB
+  # model.applyOp 'doc', OPC
+  @applyOp = (docName, opData, callback) ->
+    # All the logic for this is in makeOpQueue, above.
+    load docName, (error, doc) ->
+      return callback error if error
+
+      process.nextTick -> doc.opQueue opData, (error, newVersion) ->
+        refreshReapingTimeout docName
+        callback? error, newVersion
+
+  # TODO: store (some) metadata in DB
+  # TODO: op and meta should be combineable in the op that gets sent
+  @applyMetaOp = (docName, metaOpData, callback) ->
+    {path, value} = metaOpData.meta
+   
+    return callback? "path should be an array" unless isArray path
+
+    load docName, (error, doc) ->
+      if error?
+        callback? error
+      else
+        applied = false
+        switch path[0]
+          when 'shout'
+            doc.eventEmitter.emit 'op', metaOpData
+            applied = true
+
+        model.emit 'applyMetaOp', docName, path, value if applied
+        callback? null, doc.v
+
+  # Listen to all ops from the specified version. If version is in the past, all
+  # ops since that version are sent immediately to the listener.
+  #
+  # The callback is called once the listener is attached, but before any ops have been passed
+  # to the listener.
+  # 
+  # This will _not_ edit the document metadata.
+  #
+  # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour
+  # might change in a future version.
+  #
+  # version is the document version at which the document is opened. It can be left out if you want to open
+  # the document at the most recent version.
+  #
+  # listener is called with (opData) each time an op is applied.
+  #
+  # callback(error, openedVersion)
+  @listen = (docName, version, listener, callback) ->
+    [version, listener, callback] = [null, version, listener] if typeof version is 'function'
+
+    load docName, (error, doc) ->
+      return callback? error if error
+
+      clearTimeout doc.reapTimer
+
+      if version?
+        getOps docName, version, null, (error, data) ->
+          return callback? error if error
+
+          doc.eventEmitter.on 'op', listener
+          callback? null, version
+          for op in data
+            listener op
+
+            # The listener may well remove itself during the catchup phase. If this happens, break early.
+            # This is done in a quite inefficient way. (O(n) where n = #listeners on doc)
+            break unless listener in doc.eventEmitter.listeners 'op'
+
+      else # Version is null / undefined. Just add the listener.
+        doc.eventEmitter.on 'op', listener
+        callback? null, doc.v
+
+  # Remove a listener for a particular document.
+  #
+  # removeListener(docName, listener)
+  #
+  # This is synchronous.
+  @removeListener = (docName, listener) ->
+    # The document should already be loaded.
+    doc = docs[docName]
+    throw new Error 'removeListener called but document not loaded' unless doc
+
+    doc.eventEmitter.removeListener 'op', listener
+    refreshReapingTimeout docName
+
+  # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed -
+  # sharejs will happily replay uncommitted ops when documents are re-opened anyway.
+  @flush = (callback) ->
+    return callback?() unless db
+
+    pendingWrites = 0
+
+    for docName, doc of docs
+      if doc.committedVersion < doc.v
+        pendingWrites++
+        # I'm hoping writeSnapshot will always happen in another thread.
+        tryWriteSnapshot docName, ->
+          process.nextTick ->
+            pendingWrites--
+            callback?() if pendingWrites is 0
+
+    # If nothing was queued, terminate immediately.
+    callback?() if pendingWrites is 0
+
+  # Close the database connection. This is needed so nodejs can shut down cleanly.
+  @closeDb = ->
+    db?.close?()
+    db = null
+
+  return
+
+# Model inherits from EventEmitter.
+Model:: = new EventEmitter
+

services/document-updater/app/coffee/sharejs/server/syncqueue.coffee

@@ -0,0 +1,42 @@
+# A synchronous processing queue. The queue calls process on the arguments,
+# ensuring that process() is only executing once at a time.
+#
+# process(data, callback) _MUST_ eventually call its callback.
+#
+# Example:
+#
+# queue = require 'syncqueue'
+#
+# fn = queue (data, callback) ->
+#     asyncthing data, ->
+#         callback(321)
+#
+# fn(1)
+# fn(2)
+# fn(3, (result) -> console.log(result))
+#
+#   ^--- async thing will only be running once at any time.
+
+module.exports = (process) ->
+  throw new Error('process is not a function') unless typeof process == 'function'
+  queue = []
+  
+  enqueue = (data, callback) ->
+    queue.push [data, callback]
+    flush()
+
+  enqueue.busy = false
+
+  flush = ->
+    return if enqueue.busy or queue.length == 0
+
+    enqueue.busy = true
+    [data, callback] = queue.shift()
+    process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow.
+      enqueue.busy = false
+      # This is called after busy = false so a user can check if enqueue.busy is set in the callback.
+      callback.apply null, result if callback
+      flush()
+
+  enqueue
+

services/document-updater/app/coffee/sharejs/simple.coffee

@@ -0,0 +1,38 @@
+# This is a really simple OT type. Its not compiled with the web client, but it could be.
+#
+# Its mostly included for demonstration purposes and its used in a lot of unit tests.
+#
+# This defines a really simple text OT type which only allows inserts. (No deletes).
+#
+# Ops look like:
+#   {position:#, text:"asdf"}
+#
+# Document snapshots look like:
+#   {str:string}
+
+module.exports =
+  # The name of the OT type. The type is stored in types[type.name]. The name can be
+  # used in place of the actual type in all the API methods.
+  name: 'simple'
+
+  # Create a new document snapshot
+  create: -> {str:""}
+
+  # Apply the given op to the document snapshot. Returns the new snapshot.
+  #
+  # The original snapshot should not be modified.
+  apply: (snapshot, op) ->
+    throw new Error 'Invalid position' unless 0 <= op.position <= snapshot.str.length
+
+    str = snapshot.str
+    str = str.slice(0, op.position) + op.text + str.slice(op.position)
+    {str}
+
+  # transform op1 by op2. Return transformed version of op1.
+  # sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the
+  # op being transformed comes from the client or the server.
+  transform: (op1, op2, sym) ->
+    pos = op1.position
+    pos += op2.text.length if op2.position < pos or (op2.position == pos and sym is 'left')
+
+    return {position:pos, text:op1.text}

services/document-updater/app/coffee/sharejs/syncqueue.coffee

@@ -0,0 +1,42 @@
+# A synchronous processing queue. The queue calls process on the arguments,
+# ensuring that process() is only executing once at a time.
+#
+# process(data, callback) _MUST_ eventually call its callback.
+#
+# Example:
+#
+# queue = require 'syncqueue'
+#
+# fn = queue (data, callback) ->
+#     asyncthing data, ->
+#         callback(321)
+#
+# fn(1)
+# fn(2)
+# fn(3, (result) -> console.log(result))
+#
+#   ^--- async thing will only be running once at any time.
+
+module.exports = (process) ->
+  throw new Error('process is not a function') unless typeof process == 'function'
+  queue = []
+  
+  enqueue = (data, callback) ->
+    queue.push [data, callback]
+    flush()
+
+  enqueue.busy = false
+
+  flush = ->
+    return if enqueue.busy or queue.length == 0
+
+    enqueue.busy = true
+    [data, callback] = queue.shift()
+    process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow.
+      enqueue.busy = false
+      # This is called after busy = false so a user can check if enqueue.busy is set in the callback.
+      callback.apply null, result if callback
+      flush()
+
+  enqueue
+

services/document-updater/app/coffee/sharejs/text-api.coffee

@@ -0,0 +1,32 @@
+# Text document API for text
+
+text = require './text' if typeof WEB is 'undefined'
+
+text.api =
+  provides: {text:true}
+
+  # The number of characters in the string
+  getLength: -> @snapshot.length
+
+  # Get the text contents of a document
+  getText: -> @snapshot
+
+  insert: (pos, text, callback) ->
+    op = [{p:pos, i:text}]
+    
+    @submitOp op, callback
+    op
+  
+  del: (pos, length, callback) ->
+    op = [{p:pos, d:@snapshot[pos...(pos + length)]}]
+
+    @submitOp op, callback
+    op
+  
+  _register: ->
+    @on 'remoteop', (op) ->
+      for component in op
+        if component.i != undefined
+          @emit 'insert', component.p, component.i
+        else
+          @emit 'delete', component.p, component.d

services/document-updater/app/coffee/sharejs/text-composable-api.coffee

@@ -0,0 +1,43 @@
+# Text document API for text
+
+if WEB?
+  type = exports.types['text-composable']
+else
+  type = require './text-composable'
+
+type.api =
+  provides: {'text':true}
+
+  # The number of characters in the string
+  'getLength': -> @snapshot.length
+
+  # Get the text contents of a document
+  'getText': -> @snapshot
+
+  'insert': (pos, text, callback) ->
+    op = type.normalize [pos, 'i':text, (@snapshot.length - pos)]
+    
+    @submitOp op, callback
+    op
+  
+  'del': (pos, length, callback) ->
+    op = type.normalize [pos, 'd':@snapshot[pos...(pos + length)], (@snapshot.length - pos - length)]
+
+    @submitOp op, callback
+    op
+
+  _register: ->
+    @on 'remoteop', (op) ->
+      pos = 0
+      for component in op
+        if typeof component is 'number'
+          pos += component
+        else if component.i != undefined
+          @emit 'insert', pos, component.i
+          pos += component.i.length
+        else
+          # delete
+          @emit 'delete', pos, component.d
+          # We don't increment pos, because the position
+          # specified is after the delete has happened.
+

services/document-updater/app/coffee/sharejs/text-composable.coffee

@@ -0,0 +1,261 @@
+# An alternate composable implementation for text. This is much closer
+# to the implementation used by google wave.
+#
+# Ops are lists of components which iterate over the whole document.
+# Components are either:
+#   A number N: Skip N characters in the original document
+#   {i:'str'}:  Insert 'str' at the current position in the document
+#   {d:'str'}:  Delete 'str', which appears at the current position in the document
+#
+# Eg: [3, {i:'hi'}, 5, {d:'internet'}]
+#
+# Snapshots are strings.
+
+p = -> #require('util').debug
+i = -> #require('util').inspect
+
+exports = if WEB? then {} else module.exports
+
+exports.name = 'text-composable'
+
+exports.create = -> ''
+
+# -------- Utility methods
+
+checkOp = (op) ->
+  throw new Error('Op must be an array of components') unless Array.isArray(op)
+  last = null
+  for c in op
+    if typeof(c) == 'object'
+      throw new Error("Invalid op component: #{i c}") unless (c.i? && c.i.length > 0) or (c.d? && c.d.length > 0)
+    else
+      throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number'
+      throw new Error('Skip components must be a positive number') unless c > 0
+      throw new Error('Adjacent skip components should be added') if typeof(last) == 'number'
+
+    last = c
+
+# Makes a function for appending components to a given op.
+# Exported for the randomOpGenerator.
+exports._makeAppend = makeAppend = (op) -> (component) ->
+  if component == 0 || component.i == '' || component.d == ''
+    return
+  else if op.length == 0
+    op.push component
+  else if typeof(component) == 'number' && typeof(op[op.length - 1]) == 'number'
+    op[op.length - 1] += component
+  else if component.i? && op[op.length - 1].i?
+    op[op.length - 1].i += component.i
+  else if component.d? && op[op.length - 1].d?
+    op[op.length - 1].d += component.d
+  else
+    op.push component
+  
+#  checkOp op
+
+# Makes 2 functions for taking components from the start of an op, and for peeking
+# at the next op that could be taken.
+makeTake = (op) ->
+  # The index of the next component to take
+  idx = 0
+  # The offset into the component
+  offset = 0
+
+  # Take up to length n from the front of op. If n is null, take the next
+  # op component. If indivisableField == 'd', delete components won't be separated.
+  # If indivisableField == 'i', insert components won't be separated.
+  take = (n, indivisableField) ->
+    return null if idx == op.length
+    #assert.notStrictEqual op.length, i, 'The op is too short to traverse the document'
+
+    if typeof(op[idx]) == 'number'
+      if !n? or op[idx] - offset <= n
+        c = op[idx] - offset
+        ++idx; offset = 0
+        c
+      else
+        offset += n
+        n
+    else
+      # Take from the string
+      field = if op[idx].i then 'i' else 'd'
+      c = {}
+      if !n? or op[idx][field].length - offset <= n or field == indivisableField
+        c[field] = op[idx][field][offset..]
+        ++idx; offset = 0
+      else
+        c[field] = op[idx][field][offset...(offset + n)]
+        offset += n
+      c
+  
+  peekType = () ->
+    op[idx]
+  
+  [take, peekType]
+
+# Find and return the length of an op component
+componentLength = (component) ->
+  if typeof(component) == 'number'
+    component
+  else if component.i?
+    component.i.length
+  else
+    component.d.length
+
+# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate
+# adjacent inserts and deletes.
+exports.normalize = (op) ->
+  newOp = []
+  append = makeAppend newOp
+  append component for component in op
+  newOp
+
+# Apply the op to the string. Returns the new string.
+exports.apply = (str, op) ->
+  p "Applying #{i op} to '#{str}'"
+  throw new Error('Snapshot should be a string') unless typeof(str) == 'string'
+  checkOp op
+
+  pos = 0
+  newDoc = []
+
+  for component in op
+    if typeof(component) == 'number'
+      throw new Error('The op is too long for this document') if component > str.length
+      newDoc.push str[...component]
+      str = str[component..]
+    else if component.i?
+      newDoc.push component.i
+    else
+      throw new Error("The deleted text '#{component.d}' doesn't match the next characters in the document '#{str[...component.d.length]}'") unless component.d == str[...component.d.length]
+      str = str[component.d.length..]
+  
+  throw new Error("The applied op doesn't traverse the entire document") unless '' == str
+
+  newDoc.join ''
+
+# transform op1 by op2. Return transformed version of op1.
+# op1 and op2 are unchanged by transform.
+exports.transform = (op, otherOp, side) ->
+  throw new Error "side (#{side} must be 'left' or 'right'" unless side == 'left' or side == 'right'
+
+  checkOp op
+  checkOp otherOp
+  newOp = []
+
+  append = makeAppend newOp
+  [take, peek] = makeTake op
+
+  for component in otherOp
+    if typeof(component) == 'number' # Skip
+      length = component
+      while length > 0
+        chunk = take(length, 'i')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append chunk
+        length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.i?
+    else if component.i? # Insert
+      if side == 'left'
+        # The left insert should go first.
+        o = peek()
+        append take() if o?.i
+
+      # Otherwise, skip the inserted text.
+      append(component.i.length)
+    else # Delete.
+      #assert.ok component.d
+      length = component.d.length
+      while length > 0
+        chunk = take(length, 'i')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        if typeof(chunk) == 'number'
+          length -= chunk
+        else if chunk.i?
+          append(chunk)
+        else
+          #assert.ok chunk.d
+          # The delete is unnecessary now.
+          length -= chunk.d.length
+  
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in the op: #{i component}" unless component?.i?
+    append component
+
+  newOp
+
+
+# Compose 2 ops into 1 op.
+exports.compose = (op1, op2) ->
+  p "COMPOSE #{i op1} + #{i op2}"
+  checkOp op1
+  checkOp op2
+
+  result = []
+
+  append = makeAppend result
+  [take, _] = makeTake op1
+
+  for component in op2
+    if typeof(component) == 'number' # Skip
+      length = component
+      while length > 0
+        chunk = take(length, 'd')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append chunk
+        length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.d?
+
+    else if component.i? # Insert
+      append {i:component.i}
+
+    else # Delete
+      offset = 0
+      while offset < component.d.length
+        chunk = take(component.d.length - offset, 'd')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        # If its delete, append it. If its skip, drop it and decrease length. If its insert, check the strings match, drop it and decrease length.
+        if typeof(chunk) == 'number'
+          append {d:component.d[offset...(offset + chunk)]}
+          offset += chunk
+        else if chunk.i?
+          throw new Error("The deleted text doesn't match the inserted text") unless component.d[offset...(offset + chunk.i.length)] == chunk.i
+          offset += chunk.i.length
+          # The ops cancel each other out.
+        else
+          # Delete
+          append chunk
+    
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Trailing stuff in op1 #{i component}" unless component?.d?
+    append component
+
+  result
+  
+
+invertComponent = (c) ->
+  if typeof(c) == 'number'
+    c
+  else if c.i?
+    {d:c.i}
+  else
+    {i:c.d}
+
+# Invert an op
+exports.invert = (op) ->
+  result = []
+  append = makeAppend result
+
+  append(invertComponent component) for component in op
+  
+  result
+
+if window?
+  window.ot ||= {}
+  window.ot.types ||= {}
+  window.ot.types.text = exports
+

services/document-updater/app/coffee/sharejs/text-tp2-api.coffee

@@ -0,0 +1,89 @@
+# Text document API for text-tp2
+
+if WEB?
+  type = exports.types['text-tp2']
+else
+  type = require './text-tp2'
+
+{_takeDoc:takeDoc, _append:append} = type
+
+appendSkipChars = (op, doc, pos, maxlength) ->
+  while (maxlength == undefined || maxlength > 0) and pos.index < doc.data.length
+    part = takeDoc doc, pos, maxlength, true
+    maxlength -= part.length if maxlength != undefined and typeof part is 'string'
+    append op, (part.length || part)
+
+type['api'] =
+  'provides': {'text':true}
+
+  # The number of characters in the string
+  'getLength': -> @snapshot.charLength
+
+  # Flatten a document into a string
+  'getText': ->
+    strings = (elem for elem in @snapshot.data when typeof elem is 'string')
+    strings.join ''
+
+  'insert': (pos, text, callback) ->
+    pos = 0 if pos == undefined
+
+    op = []
+    docPos = {index:0, offset:0}
+
+    appendSkipChars op, @snapshot, docPos, pos
+    append op, {'i':text}
+    appendSkipChars op, @snapshot, docPos
+    
+    @submitOp op, callback
+    op
+  
+  'del': (pos, length, callback) ->
+    op = []
+    docPos = {index:0, offset:0}
+
+    appendSkipChars op, @snapshot, docPos, pos
+    
+    while length > 0
+      part = takeDoc @snapshot, docPos, length, true
+      if typeof part is 'string'
+        append op, {'d':part.length}
+        length -= part.length
+      else
+        append op, part
+    
+    appendSkipChars op, @snapshot, docPos
+
+    @submitOp op, callback
+    op
+
+  '_register': ->
+    # Interpret recieved ops + generate more detailed events for them
+    @on 'remoteop', (op, snapshot) ->
+      textPos = 0
+      docPos = {index:0, offset:0}
+
+      for component in op
+        if typeof component is 'number'
+          # Skip
+          remainder = component
+          while remainder > 0
+            part = takeDoc snapshot, docPos, remainder
+            if typeof part is 'string'
+              textPos += part.length
+            remainder -= part.length || part
+        else if component.i != undefined
+          # Insert
+          if typeof component.i is 'string'
+            @emit 'insert', textPos, component.i
+            textPos += component.i.length
+        else
+          # Delete
+          remainder = component.d
+          while remainder > 0
+            part = takeDoc snapshot, docPos, remainder
+            if typeof part is 'string'
+              @emit 'delete', textPos, part
+            remainder -= part.length || part
+
+      return
+

services/document-updater/app/coffee/sharejs/text-tp2.coffee

@@ -0,0 +1,322 @@
+# A TP2 implementation of text, following this spec:
+# http://code.google.com/p/lightwave/source/browse/trunk/experimental/ot/README
+#
+# A document is made up of a string and a set of tombstones inserted throughout
+# the string. For example, 'some ', (2 tombstones), 'string'.
+#
+# This is encoded in a document as: {s:'some string', t:[5, -2, 6]}
+#
+# Ops are lists of components which iterate over the whole document.
+# Components are either:
+#   N:         Skip N characters in the original document
+#   {i:'str'}: Insert 'str' at the current position in the document
+#   {i:N}:     Insert N tombstones at the current position in the document
+#   {d:N}:     Delete (tombstone) N characters at the current position in the document
+#
+# Eg: [3, {i:'hi'}, 5, {d:8}]
+#
+# Snapshots are lists with characters and tombstones. Characters are stored in strings
+# and adjacent tombstones are flattened into numbers.
+#
+# Eg, the document: 'Hello .....world' ('.' denotes tombstoned (deleted) characters)
+# would be represented by a document snapshot of ['Hello ', 5, 'world']
+
+type =
+  name: 'text-tp2'
+  tp2: true
+  create: -> {charLength:0, totalLength:0, positionCache:[], data:[]}
+  serialize: (doc) ->
+    throw new Error 'invalid doc snapshot' unless doc.data
+    doc.data
+  deserialize: (data) ->
+    doc = type.create()
+    doc.data = data
+    
+    for component in data
+      if typeof component is 'string'
+        doc.charLength += component.length
+        doc.totalLength += component.length
+      else
+        doc.totalLength += component
+    
+    doc
+
+
+checkOp = (op) ->
+  throw new Error('Op must be an array of components') unless Array.isArray(op)
+  last = null
+  for c in op
+    if typeof(c) == 'object'
+      if c.i != undefined
+        throw new Error('Inserts must insert a string or a +ive number') unless (typeof(c.i) == 'string' and c.i.length > 0) or (typeof(c.i) == 'number' and c.i > 0)
+      else if c.d != undefined
+        throw new Error('Deletes must be a +ive number') unless typeof(c.d) == 'number' and c.d > 0
+      else
+        throw new Error('Operation component must define .i or .d')
+    else
+      throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number'
+      throw new Error('Skip components must be a positive number') unless c > 0
+      throw new Error('Adjacent skip components should be combined') if typeof(last) == 'number'
+
+    last = c
+
+# Take the next part from the specified position in a document snapshot.
+# position = {index, offset}. It will be updated.
+type._takeDoc = takeDoc = (doc, position, maxlength, tombsIndivisible) ->
+  throw new Error 'Operation goes past the end of the document' if position.index >= doc.data.length
+
+  part = doc.data[position.index]
+  # peel off data[0]
+  result = if typeof(part) == 'string'
+    if maxlength != undefined
+      part[position.offset...(position.offset + maxlength)]
+    else
+      part[position.offset...]
+  else
+    if maxlength == undefined or tombsIndivisible
+      part - position.offset
+    else
+      Math.min(maxlength, part - position.offset)
+  
+  resultLen = result.length || result
+
+  if (part.length || part) - position.offset > resultLen
+    position.offset += resultLen
+  else
+    position.index++
+    position.offset = 0
+  
+  result
+
+# Append a part to the end of a document
+type._appendDoc = appendDoc = (doc, p) ->
+  return if p == 0 or p == ''
+
+  if typeof p is 'string'
+    doc.charLength += p.length
+    doc.totalLength += p.length
+  else
+    doc.totalLength += p
+
+  data = doc.data
+  if data.length == 0
+    data.push p
+  else if typeof(data[data.length - 1]) == typeof(p)
+    data[data.length - 1] += p
+  else
+    data.push p
+  return
+
+# Apply the op to the document. The document is not modified in the process.
+type.apply = (doc, op) ->
+  unless doc.totalLength != undefined and doc.charLength != undefined and doc.data.length != undefined
+    throw new Error('Snapshot is invalid')
+
+  checkOp op
+
+  newDoc = type.create()
+  position = {index:0, offset:0}
+
+  for component in op
+    if typeof(component) is 'number'
+      remainder = component
+      while remainder > 0
+        part = takeDoc doc, position, remainder
+        
+        appendDoc newDoc, part
+        remainder -= part.length || part
+
+    else if component.i != undefined
+      appendDoc newDoc, component.i
+    else if component.d != undefined
+      remainder = component.d
+      while remainder > 0
+        part = takeDoc doc, position, remainder
+        remainder -= part.length || part
+      appendDoc newDoc, component.d
+  
+  newDoc
+
+# Append an op component to the end of the specified op.
+# Exported for the randomOpGenerator.
+type._append = append = (op, component) ->
+  if component == 0 || component.i == '' || component.i == 0 || component.d == 0
+    return
+  else if op.length == 0
+    op.push component
+  else
+    last = op[op.length - 1]
+    if typeof(component) == 'number' && typeof(last) == 'number'
+      op[op.length - 1] += component
+    else if component.i != undefined && last.i? && typeof(last.i) == typeof(component.i)
+      last.i += component.i
+    else if component.d != undefined && last.d?
+      last.d += component.d
+    else
+      op.push component
+  
+# Makes 2 functions for taking components from the start of an op, and for peeking
+# at the next op that could be taken.
+makeTake = (op) ->
+  # The index of the next component to take
+  index = 0
+  # The offset into the component
+  offset = 0
+
+  # Take up to length maxlength from the op. If maxlength is not defined, there is no max.
+  # If insertsIndivisible is true, inserts (& insert tombstones) won't be separated.
+  #
+  # Returns null when op is fully consumed.
+  take = (maxlength, insertsIndivisible) ->
+    return null if index == op.length
+
+    e = op[index]
+    if typeof((current = e)) == 'number' or typeof((current = e.i)) == 'number' or (current = e.d) != undefined
+      if !maxlength? or current - offset <= maxlength or (insertsIndivisible and e.i != undefined)
+        # Return the rest of the current element.
+        c = current - offset
+        ++index; offset = 0
+      else
+        offset += maxlength
+        c = maxlength
+      if e.i != undefined then {i:c} else if e.d != undefined then {d:c} else c
+    else
+      # Take from the inserted string
+      if !maxlength? or e.i.length - offset <= maxlength or insertsIndivisible
+        result = {i:e.i[offset..]}
+        ++index; offset = 0
+      else
+        result = {i:e.i[offset...offset + maxlength]}
+        offset += maxlength
+      result
+  
+  peekType = -> op[index]
+  
+  [take, peekType]
+
+# Find and return the length of an op component
+componentLength = (component) ->
+  if typeof(component) == 'number'
+    component
+  else if typeof(component.i) == 'string'
+    component.i.length
+  else
+    # This should work because c.d and c.i must be +ive.
+    component.d or component.i
+
+# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate
+# adjacent inserts and deletes.
+type.normalize = (op) ->
+  newOp = []
+  append newOp, component for component in op
+  newOp
+
+# This is a helper method to transform and prune. goForwards is true for transform, false for prune.
+transformer = (op, otherOp, goForwards, side) ->
+  checkOp op
+  checkOp otherOp
+  newOp = []
+
+  [take, peek] = makeTake op
+
+  for component in otherOp
+    length = componentLength component
+
+    if component.i != undefined # Insert text or tombs
+      if goForwards # transform - insert skips over inserted parts
+        if side == 'left'
+          # The left insert should go first.
+          append newOp, take() while peek()?.i != undefined
+
+        # In any case, skip the inserted text.
+        append newOp, length
+
+      else # Prune. Remove skips for inserts.
+        while length > 0
+          chunk = take length, true
+
+          throw new Error 'The transformed op is invalid' unless chunk != null
+          throw new Error 'The transformed op deletes locally inserted characters - it cannot be purged of the insert.' if chunk.d != undefined
+
+          if typeof chunk is 'number'
+            length -= chunk
+          else
+            append newOp, chunk
+
+    else # Skip or delete
+      while length > 0
+        chunk = take length, true
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append newOp, chunk
+        length -= componentLength chunk unless chunk.i
+  
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in the op: #{component}" unless component.i != undefined
+    append newOp, component
+
+  newOp
+
+# transform op1 by op2. Return transformed version of op1.
+# op1 and op2 are unchanged by transform.
+# side should be 'left' or 'right', depending on if op1.id <> op2.id. 'left' == client op.
+type.transform = (op, otherOp, side) ->
+  throw new Error "side (#{side}) should be 'left' or 'right'" unless side == 'left' or side == 'right'
+  transformer op, otherOp, true, side
+
+# Prune is the inverse of transform.
+type.prune = (op, otherOp) -> transformer op, otherOp, false
+
+# Compose 2 ops into 1 op.
+type.compose = (op1, op2) ->
+  return op2 if op1 == null or op1 == undefined
+
+  checkOp op1
+  checkOp op2
+
+  result = []
+
+  [take, _] = makeTake op1
+
+  for component in op2
+
+    if typeof(component) == 'number' # Skip
+      # Just copy from op1.
+      length = component
+      while length > 0
+        chunk = take length
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append result, chunk
+        length -= componentLength chunk
+
+    else if component.i != undefined # Insert
+      append result, {i:component.i}
+
+    else # Delete
+      length = component.d
+      while length > 0
+        chunk = take length
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        chunkLength = componentLength chunk
+        if chunk.i != undefined
+          append result, {i:chunkLength}
+        else
+          append result, {d:chunkLength}
+
+        length -= chunkLength
+    
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in op1: #{component}" unless component.i != undefined
+    append result, component
+
+  result
+
+if WEB?
+  exports.types['text-tp2'] = type
+else
+  module.exports = type
+

services/document-updater/app/coffee/sharejs/text.coffee

@@ -0,0 +1,209 @@
+# A simple text implementation
+#
+# Operations are lists of components.
+# Each component either inserts or deletes at a specified position in the document.
+#
+# Components are either:
+#  {i:'str', p:100}: Insert 'str' at position 100 in the document
+#  {d:'str', p:100}: Delete 'str' at position 100 in the document
+#
+# Components in an operation are executed sequentially, so the position of components
+# assumes previous components have already executed.
+#
+# Eg: This op:
+#   [{i:'abc', p:0}]
+# is equivalent to this op:
+#   [{i:'a', p:0}, {i:'b', p:1}, {i:'c', p:2}]
+
+# NOTE: The global scope here is shared with other sharejs files when built with closure.
+# Be careful what ends up in your namespace.
+
+text = {}
+
+text.name = 'text'
+
+text.create = -> ''
+
+strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..]
+
+checkValidComponent = (c) ->
+  throw new Error 'component missing position field' if typeof c.p != 'number'
+
+  i_type = typeof c.i
+  d_type = typeof c.d
+  throw new Error 'component needs an i or d field' unless (i_type == 'string') ^ (d_type == 'string')
+
+  throw new Error 'position cannot be negative' unless c.p >= 0
+
+checkValidOp = (op) ->
+  checkValidComponent(c) for c in op
+  true
+
+text.apply = (snapshot, op) ->
+  checkValidOp op
+  for component in op
+    if component.i?
+      snapshot = strInject snapshot, component.p, component.i
+    else
+      deleted = snapshot[component.p...(component.p + component.d.length)]
+      throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted
+      snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..]
+  
+  snapshot
+
+
+# Exported for use by the random op generator.
+#
+# For simplicity, this version of append does not compress adjacent inserts and deletes of
+# the same text. It would be nice to change that at some stage.
+text._append = append = (newOp, c) ->
+  return if c.i == '' or c.d == ''
+  if newOp.length == 0
+    newOp.push c
+  else
+    last = newOp[newOp.length - 1]
+
+    # Compose the insert into the previous insert if possible
+    if last.i? && c.i? and last.p <= c.p <= (last.p + last.i.length)
+      newOp[newOp.length - 1] = {i:strInject(last.i, c.p - last.p, c.i), p:last.p}
+    else if last.d? && c.d? and c.p <= last.p <= (c.p + c.d.length)
+      newOp[newOp.length - 1] = {d:strInject(c.d, last.p - c.p, last.d), p:c.p}
+    else
+      newOp.push c
+
+text.compose = (op1, op2) ->
+  checkValidOp op1
+  checkValidOp op2
+
+  newOp = op1.slice()
+  append newOp, c for c in op2
+
+  newOp
+
+# Attempt to compress the op components together 'as much as possible'.
+# This implementation preserves order and preserves create/delete pairs.
+text.compress = (op) -> text.compose [], op
+
+text.normalize = (op) ->
+  newOp = []
+  
+  # Normalize should allow ops which are a single (unwrapped) component:
+  # {i:'asdf', p:23}.
+  # There's no good way to test if something is an array:
+  # http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/
+  # so this is probably the least bad solution.
+  op = [op] if op.i? or op.p?
+
+  for c in op
+    c.p ?= 0
+    append newOp, c
+  
+  newOp
+
+# This helper method transforms a position by an op component.
+#
+# If c is an insert, insertAfter specifies whether the transform
+# is pushed after the insert (true) or before it (false).
+#
+# insertAfter is optional for deletes.
+transformPosition = (pos, c, insertAfter) ->
+  if c.i?
+    if c.p < pos || (c.p == pos && insertAfter)
+      pos + c.i.length
+    else
+      pos
+  else
+    # I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length))
+    # but I think its harder to read that way, and it compiles using ternary operators anyway
+    # so its no slower written like this.
+    if pos <= c.p
+      pos
+    else if pos <= c.p + c.d.length
+      c.p
+    else
+      pos - c.d.length
+
+# Helper method to transform a cursor position as a result of an op.
+#
+# Like transformPosition above, if c is an insert, insertAfter specifies whether the cursor position
+# is pushed after an insert (true) or before it (false).
+text.transformCursor = (position, op, side) ->
+  insertAfter = side == 'right'
+  position = transformPosition position, c, insertAfter for c in op
+  position
+
+# Transform an op component by another op component. Asymmetric.
+# The result will be appended to destination.
+#
+# exported for use in JSON type
+text._tc = transformComponent = (dest, c, otherC, side) ->
+  checkValidOp [c]
+  checkValidOp [otherC]
+
+  if c.i?
+    append dest, {i:c.i, p:transformPosition(c.p, otherC, side == 'right')}
+
+  else # Delete
+    if otherC.i? # delete vs insert
+      s = c.d
+      if c.p < otherC.p
+        append dest, {d:s[...otherC.p - c.p], p:c.p}
+        s = s[(otherC.p - c.p)..]
+      if s != ''
+        append dest, {d:s, p:c.p + otherC.i.length}
+
+    else # Delete vs delete
+      if c.p >= otherC.p + otherC.d.length
+        append dest, {d:c.d, p:c.p - otherC.d.length}
+      else if c.p + c.d.length <= otherC.p
+        append dest, c
+      else
+        # They overlap somewhere.
+        newC = {d:'', p:c.p}
+        if c.p < otherC.p
+          newC.d = c.d[...(otherC.p - c.p)]
+        if c.p + c.d.length > otherC.p + otherC.d.length
+          newC.d += c.d[(otherC.p + otherC.d.length - c.p)..]
+
+        # This is entirely optional - just for a check that the deleted
+        # text in the two ops matches
+        intersectStart = Math.max c.p, otherC.p
+        intersectEnd = Math.min c.p + c.d.length, otherC.p + otherC.d.length
+        cIntersect = c.d[intersectStart - c.p...intersectEnd - c.p]
+        otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p]
+        throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect
+
+        if newC.d != ''
+          # This could be rewritten similarly to insert v delete, above.
+          newC.p = transformPosition newC.p, otherC
+          append dest, newC
+  
+  dest
+
+invertComponent = (c) ->
+  if c.i?
+    {d:c.i, p:c.p}
+  else
+    {i:c.d, p:c.p}
+
+# No need to use append for invert, because the components won't be able to
+# cancel with one another.
+text.invert = (op) -> (invertComponent c for c in op.slice().reverse())
+
+
+if WEB?
+  exports.types ||= {}
+
+  # This is kind of awful - come up with a better way to hook this helper code up.
+  bootstrapTransform(text, transformComponent, checkValidOp, append)
+
+  # [] is used to prevent closure from renaming types.text
+  exports.types.text = text
+else
+  module.exports = text
+
+  # The text type really shouldn't need this - it should be possible to define
+  # an efficient transform function by making a sort of transform map and passing each
+  # op component through it.
+  require('./helpers').bootstrapTransform(text, transformComponent, checkValidOp, append)
+

services/document-updater/app/coffee/sharejs/types/count.coffee

@@ -0,0 +1,22 @@
+# This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment]
+
+exports.name = 'count'
+exports.create = -> 1
+
+exports.apply = (snapshot, op) ->
+  [v, inc] = op
+  throw new Error "Op #{v} != snapshot #{snapshot}" unless snapshot == v
+  snapshot + inc
+
+# transform op1 by op2. Return transformed version of op1.
+exports.transform = (op1, op2) ->
+  throw new Error "Op1 #{op1[0]} != op2 #{op2[0]}" unless op1[0] == op2[0]
+  [op1[0] + op2[1], op1[1]]
+
+exports.compose = (op1, op2) ->
+  throw new Error "Op1 #{op1} + 1 != op2 #{op2}" unless op1[0] + op1[1] == op2[0]
+  [op1[0], op1[1] + op2[1]]
+
+exports.generateRandomOp = (doc) ->
+  [[doc, 1], doc + 1]
+

services/document-updater/app/coffee/sharejs/types/helpers.coffee

@@ -0,0 +1,65 @@
+# These methods let you build a transform function from a transformComponent function
+# for OT types like text and JSON in which operations are lists of components
+# and transforming them requires N^2 work.
+
+# Add transform and transformX functions for an OT type which has transformComponent defined.
+# transformComponent(destination array, component, other component, side)
+exports['_bt'] = bootstrapTransform = (type, transformComponent, checkValidOp, append) ->
+  transformComponentX = (left, right, destLeft, destRight) ->
+    transformComponent destLeft, left, right, 'left'
+    transformComponent destRight, right, left, 'right'
+
+  # Transforms rightOp by leftOp. Returns ['rightOp', clientOp']
+  type.transformX = type['transformX'] = transformX = (leftOp, rightOp) ->
+    checkValidOp leftOp
+    checkValidOp rightOp
+
+    newRightOp = []
+
+    for rightComponent in rightOp
+      # Generate newLeftOp by composing leftOp by rightComponent
+      newLeftOp = []
+
+      k = 0
+      while k < leftOp.length
+        nextC = []
+        transformComponentX leftOp[k], rightComponent, newLeftOp, nextC
+        k++
+
+        if nextC.length == 1
+          rightComponent = nextC[0]
+        else if nextC.length == 0
+          append newLeftOp, l for l in leftOp[k..]
+          rightComponent = null
+          break
+        else
+          # Recurse.
+          [l_, r_] = transformX leftOp[k..], nextC
+          append newLeftOp, l for l in l_
+          append newRightOp, r for r in r_
+          rightComponent = null
+          break
+    
+      append newRightOp, rightComponent if rightComponent?
+      leftOp = newLeftOp
+    
+    [leftOp, newRightOp]
+
+  # Transforms op with specified type ('left' or 'right') by otherOp.
+  type.transform = type['transform'] = (op, otherOp, type) ->
+    throw new Error "type must be 'left' or 'right'" unless type == 'left' or type == 'right'
+
+    return op if otherOp.length == 0
+
+    # TODO: Benchmark with and without this line. I _think_ it'll make a big difference...?
+    return transformComponent [], op[0], otherOp[0], type if op.length == 1 and otherOp.length == 1
+
+    if type == 'left'
+      [left, _] = transformX op, otherOp
+      left
+    else
+      [_, right] = transformX otherOp, op
+      right
+
+if typeof WEB is 'undefined'
+  exports.bootstrapTransform = bootstrapTransform

services/document-updater/app/coffee/sharejs/types/index.coffee

@@ -0,0 +1,15 @@
+
+register = (file) ->
+  type = require file
+  exports[type.name] = type
+  try require "#{file}-api"
+
+# Import all the built-in types.
+register './simple'
+register './count'
+
+register './text'
+register './text-composable'
+register './text-tp2'
+
+register './json'

services/document-updater/app/coffee/sharejs/types/json-api.coffee

@@ -0,0 +1,180 @@
+# API for JSON OT
+
+json = require './json' if typeof WEB is 'undefined'
+
+if WEB?
+  extendDoc = exports.extendDoc
+  exports.extendDoc = (name, fn) ->
+    SubDoc::[name] = fn
+    extendDoc name, fn
+
+depath = (path) ->
+  if path.length == 1 and path[0].constructor == Array
+    path[0]
+  else path
+
+class SubDoc
+  constructor: (@doc, @path) ->
+  at: (path...) -> @doc.at @path.concat depath path
+  get: -> @doc.getAt @path
+  # for objects and lists
+  set: (value, cb) -> @doc.setAt @path, value, cb
+  # for strings and lists.
+  insert: (pos, value, cb) -> @doc.insertAt @path, pos, value, cb
+  # for strings
+  del: (pos, length, cb) -> @doc.deleteTextAt @path, length, pos, cb
+  # for objects and lists
+  remove: (cb) -> @doc.removeAt @path, cb
+  push: (value, cb) -> @insert @get().length, value, cb
+  move: (from, to, cb) -> @doc.moveAt @path, from, to, cb
+  add: (amount, cb) -> @doc.addAt @path, amount, cb
+  on: (event, cb) -> @doc.addListener @path, event, cb
+  removeListener: (l) -> @doc.removeListener l
+
+  # text API compatibility
+  getLength: -> @get().length
+  getText: -> @get()
+
+traverse = (snapshot, path) ->
+  container = data:snapshot
+  key = 'data'
+  elem = container
+  for p in path
+    elem = elem[key]
+    key = p
+    throw new Error 'bad path' if typeof elem == 'undefined'
+  {elem, key}
+
+pathEquals = (p1, p2) ->
+  return false if p1.length != p2.length
+  for e,i in p1
+    return false if e != p2[i]
+  true
+
+json.api =
+  provides: {json:true}
+
+  at: (path...) -> new SubDoc this, depath path
+
+  get: -> @snapshot
+  set: (value, cb) -> @setAt [], value, cb
+
+  getAt: (path) ->
+    {elem, key} = traverse @snapshot, path
+    return elem[key]
+
+  setAt: (path, value, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = {p:path}
+    if elem.constructor == Array
+      op.li = value
+      op.ld = elem[key] if typeof elem[key] != 'undefined'
+    else if typeof elem == 'object'
+      op.oi = value
+      op.od = elem[key] if typeof elem[key] != 'undefined'
+    else throw new Error 'bad path'
+    @submitOp [op], cb
+
+  removeAt: (path, cb) ->
+    {elem, key} = traverse @snapshot, path
+    throw new Error 'no element at that path' unless typeof elem[key] != 'undefined'
+    op = {p:path}
+    if elem.constructor == Array
+      op.ld = elem[key]
+    else if typeof elem == 'object'
+      op.od = elem[key]
+    else throw new Error 'bad path'
+    @submitOp [op], cb
+
+  insertAt: (path, pos, value, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = {p:path.concat pos}
+    if elem[key].constructor == Array
+      op.li = value
+    else if typeof elem[key] == 'string'
+      op.si = value
+    @submitOp [op], cb
+
+  moveAt: (path, from, to, cb) ->
+    op = [{p:path.concat(from), lm:to}]
+    @submitOp op, cb
+
+  addAt: (path, amount, cb) ->
+    op = [{p:path, na:amount}]
+    @submitOp op, cb
+
+  deleteTextAt: (path, length, pos, cb) ->
+    {elem, key} = traverse @snapshot, path
+    op = [{p:path.concat(pos), sd:elem[key][pos...(pos + length)]}]
+    @submitOp op, cb
+
+  addListener: (path, event, cb) ->
+    l = {path, event, cb}
+    @_listeners.push l
+    l
+  removeListener: (l) ->
+    i = @_listeners.indexOf l
+    return false if i < 0
+    @_listeners.splice i, 1
+    return true
+  _register: ->
+    @_listeners = []
+    @on 'change', (op) ->
+      for c in op
+        if c.na != undefined or c.si != undefined or c.sd != undefined
+          # no change to structure
+          continue
+        to_remove = []
+        for l, i in @_listeners
+          # Transform a dummy op by the incoming op to work out what
+          # should happen to the listener.
+          dummy = {p:l.path, na:0}
+          xformed = @type.transformComponent [], dummy, c, 'left'
+          if xformed.length == 0
+            # The op was transformed to noop, so we should delete the listener.
+            to_remove.push i
+          else if xformed.length == 1
+            # The op remained, so grab its new path into the listener.
+            l.path = xformed[0].p
+          else
+            throw new Error "Bad assumption in json-api: xforming an 'si' op will always result in 0 or 1 components."
+        to_remove.sort (a, b) -> b - a
+        for i in to_remove
+          @_listeners.splice i, 1
+    @on 'remoteop', (op) ->
+      for c in op
+        match_path = if c.na == undefined then c.p[...c.p.length-1] else c.p
+        for {path, event, cb} in @_listeners
+          if pathEquals path, match_path
+            switch event
+              when 'insert'
+                if c.li != undefined and c.ld == undefined
+                  cb(c.p[c.p.length-1], c.li)
+                else if c.oi != undefined and c.od == undefined
+                  cb(c.p[c.p.length-1], c.oi)
+                else if c.si != undefined
+                  cb(c.p[c.p.length-1], c.si)
+              when 'delete'
+                if c.li == undefined and c.ld != undefined
+                  cb(c.p[c.p.length-1], c.ld)
+                else if c.oi == undefined and c.od != undefined
+                  cb(c.p[c.p.length-1], c.od)
+                else if c.sd != undefined
+                  cb(c.p[c.p.length-1], c.sd)
+              when 'replace'
+                if c.li != undefined and c.ld != undefined
+                  cb(c.p[c.p.length-1], c.ld, c.li)
+                else if c.oi != undefined and c.od != undefined
+                  cb(c.p[c.p.length-1], c.od, c.oi)
+              when 'move'
+                if c.lm != undefined
+                  cb(c.p[c.p.length-1], c.lm)
+              when 'add'
+                if c.na != undefined
+                  cb(c.na)
+          else if (common = @type.commonPath match_path, path)?
+            if event == 'child op'
+              if match_path.length == path.length == common
+                throw new Error "paths match length and have commonality, but aren't equal?"
+              child_path = c.p[common+1..]
+              cb(child_path, c)

services/document-updater/app/coffee/sharejs/types/json.coffee

@@ -0,0 +1,441 @@
+# This is the implementation of the JSON OT type.
+#
+# Spec is here: https://github.com/josephg/ShareJS/wiki/JSON-Operations
+
+if WEB?
+  text = exports.types.text
+else
+  text = require './text'
+
+json = {}
+
+json.name = 'json'
+
+json.create = -> null
+
+json.invertComponent = (c) ->
+  c_ = {p: c.p}
+  c_.sd = c.si if c.si != undefined
+  c_.si = c.sd if c.sd != undefined
+  c_.od = c.oi if c.oi != undefined
+  c_.oi = c.od if c.od != undefined
+  c_.ld = c.li if c.li != undefined
+  c_.li = c.ld if c.ld != undefined
+  c_.na = -c.na if c.na != undefined
+  if c.lm != undefined
+    c_.lm = c.p[c.p.length-1]
+    c_.p = c.p[0...c.p.length - 1].concat([c.lm])
+  c_
+
+json.invert = (op) -> json.invertComponent c for c in op.slice().reverse()
+
+json.checkValidOp = (op) ->
+
+isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]'
+json.checkList = (elem) ->
+  throw new Error 'Referenced element not a list' unless isArray(elem)
+
+json.checkObj = (elem) ->
+  throw new Error "Referenced element not an object (it was #{JSON.stringify elem})" unless elem.constructor is Object
+
+json.apply = (snapshot, op) ->
+  json.checkValidOp op
+  op = clone op
+
+  container = {data: clone snapshot}
+
+  try
+    for c, i in op
+      parent = null
+      parentkey = null
+      elem = container
+      key = 'data'
+
+      for p in c.p
+        parent = elem
+        parentkey = key
+        elem = elem[key]
+        key = p
+
+        throw new Error 'Path invalid' unless parent?
+
+      if c.na != undefined
+        # Number add
+        throw new Error 'Referenced element not a number' unless typeof elem[key] is 'number'
+        elem[key] += c.na
+
+      else if c.si != undefined
+        # String insert
+        throw new Error "Referenced element not a string (it was #{JSON.stringify elem})" unless typeof elem is 'string'
+        parent[parentkey] = elem[...key] + c.si + elem[key..]
+      else if c.sd != undefined
+        # String delete
+        throw new Error 'Referenced element not a string' unless typeof elem is 'string'
+        throw new Error 'Deleted string does not match' unless elem[key...key + c.sd.length] == c.sd
+        parent[parentkey] = elem[...key] + elem[key + c.sd.length..]
+
+      else if c.li != undefined && c.ld != undefined
+        # List replace
+        json.checkList elem
+
+        # Should check the list element matches c.ld
+        elem[key] = c.li
+      else if c.li != undefined
+        # List insert
+        json.checkList elem
+
+        elem.splice key, 0, c.li
+      else if c.ld != undefined
+        # List delete
+        json.checkList elem
+
+        # Should check the list element matches c.ld here too.
+        elem.splice key, 1
+      else if c.lm != undefined
+        # List move
+        json.checkList elem
+        if c.lm != key
+          e = elem[key]
+          # Remove it...
+          elem.splice key, 1
+          # And insert it back.
+          elem.splice c.lm, 0, e
+
+      else if c.oi != undefined
+        # Object insert / replace
+        json.checkObj elem
+        
+        # Should check that elem[key] == c.od
+        elem[key] = c.oi
+      else if c.od != undefined
+        # Object delete
+        json.checkObj elem
+
+        # Should check that elem[key] == c.od
+        delete elem[key]
+      else
+        throw new Error 'invalid / missing instruction in op'
+  catch error
+    # TODO: Roll back all already applied changes. Write tests before implementing this code.
+    throw error
+
+  container.data
+
+# Checks if two paths, p1 and p2 match.
+json.pathMatches = (p1, p2, ignoreLast) ->
+  return false unless p1.length == p2.length
+
+  for p, i in p1
+    return false if p != p2[i] and (!ignoreLast or i != p1.length - 1)
+      
+  true
+
+json.append = (dest, c) ->
+  c = clone c
+  if dest.length != 0 and json.pathMatches c.p, (last = dest[dest.length - 1]).p
+    if last.na != undefined and c.na != undefined
+      dest[dest.length - 1] = { p: last.p, na: last.na + c.na }
+    else if last.li != undefined and c.li == undefined and c.ld == last.li
+      # insert immediately followed by delete becomes a noop.
+      if last.ld != undefined
+        # leave the delete part of the replace
+        delete last.li
+      else
+        dest.pop()
+    else if last.od != undefined and last.oi == undefined and
+        c.oi != undefined and c.od == undefined
+      last.oi = c.oi
+    else if c.lm != undefined and c.p[c.p.length-1] == c.lm
+      null # don't do anything
+    else
+      dest.push c
+  else
+    dest.push c
+
+json.compose = (op1, op2) ->
+  json.checkValidOp op1
+  json.checkValidOp op2
+
+  newOp = clone op1
+  json.append newOp, c for c in op2
+
+  newOp
+
+json.normalize = (op) ->
+  newOp = []
+  
+  op = [op] unless isArray op
+
+  for c in op
+    c.p ?= []
+    json.append newOp, c
+  
+  newOp
+
+# hax, copied from test/types/json. Apparently this is still the fastest way to deep clone an object, assuming
+# we have browser support for JSON.
+# http://jsperf.com/cloning-an-object/12
+clone = (o) -> JSON.parse(JSON.stringify o)
+
+json.commonPath = (p1, p2) ->
+  p1 = p1.slice()
+  p2 = p2.slice()
+  p1.unshift('data')
+  p2.unshift('data')
+  p1 = p1[...p1.length-1]
+  p2 = p2[...p2.length-1]
+  return -1 if p2.length == 0
+  i = 0
+  while p1[i] == p2[i] && i < p1.length
+    i++
+    if i == p2.length
+      return i-1
+  return
+
+# transform c so it applies to a document with otherC applied.
+json.transformComponent = (dest, c, otherC, type) ->
+  c = clone c
+  c.p.push(0) if c.na != undefined
+  otherC.p.push(0) if otherC.na != undefined
+
+  common = json.commonPath c.p, otherC.p
+  common2 = json.commonPath otherC.p, c.p
+
+  cplength = c.p.length
+  otherCplength = otherC.p.length
+
+  c.p.pop() if c.na != undefined # hax
+  otherC.p.pop() if otherC.na != undefined
+
+  if otherC.na
+    if common2? && otherCplength >= cplength && otherC.p[common2] == c.p[common2]
+      if c.ld != undefined
+        oc = clone otherC
+        oc.p = oc.p[cplength..]
+        c.ld = json.apply clone(c.ld), [oc]
+      else if c.od != undefined
+        oc = clone otherC
+        oc.p = oc.p[cplength..]
+        c.od = json.apply clone(c.od), [oc]
+    json.append dest, c
+    return dest
+
+  if common2? && otherCplength > cplength && c.p[common2] == otherC.p[common2]
+    # transform based on c
+    if c.ld != undefined
+      oc = clone otherC
+      oc.p = oc.p[cplength..]
+      c.ld = json.apply clone(c.ld), [oc]
+    else if c.od != undefined
+      oc = clone otherC
+      oc.p = oc.p[cplength..]
+      c.od = json.apply clone(c.od), [oc]
+
+
+  if common?
+    commonOperand = cplength == otherCplength
+    # transform based on otherC
+    if otherC.na != undefined
+      # this case is handled above due to icky path hax
+    else if otherC.si != undefined || otherC.sd != undefined
+      # String op vs string op - pass through to text type
+      if c.si != undefined || c.sd != undefined
+        throw new Error("must be a string?") unless commonOperand
+
+        # Convert an op component to a text op component
+        convert = (component) ->
+          newC = p:component.p[component.p.length - 1]
+          if component.si
+            newC.i = component.si
+          else
+            newC.d = component.sd
+          newC
+
+        tc1 = convert c
+        tc2 = convert otherC
+          
+        res = []
+        text._tc res, tc1, tc2, type
+        for tc in res
+          jc = { p: c.p[...common] }
+          jc.p.push(tc.p)
+          jc.si = tc.i if tc.i?
+          jc.sd = tc.d if tc.d?
+          json.append dest, jc
+        return dest
+    else if otherC.li != undefined && otherC.ld != undefined
+      if otherC.p[common] == c.p[common]
+        # noop
+        if !commonOperand
+          # we're below the deleted element, so -> noop
+          return dest
+        else if c.ld != undefined
+          # we're trying to delete the same element, -> noop
+          if c.li != undefined and type == 'left'
+            # we're both replacing one element with another. only one can
+            # survive!
+            c.ld = clone otherC.li
+          else
+            return dest
+    else if otherC.li != undefined
+      if c.li != undefined and c.ld == undefined and commonOperand and c.p[common] == otherC.p[common]
+        # in li vs. li, left wins.
+        if type == 'right'
+          c.p[common]++
+      else if otherC.p[common] <= c.p[common]
+        c.p[common]++
+
+      if c.lm != undefined
+        if commonOperand
+          # otherC edits the same list we edit
+          if otherC.p[common] <= c.lm
+            c.lm++
+          # changing c.from is handled above.
+    else if otherC.ld != undefined
+      if c.lm != undefined
+        if commonOperand
+          if otherC.p[common] == c.p[common]
+            # they deleted the thing we're trying to move
+            return dest
+          # otherC edits the same list we edit
+          p = otherC.p[common]
+          from = c.p[common]
+          to = c.lm
+          if p < to || (p == to && from < to)
+            c.lm--
+
+      if otherC.p[common] < c.p[common]
+        c.p[common]--
+      else if otherC.p[common] == c.p[common]
+        if otherCplength < cplength
+          # we're below the deleted element, so -> noop
+          return dest
+        else if c.ld != undefined
+          if c.li != undefined
+            # we're replacing, they're deleting. we become an insert.
+            delete c.ld
+          else
+            # we're trying to delete the same element, -> noop
+            return dest
+    else if otherC.lm != undefined
+      if c.lm != undefined and cplength == otherCplength
+        # lm vs lm, here we go!
+        from = c.p[common]
+        to = c.lm
+        otherFrom = otherC.p[common]
+        otherTo = otherC.lm
+        if otherFrom != otherTo
+          # if otherFrom == otherTo, we don't need to change our op.
+
+          # where did my thing go?
+          if from == otherFrom
+            # they moved it! tie break.
+            if type == 'left'
+              c.p[common] = otherTo
+              if from == to # ugh
+                c.lm = otherTo
+            else
+              return dest
+          else
+            # they moved around it
+            if from > otherFrom
+              c.p[common]--
+            if from > otherTo
+              c.p[common]++
+            else if from == otherTo
+              if otherFrom > otherTo
+                c.p[common]++
+                if from == to # ugh, again
+                  c.lm++
+
+            # step 2: where am i going to put it?
+            if to > otherFrom
+              c.lm--
+            else if to == otherFrom
+              if to > from
+                c.lm--
+            if to > otherTo
+              c.lm++
+            else if to == otherTo
+              # if we're both moving in the same direction, tie break
+              if (otherTo > otherFrom and to > from) or
+                 (otherTo < otherFrom and to < from)
+                if type == 'right'
+                  c.lm++
+              else
+                if to > from
+                  c.lm++
+                else if to == otherFrom
+                  c.lm--
+      else if c.li != undefined and c.ld == undefined and commonOperand
+        # li
+        from = otherC.p[common]
+        to = otherC.lm
+        p = c.p[common]
+        if p > from
+          c.p[common]--
+        if p > to
+          c.p[common]++
+      else
+        # ld, ld+li, si, sd, na, oi, od, oi+od, any li on an element beneath
+        # the lm
+        #
+        # i.e. things care about where their item is after the move.
+        from = otherC.p[common]
+        to = otherC.lm
+        p = c.p[common]
+        if p == from
+          c.p[common] = to
+        else
+          if p > from
+            c.p[common]--
+          if p > to
+            c.p[common]++
+          else if p == to
+            if from > to
+              c.p[common]++
+    else if otherC.oi != undefined && otherC.od != undefined
+      if c.p[common] == otherC.p[common]
+        if c.oi != undefined and commonOperand
+          # we inserted where someone else replaced
+          if type == 'right'
+            # left wins
+            return dest
+          else
+            # we win, make our op replace what they inserted
+            c.od = otherC.oi
+        else
+          # -> noop if the other component is deleting the same object (or any
+          # parent)
+          return dest
+    else if otherC.oi != undefined
+      if c.oi != undefined and c.p[common] == otherC.p[common]
+        # left wins if we try to insert at the same place
+        if type == 'left'
+          json.append dest, {p:c.p, od:otherC.oi}
+        else
+          return dest
+    else if otherC.od != undefined
+      if c.p[common] == otherC.p[common]
+        return dest if !commonOperand
+        if c.oi != undefined
+          delete c.od
+        else
+          return dest
+  
+  json.append dest, c
+  return dest
+
+if WEB?
+  exports.types ||= {}
+
+  # This is kind of awful - come up with a better way to hook this helper code up.
+  exports._bt(json, json.transformComponent, json.checkValidOp, json.append)
+
+  # [] is used to prevent closure from renaming types.text
+  exports.types.json = json
+else
+  module.exports = json
+
+  require('./helpers').bootstrapTransform(json, json.transformComponent, json.checkValidOp, json.append)
+

services/document-updater/app/coffee/sharejs/types/model.coffee

@@ -0,0 +1,603 @@
+# The model of all the ops. Responsible for applying & transforming remote deltas
+# and managing the storage layer.
+#
+# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
+
+{EventEmitter} = require 'events'
+
+queue = require './syncqueue'
+types = require '../types'
+
+isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]'
+
+# This constructor creates a new Model object. There will be one model object
+# per server context.
+#
+# The model object is responsible for a lot of things:
+#
+# - It manages the interactions with the database
+# - It maintains (in memory) a set of all active documents
+# - It calls out to the OT functions when necessary
+#
+# The model is an event emitter. It emits the following events:
+#
+# create(docName, data): A document has been created with the specified name & data
+module.exports = Model = (db, options) ->
+  # db can be null if the user doesn't want persistance.
+
+  return new Model(db, options) if !(this instanceof Model)
+
+  model = this
+
+  options ?= {}
+
+  # This is a cache of 'live' documents.
+  #
+  # The cache is a map from docName -> {
+  #   ops:[{op, meta}]
+  #   snapshot
+  #   type
+  #   v
+  #   meta
+  #   eventEmitter
+  #   reapTimer
+  #   committedVersion: v
+  #   snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant
+  #   dbMeta: database specific data
+  #   opQueue: syncQueue for processing ops
+  # }
+  #
+  # The ops list contains the document's last options.numCachedOps ops. (Or all
+  # of them if we're using a memory store).
+  #
+  # Documents are stored in this set so long as the document has been accessed in
+  # the last few seconds (options.reapTime) OR at least one client has the document
+  # open. I don't know if I should keep open (but not being edited) documents live -
+  # maybe if a client has a document open but the document isn't being edited, I should
+  # flush it from the cache.
+  #
+  # In any case, the API to model is designed such that if we want to change that later
+  # it should be pretty easy to do so without any external-to-the-model code changes.
+  docs = {}
+
+  # This is a map from docName -> [callback]. It is used when a document hasn't been
+  # cached and multiple getSnapshot() / getVersion() requests come in. All requests
+  # are added to the callback list and called when db.getSnapshot() returns.
+  #
+  # callback(error, snapshot data)
+  awaitingGetSnapshot = {}
+
+  # The time that documents which no clients have open will stay in the cache.
+  # Should be > 0.
+  options.reapTime ?= 3000
+
+  # The number of operations the cache holds before reusing the space
+  options.numCachedOps ?= 10
+
+  # This option forces documents to be reaped, even when there's no database backend.
+  # This is useful when you don't care about persistance and don't want to gradually
+  # fill memory.
+  #
+  # You might want to set reapTime to a day or something.
+  options.forceReaping ?= false
+
+  # Until I come up with a better strategy, we'll save a copy of the document snapshot
+  # to the database every ~20 submitted ops.
+  options.opsBeforeCommit ?= 20
+
+  # It takes some processing time to transform client ops. The server will punt ops back to the
+  # client to transform if they're too old.
+  options.maximumAge ?= 40
+
+  # **** Cache API methods
+
+  # Its important that all ops are applied in order. This helper method creates the op submission queue
+  # for a single document. This contains the logic for transforming & applying ops.
+  makeOpQueue = (docName, doc) -> queue (opData, callback) ->
+    return callback 'Version missing' unless opData.v >= 0
+    return callback 'Op at future version' if opData.v > doc.v
+
+    # Punt the transforming work back to the client if the op is too old.
+    return callback 'Op too old' if opData.v + options.maximumAge < doc.v
+
+    opData.meta ||= {}
+    opData.meta.ts = Date.now()
+
+    # We'll need to transform the op to the current version of the document. This
+    # calls the callback immediately if opVersion == doc.v.
+    getOps docName, opData.v, doc.v, (error, ops) ->
+      return callback error if error
+
+      unless doc.v - opData.v == ops.length
+        # This should never happen. It indicates that we didn't get all the ops we
+        # asked for. Its important that the submitted op is correctly transformed.
+        console.error "Could not get old ops in model for document #{docName}"
+        console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops"
+        return callback 'Internal error'
+
+      if ops.length > 0
+        try
+          # If there's enough ops, it might be worth spinning this out into a webworker thread.
+          for oldOp in ops
+            # Dup detection works by sending the id(s) the op has been submitted with previously.
+            # If the id matches, we reject it. The client can also detect the op has been submitted
+            # already if it sees its own previous id in the ops it sees when it does catchup.
+            if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource
+              return callback 'Op already submitted'
+
+            opData.op = doc.type.transform opData.op, oldOp.op, 'left'
+            opData.v++
+        catch error
+          console.error error.stack
+          return callback error.message
+
+      try
+        snapshot = doc.type.apply doc.snapshot, opData.op
+      catch error
+        console.error error.stack
+        return callback error.message
+
+      # The op data should be at the current version, and the new document data should be at
+      # the next version.
+      #
+      # This should never happen in practice, but its a nice little check to make sure everything
+      # is hunky-dory.
+      unless opData.v == doc.v
+        # This should never happen.
+        console.error "Version mismatch detected in model. File a ticket - this is a bug."
+        console.error "Expecting #{opData.v} == #{doc.v}"
+        return callback 'Internal error'
+
+      #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta}
+      writeOp = db?.writeOp or (docName, newOpData, callback) -> callback()
+
+      writeOp docName, opData, (error) ->
+        if error
+          # The user should probably know about this.
+          console.warn "Error writing ops to database: #{error}"
+          return callback error
+
+        options.stats?.writeOp?()
+
+        # This is needed when we emit the 'change' event, below.
+        oldSnapshot = doc.snapshot
+
+        # All the heavy lifting is now done. Finally, we'll update the cache with the new data
+        # and (maybe!) save a new document snapshot to the database.
+
+        doc.v = opData.v + 1
+        doc.snapshot = snapshot
+
+        doc.ops.push opData
+        doc.ops.shift() if db and doc.ops.length > options.numCachedOps
+
+        model.emit 'applyOp', docName, opData, snapshot, oldSnapshot
+        doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot
+
+        # The callback is called with the version of the document at which the op was applied.
+        # This is the op.v after transformation, and its doc.v - 1.
+        callback null, opData.v
+    
+        # I need a decent strategy here for deciding whether or not to save the snapshot.
+        #
+        # The 'right' strategy looks something like "Store the snapshot whenever the snapshot
+        # is smaller than the accumulated op data". For now, I'll just store it every 20
+        # ops or something. (Configurable with doc.committedVersion)
+        if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v
+          tryWriteSnapshot docName, (error) ->
+            console.warn "Error writing snapshot #{error}. This is nonfatal" if error
+
+  # Add the data for the given docName to the cache. The named document shouldn't already
+  # exist in the doc set.
+  #
+  # Returns the new doc.
+  add = (docName, error, data, committedVersion, ops, dbMeta) ->
+    callbacks = awaitingGetSnapshot[docName]
+    delete awaitingGetSnapshot[docName]
+
+    if error
+      callback error for callback in callbacks if callbacks
+    else
+      doc = docs[docName] =
+        snapshot: data.snapshot
+        v: data.v
+        type: data.type
+        meta: data.meta
+
+        # Cache of ops
+        ops: ops or []
+
+        eventEmitter: new EventEmitter
+
+        # Timer before the document will be invalidated from the cache (if the document has no
+        # listeners)
+        reapTimer: null
+
+        # Version of the snapshot thats in the database
+        committedVersion: committedVersion ? data.v
+        snapshotWriteLock: false
+        dbMeta: dbMeta
+
+      doc.opQueue = makeOpQueue docName, doc
+      
+      refreshReapingTimeout docName
+      model.emit 'add', docName, data
+      callback null, doc for callback in callbacks if callbacks
+
+    doc
+  
+  # This is a little helper wrapper around db.getOps. It does two things:
+  #
+  # - If there's no database set, it returns an error to the callback
+  # - It adds version numbers to each op returned from the database
+  # (These can be inferred from context so the DB doesn't store them, but its useful to have them).
+  getOpsInternal = (docName, start, end, callback) ->
+    return callback? 'Document does not exist' unless db
+
+    db.getOps docName, start, end, (error, ops) ->
+      return callback? error if error
+
+      v = start
+      op.v = v++ for op in ops
+
+      callback? null, ops
+
+  # Load the named document into the cache. This function is re-entrant.
+  #
+  # The callback is called with (error, doc)
+  load = (docName, callback) ->
+    if docs[docName]
+      # The document is already loaded. Return immediately.
+      options.stats?.cacheHit? 'getSnapshot'
+      return callback null, docs[docName]
+
+    # We're a memory store. If we don't have it, nobody does.
+    return callback 'Document does not exist' unless db
+
+    callbacks = awaitingGetSnapshot[docName]
+
+    # The document is being loaded already. Add ourselves as a callback.
+    return callbacks.push callback if callbacks
+
+    options.stats?.cacheMiss? 'getSnapshot'
+
+    # The document isn't loaded and isn't being loaded. Load it.
+    awaitingGetSnapshot[docName] = [callback]
+    db.getSnapshot docName, (error, data, dbMeta) ->
+      return add docName, error if error
+
+      type = types[data.type]
+      unless type
+        console.warn "Type '#{data.type}' missing"
+        return callback "Type not found"
+      data.type = type
+
+      committedVersion = data.v
+
+      # The server can close without saving the most recent document snapshot.
+      # In this case, there are extra ops which need to be applied before
+      # returning the snapshot.
+      getOpsInternal docName, data.v, null, (error, ops) ->
+        return callback error if error
+
+        if ops.length > 0
+          console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}"
+
+          try
+            for op in ops
+              data.snapshot = type.apply data.snapshot, op.op
+              data.v++
+          catch e
+            # This should never happen - it indicates that whats in the
+            # database is invalid.
+            console.error "Op data invalid for #{docName}: #{e.stack}"
+            return callback 'Op data invalid'
+
+        model.emit 'load', docName, data
+        add docName, error, data, committedVersion, ops, dbMeta
+
+  # This makes sure the cache contains a document. If the doc cache doesn't contain
+  # a document, it is loaded from the database and stored.
+  #
+  # Documents are stored so long as either:
+  # - They have been accessed within the past #{PERIOD}
+  # - At least one client has the document open
+  refreshReapingTimeout = (docName) ->
+    doc = docs[docName]
+    return unless doc
+
+    # I want to let the clients list be updated before this is called.
+    process.nextTick ->
+      # This is an awkward way to find out the number of clients on a document. If this
+      # causes performance issues, add a numClients field to the document.
+      #
+      # The first check is because its possible that between refreshReapingTimeout being called and this
+      # event being fired, someone called delete() on the document and hence the doc is something else now.
+      if doc == docs[docName] and
+          doc.eventEmitter.listeners('op').length == 0 and
+          (db or options.forceReaping) and
+          doc.opQueue.busy is false
+
+        clearTimeout doc.reapTimer
+        doc.reapTimer = reapTimer = setTimeout ->
+            tryWriteSnapshot docName, ->
+              # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're
+              # in the middle of applying an operation, don't reap.
+              delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false
+          , options.reapTime
+
+  tryWriteSnapshot = (docName, callback) ->
+    return callback?() unless db
+
+    doc = docs[docName]
+
+    # The doc is closed
+    return callback?() unless doc
+
+    # The document is already saved.
+    return callback?() if doc.committedVersion is doc.v
+
+    return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock
+
+    doc.snapshotWriteLock = true
+
+    options.stats?.writeSnapshot?()
+
+    writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback()
+
+    data =
+      v: doc.v
+      meta: doc.meta
+      snapshot: doc.snapshot
+      # The database doesn't know about object types.
+      type: doc.type.name
+
+    # Commit snapshot.
+    writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) ->
+      doc.snapshotWriteLock = false
+
+      # We have to use data.v here because the version in the doc could
+      # have been updated between the call to writeSnapshot() and now.
+      doc.committedVersion = data.v
+      doc.dbMeta = dbMeta
+
+      callback? error
+
+  # *** Model interface methods
+
+  # Create a new document.
+  #
+  # data should be {snapshot, type, [meta]}. The version of a new document is 0.
+  @create = (docName, type, meta, callback) ->
+    [meta, callback] = [{}, meta] if typeof meta is 'function'
+
+    return callback? 'Invalid document name' if docName.match /\//
+    return callback? 'Document already exists' if docs[docName]
+
+    type = types[type] if typeof type == 'string'
+    return callback? 'Type not found' unless type
+
+    data =
+      snapshot:type.create()
+      type:type.name
+      meta:meta or {}
+      v:0
+
+    done = (error, dbMeta) ->
+      # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something.
+      return callback? error if error
+
+      # From here on we'll store the object version of the type name.
+      data.type = type
+      add docName, null, data, 0, [], dbMeta
+      model.emit 'create', docName, data
+      callback?()
+
+    if db
+      db.create docName, data, done
+    else
+      done()
+
+  # Perminantly deletes the specified document.
+  # If listeners are attached, they are removed.
+  # 
+  # The callback is called with (error) if there was an error. If error is null / undefined, the
+  # document was deleted.
+  #
+  # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the
+  # deletion. Subsequent op submissions will fail).
+  @delete = (docName, callback) ->
+    doc = docs[docName]
+
+    if doc
+      clearTimeout doc.reapTimer
+      delete docs[docName]
+
+    done = (error) ->
+      model.emit 'delete', docName unless error
+      callback? error
+
+    if db
+      db.delete docName, doc?.dbMeta, done
+    else
+      done (if !doc then 'Document does not exist')
+
+  # This gets all operations from [start...end]. (That is, its not inclusive.)
+  #
+  # end can be null. This means 'get me all ops from start'.
+  #
+  # Each op returned is in the form {op:o, meta:m, v:version}.
+  #
+  # Callback is called with (error, [ops])
+  #
+  # If the document does not exist, getOps doesn't necessarily return an error. This is because
+  # its awkward to figure out whether or not the document exists for things
+  # like the redis database backend. I guess its a bit gross having this inconsistant
+  # with the other DB calls, but its certainly convenient.
+  #
+  # Use getVersion() to determine if a document actually exists, if thats what you're
+  # after.
+  @getOps = getOps = (docName, start, end, callback) ->
+    # getOps will only use the op cache if its there. It won't fill the op cache in.
+    throw new Error 'start must be 0+' unless start >= 0
+
+    [end, callback] = [null, end] if typeof end is 'function'
+
+    ops = docs[docName]?.ops
+
+    if ops
+      version = docs[docName].v
+
+      # Ops contains an array of ops. The last op in the list is the last op applied
+      end ?= version
+      start = Math.min start, end
+
+      return callback null, [] if start == end
+
+      # Base is the version number of the oldest op we have cached
+      base = version - ops.length
+
+      # If the database is null, we'll trim to the ops we do have and hope thats enough.
+      if start >= base or db is null
+        refreshReapingTimeout docName
+        options.stats?.cacheHit 'getOps'
+
+        return callback null, ops[(start - base)...(end - base)]
+
+    options.stats?.cacheMiss 'getOps'
+
+    getOpsInternal docName, start, end, callback
+
+  # Gets the snapshot data for the specified document.
+  # getSnapshot(docName, callback)
+  # Callback is called with (error, {v: <version>, type: <type>, snapshot: <snapshot>, meta: <meta>})
+  @getSnapshot = (docName, callback) ->
+    load docName, (error, doc) ->
+      callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta}
+
+  # Gets the latest version # of the document.
+  # getVersion(docName, callback)
+  # callback is called with (error, version).
+  @getVersion = (docName, callback) ->
+    load docName, (error, doc) -> callback error, doc?.v
+
+  # Apply an op to the specified document.
+  # The callback is passed (error, applied version #)
+  # opData = {op:op, v:v, meta:metadata}
+  # 
+  # Ops are queued before being applied so that the following code applies op C before op B:
+  # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB
+  # model.applyOp 'doc', OPC
+  @applyOp = (docName, opData, callback) ->
+    # All the logic for this is in makeOpQueue, above.
+    load docName, (error, doc) ->
+      return callback error if error
+
+      process.nextTick -> doc.opQueue opData, (error, newVersion) ->
+        refreshReapingTimeout docName
+        callback? error, newVersion
+
+  # TODO: store (some) metadata in DB
+  # TODO: op and meta should be combineable in the op that gets sent
+  @applyMetaOp = (docName, metaOpData, callback) ->
+    {path, value} = metaOpData.meta
+   
+    return callback? "path should be an array" unless isArray path
+
+    load docName, (error, doc) ->
+      if error?
+        callback? error
+      else
+        applied = false
+        switch path[0]
+          when 'shout'
+            doc.eventEmitter.emit 'op', metaOpData
+            applied = true
+
+        model.emit 'applyMetaOp', docName, path, value if applied
+        callback? null, doc.v
+
+  # Listen to all ops from the specified version. If version is in the past, all
+  # ops since that version are sent immediately to the listener.
+  #
+  # The callback is called once the listener is attached, but before any ops have been passed
+  # to the listener.
+  # 
+  # This will _not_ edit the document metadata.
+  #
+  # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour
+  # might change in a future version.
+  #
+  # version is the document version at which the document is opened. It can be left out if you want to open
+  # the document at the most recent version.
+  #
+  # listener is called with (opData) each time an op is applied.
+  #
+  # callback(error, openedVersion)
+  @listen = (docName, version, listener, callback) ->
+    [version, listener, callback] = [null, version, listener] if typeof version is 'function'
+
+    load docName, (error, doc) ->
+      return callback? error if error
+
+      clearTimeout doc.reapTimer
+
+      if version?
+        getOps docName, version, null, (error, data) ->
+          return callback? error if error
+
+          doc.eventEmitter.on 'op', listener
+          callback? null, version
+          for op in data
+            listener op
+
+            # The listener may well remove itself during the catchup phase. If this happens, break early.
+            # This is done in a quite inefficient way. (O(n) where n = #listeners on doc)
+            break unless listener in doc.eventEmitter.listeners 'op'
+
+      else # Version is null / undefined. Just add the listener.
+        doc.eventEmitter.on 'op', listener
+        callback? null, doc.v
+
+  # Remove a listener for a particular document.
+  #
+  # removeListener(docName, listener)
+  #
+  # This is synchronous.
+  @removeListener = (docName, listener) ->
+    # The document should already be loaded.
+    doc = docs[docName]
+    throw new Error 'removeListener called but document not loaded' unless doc
+
+    doc.eventEmitter.removeListener 'op', listener
+    refreshReapingTimeout docName
+
+  # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed -
+  # sharejs will happily replay uncommitted ops when documents are re-opened anyway.
+  @flush = (callback) ->
+    return callback?() unless db
+
+    pendingWrites = 0
+
+    for docName, doc of docs
+      if doc.committedVersion < doc.v
+        pendingWrites++
+        # I'm hoping writeSnapshot will always happen in another thread.
+        tryWriteSnapshot docName, ->
+          process.nextTick ->
+            pendingWrites--
+            callback?() if pendingWrites is 0
+
+    # If nothing was queued, terminate immediately.
+    callback?() if pendingWrites is 0
+
+  # Close the database connection. This is needed so nodejs can shut down cleanly.
+  @closeDb = ->
+    db?.close?()
+    db = null
+
+  return
+
+# Model inherits from EventEmitter.
+Model:: = new EventEmitter
+

services/document-updater/app/coffee/sharejs/types/simple.coffee

@@ -0,0 +1,38 @@
+# This is a really simple OT type. Its not compiled with the web client, but it could be.
+#
+# Its mostly included for demonstration purposes and its used in a lot of unit tests.
+#
+# This defines a really simple text OT type which only allows inserts. (No deletes).
+#
+# Ops look like:
+#   {position:#, text:"asdf"}
+#
+# Document snapshots look like:
+#   {str:string}
+
+module.exports =
+  # The name of the OT type. The type is stored in types[type.name]. The name can be
+  # used in place of the actual type in all the API methods.
+  name: 'simple'
+
+  # Create a new document snapshot
+  create: -> {str:""}
+
+  # Apply the given op to the document snapshot. Returns the new snapshot.
+  #
+  # The original snapshot should not be modified.
+  apply: (snapshot, op) ->
+    throw new Error 'Invalid position' unless 0 <= op.position <= snapshot.str.length
+
+    str = snapshot.str
+    str = str.slice(0, op.position) + op.text + str.slice(op.position)
+    {str}
+
+  # transform op1 by op2. Return transformed version of op1.
+  # sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the
+  # op being transformed comes from the client or the server.
+  transform: (op1, op2, sym) ->
+    pos = op1.position
+    pos += op2.text.length if op2.position < pos or (op2.position == pos and sym is 'left')
+
+    return {position:pos, text:op1.text}

services/document-updater/app/coffee/sharejs/types/syncqueue.coffee

@@ -0,0 +1,42 @@
+# A synchronous processing queue. The queue calls process on the arguments,
+# ensuring that process() is only executing once at a time.
+#
+# process(data, callback) _MUST_ eventually call its callback.
+#
+# Example:
+#
+# queue = require 'syncqueue'
+#
+# fn = queue (data, callback) ->
+#     asyncthing data, ->
+#         callback(321)
+#
+# fn(1)
+# fn(2)
+# fn(3, (result) -> console.log(result))
+#
+#   ^--- async thing will only be running once at any time.
+
+module.exports = (process) ->
+  throw new Error('process is not a function') unless typeof process == 'function'
+  queue = []
+  
+  enqueue = (data, callback) ->
+    queue.push [data, callback]
+    flush()
+
+  enqueue.busy = false
+
+  flush = ->
+    return if enqueue.busy or queue.length == 0
+
+    enqueue.busy = true
+    [data, callback] = queue.shift()
+    process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow.
+      enqueue.busy = false
+      # This is called after busy = false so a user can check if enqueue.busy is set in the callback.
+      callback.apply null, result if callback
+      flush()
+
+  enqueue
+

services/document-updater/app/coffee/sharejs/types/text-api.coffee

@@ -0,0 +1,32 @@
+# Text document API for text
+
+text = require './text' if typeof WEB is 'undefined'
+
+text.api =
+  provides: {text:true}
+
+  # The number of characters in the string
+  getLength: -> @snapshot.length
+
+  # Get the text contents of a document
+  getText: -> @snapshot
+
+  insert: (pos, text, callback) ->
+    op = [{p:pos, i:text}]
+    
+    @submitOp op, callback
+    op
+  
+  del: (pos, length, callback) ->
+    op = [{p:pos, d:@snapshot[pos...(pos + length)]}]
+
+    @submitOp op, callback
+    op
+  
+  _register: ->
+    @on 'remoteop', (op) ->
+      for component in op
+        if component.i != undefined
+          @emit 'insert', component.p, component.i
+        else
+          @emit 'delete', component.p, component.d

services/document-updater/app/coffee/sharejs/types/text-composable-api.coffee

@@ -0,0 +1,43 @@
+# Text document API for text
+
+if WEB?
+  type = exports.types['text-composable']
+else
+  type = require './text-composable'
+
+type.api =
+  provides: {'text':true}
+
+  # The number of characters in the string
+  'getLength': -> @snapshot.length
+
+  # Get the text contents of a document
+  'getText': -> @snapshot
+
+  'insert': (pos, text, callback) ->
+    op = type.normalize [pos, 'i':text, (@snapshot.length - pos)]
+    
+    @submitOp op, callback
+    op
+  
+  'del': (pos, length, callback) ->
+    op = type.normalize [pos, 'd':@snapshot[pos...(pos + length)], (@snapshot.length - pos - length)]
+
+    @submitOp op, callback
+    op
+
+  _register: ->
+    @on 'remoteop', (op) ->
+      pos = 0
+      for component in op
+        if typeof component is 'number'
+          pos += component
+        else if component.i != undefined
+          @emit 'insert', pos, component.i
+          pos += component.i.length
+        else
+          # delete
+          @emit 'delete', pos, component.d
+          # We don't increment pos, because the position
+          # specified is after the delete has happened.
+

services/document-updater/app/coffee/sharejs/types/text-composable.coffee

@@ -0,0 +1,261 @@
+# An alternate composable implementation for text. This is much closer
+# to the implementation used by google wave.
+#
+# Ops are lists of components which iterate over the whole document.
+# Components are either:
+#   A number N: Skip N characters in the original document
+#   {i:'str'}:  Insert 'str' at the current position in the document
+#   {d:'str'}:  Delete 'str', which appears at the current position in the document
+#
+# Eg: [3, {i:'hi'}, 5, {d:'internet'}]
+#
+# Snapshots are strings.
+
+p = -> #require('util').debug
+i = -> #require('util').inspect
+
+exports = if WEB? then {} else module.exports
+
+exports.name = 'text-composable'
+
+exports.create = -> ''
+
+# -------- Utility methods
+
+checkOp = (op) ->
+  throw new Error('Op must be an array of components') unless Array.isArray(op)
+  last = null
+  for c in op
+    if typeof(c) == 'object'
+      throw new Error("Invalid op component: #{i c}") unless (c.i? && c.i.length > 0) or (c.d? && c.d.length > 0)
+    else
+      throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number'
+      throw new Error('Skip components must be a positive number') unless c > 0
+      throw new Error('Adjacent skip components should be added') if typeof(last) == 'number'
+
+    last = c
+
+# Makes a function for appending components to a given op.
+# Exported for the randomOpGenerator.
+exports._makeAppend = makeAppend = (op) -> (component) ->
+  if component == 0 || component.i == '' || component.d == ''
+    return
+  else if op.length == 0
+    op.push component
+  else if typeof(component) == 'number' && typeof(op[op.length - 1]) == 'number'
+    op[op.length - 1] += component
+  else if component.i? && op[op.length - 1].i?
+    op[op.length - 1].i += component.i
+  else if component.d? && op[op.length - 1].d?
+    op[op.length - 1].d += component.d
+  else
+    op.push component
+  
+#  checkOp op
+
+# Makes 2 functions for taking components from the start of an op, and for peeking
+# at the next op that could be taken.
+makeTake = (op) ->
+  # The index of the next component to take
+  idx = 0
+  # The offset into the component
+  offset = 0
+
+  # Take up to length n from the front of op. If n is null, take the next
+  # op component. If indivisableField == 'd', delete components won't be separated.
+  # If indivisableField == 'i', insert components won't be separated.
+  take = (n, indivisableField) ->
+    return null if idx == op.length
+    #assert.notStrictEqual op.length, i, 'The op is too short to traverse the document'
+
+    if typeof(op[idx]) == 'number'
+      if !n? or op[idx] - offset <= n
+        c = op[idx] - offset
+        ++idx; offset = 0
+        c
+      else
+        offset += n
+        n
+    else
+      # Take from the string
+      field = if op[idx].i then 'i' else 'd'
+      c = {}
+      if !n? or op[idx][field].length - offset <= n or field == indivisableField
+        c[field] = op[idx][field][offset..]
+        ++idx; offset = 0
+      else
+        c[field] = op[idx][field][offset...(offset + n)]
+        offset += n
+      c
+  
+  peekType = () ->
+    op[idx]
+  
+  [take, peekType]
+
+# Find and return the length of an op component
+componentLength = (component) ->
+  if typeof(component) == 'number'
+    component
+  else if component.i?
+    component.i.length
+  else
+    component.d.length
+
+# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate
+# adjacent inserts and deletes.
+exports.normalize = (op) ->
+  newOp = []
+  append = makeAppend newOp
+  append component for component in op
+  newOp
+
+# Apply the op to the string. Returns the new string.
+exports.apply = (str, op) ->
+  p "Applying #{i op} to '#{str}'"
+  throw new Error('Snapshot should be a string') unless typeof(str) == 'string'
+  checkOp op
+
+  pos = 0
+  newDoc = []
+
+  for component in op
+    if typeof(component) == 'number'
+      throw new Error('The op is too long for this document') if component > str.length
+      newDoc.push str[...component]
+      str = str[component..]
+    else if component.i?
+      newDoc.push component.i
+    else
+      throw new Error("The deleted text '#{component.d}' doesn't match the next characters in the document '#{str[...component.d.length]}'") unless component.d == str[...component.d.length]
+      str = str[component.d.length..]
+  
+  throw new Error("The applied op doesn't traverse the entire document") unless '' == str
+
+  newDoc.join ''
+
+# transform op1 by op2. Return transformed version of op1.
+# op1 and op2 are unchanged by transform.
+exports.transform = (op, otherOp, side) ->
+  throw new Error "side (#{side} must be 'left' or 'right'" unless side == 'left' or side == 'right'
+
+  checkOp op
+  checkOp otherOp
+  newOp = []
+
+  append = makeAppend newOp
+  [take, peek] = makeTake op
+
+  for component in otherOp
+    if typeof(component) == 'number' # Skip
+      length = component
+      while length > 0
+        chunk = take(length, 'i')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append chunk
+        length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.i?
+    else if component.i? # Insert
+      if side == 'left'
+        # The left insert should go first.
+        o = peek()
+        append take() if o?.i
+
+      # Otherwise, skip the inserted text.
+      append(component.i.length)
+    else # Delete.
+      #assert.ok component.d
+      length = component.d.length
+      while length > 0
+        chunk = take(length, 'i')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        if typeof(chunk) == 'number'
+          length -= chunk
+        else if chunk.i?
+          append(chunk)
+        else
+          #assert.ok chunk.d
+          # The delete is unnecessary now.
+          length -= chunk.d.length
+  
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in the op: #{i component}" unless component?.i?
+    append component
+
+  newOp
+
+
+# Compose 2 ops into 1 op.
+exports.compose = (op1, op2) ->
+  p "COMPOSE #{i op1} + #{i op2}"
+  checkOp op1
+  checkOp op2
+
+  result = []
+
+  append = makeAppend result
+  [take, _] = makeTake op1
+
+  for component in op2
+    if typeof(component) == 'number' # Skip
+      length = component
+      while length > 0
+        chunk = take(length, 'd')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append chunk
+        length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.d?
+
+    else if component.i? # Insert
+      append {i:component.i}
+
+    else # Delete
+      offset = 0
+      while offset < component.d.length
+        chunk = take(component.d.length - offset, 'd')
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        # If its delete, append it. If its skip, drop it and decrease length. If its insert, check the strings match, drop it and decrease length.
+        if typeof(chunk) == 'number'
+          append {d:component.d[offset...(offset + chunk)]}
+          offset += chunk
+        else if chunk.i?
+          throw new Error("The deleted text doesn't match the inserted text") unless component.d[offset...(offset + chunk.i.length)] == chunk.i
+          offset += chunk.i.length
+          # The ops cancel each other out.
+        else
+          # Delete
+          append chunk
+    
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Trailing stuff in op1 #{i component}" unless component?.d?
+    append component
+
+  result
+  
+
+invertComponent = (c) ->
+  if typeof(c) == 'number'
+    c
+  else if c.i?
+    {d:c.i}
+  else
+    {i:c.d}
+
+# Invert an op
+exports.invert = (op) ->
+  result = []
+  append = makeAppend result
+
+  append(invertComponent component) for component in op
+  
+  result
+
+if window?
+  window.ot ||= {}
+  window.ot.types ||= {}
+  window.ot.types.text = exports
+

services/document-updater/app/coffee/sharejs/types/text-tp2-api.coffee

@@ -0,0 +1,89 @@
+# Text document API for text-tp2
+
+if WEB?
+  type = exports.types['text-tp2']
+else
+  type = require './text-tp2'
+
+{_takeDoc:takeDoc, _append:append} = type
+
+appendSkipChars = (op, doc, pos, maxlength) ->
+  while (maxlength == undefined || maxlength > 0) and pos.index < doc.data.length
+    part = takeDoc doc, pos, maxlength, true
+    maxlength -= part.length if maxlength != undefined and typeof part is 'string'
+    append op, (part.length || part)
+
+type['api'] =
+  'provides': {'text':true}
+
+  # The number of characters in the string
+  'getLength': -> @snapshot.charLength
+
+  # Flatten a document into a string
+  'getText': ->
+    strings = (elem for elem in @snapshot.data when typeof elem is 'string')
+    strings.join ''
+
+  'insert': (pos, text, callback) ->
+    pos = 0 if pos == undefined
+
+    op = []
+    docPos = {index:0, offset:0}
+
+    appendSkipChars op, @snapshot, docPos, pos
+    append op, {'i':text}
+    appendSkipChars op, @snapshot, docPos
+    
+    @submitOp op, callback
+    op
+  
+  'del': (pos, length, callback) ->
+    op = []
+    docPos = {index:0, offset:0}
+
+    appendSkipChars op, @snapshot, docPos, pos
+    
+    while length > 0
+      part = takeDoc @snapshot, docPos, length, true
+      if typeof part is 'string'
+        append op, {'d':part.length}
+        length -= part.length
+      else
+        append op, part
+    
+    appendSkipChars op, @snapshot, docPos
+
+    @submitOp op, callback
+    op
+
+  '_register': ->
+    # Interpret recieved ops + generate more detailed events for them
+    @on 'remoteop', (op, snapshot) ->
+      textPos = 0
+      docPos = {index:0, offset:0}
+
+      for component in op
+        if typeof component is 'number'
+          # Skip
+          remainder = component
+          while remainder > 0
+            part = takeDoc snapshot, docPos, remainder
+            if typeof part is 'string'
+              textPos += part.length
+            remainder -= part.length || part
+        else if component.i != undefined
+          # Insert
+          if typeof component.i is 'string'
+            @emit 'insert', textPos, component.i
+            textPos += component.i.length
+        else
+          # Delete
+          remainder = component.d
+          while remainder > 0
+            part = takeDoc snapshot, docPos, remainder
+            if typeof part is 'string'
+              @emit 'delete', textPos, part
+            remainder -= part.length || part
+
+      return
+

services/document-updater/app/coffee/sharejs/types/text-tp2.coffee

@@ -0,0 +1,322 @@
+# A TP2 implementation of text, following this spec:
+# http://code.google.com/p/lightwave/source/browse/trunk/experimental/ot/README
+#
+# A document is made up of a string and a set of tombstones inserted throughout
+# the string. For example, 'some ', (2 tombstones), 'string'.
+#
+# This is encoded in a document as: {s:'some string', t:[5, -2, 6]}
+#
+# Ops are lists of components which iterate over the whole document.
+# Components are either:
+#   N:         Skip N characters in the original document
+#   {i:'str'}: Insert 'str' at the current position in the document
+#   {i:N}:     Insert N tombstones at the current position in the document
+#   {d:N}:     Delete (tombstone) N characters at the current position in the document
+#
+# Eg: [3, {i:'hi'}, 5, {d:8}]
+#
+# Snapshots are lists with characters and tombstones. Characters are stored in strings
+# and adjacent tombstones are flattened into numbers.
+#
+# Eg, the document: 'Hello .....world' ('.' denotes tombstoned (deleted) characters)
+# would be represented by a document snapshot of ['Hello ', 5, 'world']
+
+type =
+  name: 'text-tp2'
+  tp2: true
+  create: -> {charLength:0, totalLength:0, positionCache:[], data:[]}
+  serialize: (doc) ->
+    throw new Error 'invalid doc snapshot' unless doc.data
+    doc.data
+  deserialize: (data) ->
+    doc = type.create()
+    doc.data = data
+    
+    for component in data
+      if typeof component is 'string'
+        doc.charLength += component.length
+        doc.totalLength += component.length
+      else
+        doc.totalLength += component
+    
+    doc
+
+
+checkOp = (op) ->
+  throw new Error('Op must be an array of components') unless Array.isArray(op)
+  last = null
+  for c in op
+    if typeof(c) == 'object'
+      if c.i != undefined
+        throw new Error('Inserts must insert a string or a +ive number') unless (typeof(c.i) == 'string' and c.i.length > 0) or (typeof(c.i) == 'number' and c.i > 0)
+      else if c.d != undefined
+        throw new Error('Deletes must be a +ive number') unless typeof(c.d) == 'number' and c.d > 0
+      else
+        throw new Error('Operation component must define .i or .d')
+    else
+      throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number'
+      throw new Error('Skip components must be a positive number') unless c > 0
+      throw new Error('Adjacent skip components should be combined') if typeof(last) == 'number'
+
+    last = c
+
+# Take the next part from the specified position in a document snapshot.
+# position = {index, offset}. It will be updated.
+type._takeDoc = takeDoc = (doc, position, maxlength, tombsIndivisible) ->
+  throw new Error 'Operation goes past the end of the document' if position.index >= doc.data.length
+
+  part = doc.data[position.index]
+  # peel off data[0]
+  result = if typeof(part) == 'string'
+    if maxlength != undefined
+      part[position.offset...(position.offset + maxlength)]
+    else
+      part[position.offset...]
+  else
+    if maxlength == undefined or tombsIndivisible
+      part - position.offset
+    else
+      Math.min(maxlength, part - position.offset)
+  
+  resultLen = result.length || result
+
+  if (part.length || part) - position.offset > resultLen
+    position.offset += resultLen
+  else
+    position.index++
+    position.offset = 0
+  
+  result
+
+# Append a part to the end of a document
+type._appendDoc = appendDoc = (doc, p) ->
+  return if p == 0 or p == ''
+
+  if typeof p is 'string'
+    doc.charLength += p.length
+    doc.totalLength += p.length
+  else
+    doc.totalLength += p
+
+  data = doc.data
+  if data.length == 0
+    data.push p
+  else if typeof(data[data.length - 1]) == typeof(p)
+    data[data.length - 1] += p
+  else
+    data.push p
+  return
+
+# Apply the op to the document. The document is not modified in the process.
+type.apply = (doc, op) ->
+  unless doc.totalLength != undefined and doc.charLength != undefined and doc.data.length != undefined
+    throw new Error('Snapshot is invalid')
+
+  checkOp op
+
+  newDoc = type.create()
+  position = {index:0, offset:0}
+
+  for component in op
+    if typeof(component) is 'number'
+      remainder = component
+      while remainder > 0
+        part = takeDoc doc, position, remainder
+        
+        appendDoc newDoc, part
+        remainder -= part.length || part
+
+    else if component.i != undefined
+      appendDoc newDoc, component.i
+    else if component.d != undefined
+      remainder = component.d
+      while remainder > 0
+        part = takeDoc doc, position, remainder
+        remainder -= part.length || part
+      appendDoc newDoc, component.d
+  
+  newDoc
+
+# Append an op component to the end of the specified op.
+# Exported for the randomOpGenerator.
+type._append = append = (op, component) ->
+  if component == 0 || component.i == '' || component.i == 0 || component.d == 0
+    return
+  else if op.length == 0
+    op.push component
+  else
+    last = op[op.length - 1]
+    if typeof(component) == 'number' && typeof(last) == 'number'
+      op[op.length - 1] += component
+    else if component.i != undefined && last.i? && typeof(last.i) == typeof(component.i)
+      last.i += component.i
+    else if component.d != undefined && last.d?
+      last.d += component.d
+    else
+      op.push component
+  
+# Makes 2 functions for taking components from the start of an op, and for peeking
+# at the next op that could be taken.
+makeTake = (op) ->
+  # The index of the next component to take
+  index = 0
+  # The offset into the component
+  offset = 0
+
+  # Take up to length maxlength from the op. If maxlength is not defined, there is no max.
+  # If insertsIndivisible is true, inserts (& insert tombstones) won't be separated.
+  #
+  # Returns null when op is fully consumed.
+  take = (maxlength, insertsIndivisible) ->
+    return null if index == op.length
+
+    e = op[index]
+    if typeof((current = e)) == 'number' or typeof((current = e.i)) == 'number' or (current = e.d) != undefined
+      if !maxlength? or current - offset <= maxlength or (insertsIndivisible and e.i != undefined)
+        # Return the rest of the current element.
+        c = current - offset
+        ++index; offset = 0
+      else
+        offset += maxlength
+        c = maxlength
+      if e.i != undefined then {i:c} else if e.d != undefined then {d:c} else c
+    else
+      # Take from the inserted string
+      if !maxlength? or e.i.length - offset <= maxlength or insertsIndivisible
+        result = {i:e.i[offset..]}
+        ++index; offset = 0
+      else
+        result = {i:e.i[offset...offset + maxlength]}
+        offset += maxlength
+      result
+  
+  peekType = -> op[index]
+  
+  [take, peekType]
+
+# Find and return the length of an op component
+componentLength = (component) ->
+  if typeof(component) == 'number'
+    component
+  else if typeof(component.i) == 'string'
+    component.i.length
+  else
+    # This should work because c.d and c.i must be +ive.
+    component.d or component.i
+
+# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate
+# adjacent inserts and deletes.
+type.normalize = (op) ->
+  newOp = []
+  append newOp, component for component in op
+  newOp
+
+# This is a helper method to transform and prune. goForwards is true for transform, false for prune.
+transformer = (op, otherOp, goForwards, side) ->
+  checkOp op
+  checkOp otherOp
+  newOp = []
+
+  [take, peek] = makeTake op
+
+  for component in otherOp
+    length = componentLength component
+
+    if component.i != undefined # Insert text or tombs
+      if goForwards # transform - insert skips over inserted parts
+        if side == 'left'
+          # The left insert should go first.
+          append newOp, take() while peek()?.i != undefined
+
+        # In any case, skip the inserted text.
+        append newOp, length
+
+      else # Prune. Remove skips for inserts.
+        while length > 0
+          chunk = take length, true
+
+          throw new Error 'The transformed op is invalid' unless chunk != null
+          throw new Error 'The transformed op deletes locally inserted characters - it cannot be purged of the insert.' if chunk.d != undefined
+
+          if typeof chunk is 'number'
+            length -= chunk
+          else
+            append newOp, chunk
+
+    else # Skip or delete
+      while length > 0
+        chunk = take length, true
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append newOp, chunk
+        length -= componentLength chunk unless chunk.i
+  
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in the op: #{component}" unless component.i != undefined
+    append newOp, component
+
+  newOp
+
+# transform op1 by op2. Return transformed version of op1.
+# op1 and op2 are unchanged by transform.
+# side should be 'left' or 'right', depending on if op1.id <> op2.id. 'left' == client op.
+type.transform = (op, otherOp, side) ->
+  throw new Error "side (#{side}) should be 'left' or 'right'" unless side == 'left' or side == 'right'
+  transformer op, otherOp, true, side
+
+# Prune is the inverse of transform.
+type.prune = (op, otherOp) -> transformer op, otherOp, false
+
+# Compose 2 ops into 1 op.
+type.compose = (op1, op2) ->
+  return op2 if op1 == null or op1 == undefined
+
+  checkOp op1
+  checkOp op2
+
+  result = []
+
+  [take, _] = makeTake op1
+
+  for component in op2
+
+    if typeof(component) == 'number' # Skip
+      # Just copy from op1.
+      length = component
+      while length > 0
+        chunk = take length
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        append result, chunk
+        length -= componentLength chunk
+
+    else if component.i != undefined # Insert
+      append result, {i:component.i}
+
+    else # Delete
+      length = component.d
+      while length > 0
+        chunk = take length
+        throw new Error('The op traverses more elements than the document has') unless chunk != null
+
+        chunkLength = componentLength chunk
+        if chunk.i != undefined
+          append result, {i:chunkLength}
+        else
+          append result, {d:chunkLength}
+
+        length -= chunkLength
+    
+  # Append extras from op1
+  while (component = take())
+    throw new Error "Remaining fragments in op1: #{component}" unless component.i != undefined
+    append result, component
+
+  result
+
+if WEB?
+  exports.types['text-tp2'] = type
+else
+  module.exports = type
+

services/document-updater/app/coffee/sharejs/types/text.coffee

@@ -0,0 +1,209 @@
+# A simple text implementation
+#
+# Operations are lists of components.
+# Each component either inserts or deletes at a specified position in the document.
+#
+# Components are either:
+#  {i:'str', p:100}: Insert 'str' at position 100 in the document
+#  {d:'str', p:100}: Delete 'str' at position 100 in the document
+#
+# Components in an operation are executed sequentially, so the position of components
+# assumes previous components have already executed.
+#
+# Eg: This op:
+#   [{i:'abc', p:0}]
+# is equivalent to this op:
+#   [{i:'a', p:0}, {i:'b', p:1}, {i:'c', p:2}]
+
+# NOTE: The global scope here is shared with other sharejs files when built with closure.
+# Be careful what ends up in your namespace.
+
+text = {}
+
+text.name = 'text'
+
+text.create = -> ''
+
+strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..]
+
+checkValidComponent = (c) ->
+  throw new Error 'component missing position field' if typeof c.p != 'number'
+
+  i_type = typeof c.i
+  d_type = typeof c.d
+  throw new Error 'component needs an i or d field' unless (i_type == 'string') ^ (d_type == 'string')
+
+  throw new Error 'position cannot be negative' unless c.p >= 0
+
+checkValidOp = (op) ->
+  checkValidComponent(c) for c in op
+  true
+
+text.apply = (snapshot, op) ->
+  checkValidOp op
+  for component in op
+    if component.i?
+      snapshot = strInject snapshot, component.p, component.i
+    else
+      deleted = snapshot[component.p...(component.p + component.d.length)]
+      throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted
+      snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..]
+  
+  snapshot
+
+
+# Exported for use by the random op generator.
+#
+# For simplicity, this version of append does not compress adjacent inserts and deletes of
+# the same text. It would be nice to change that at some stage.
+text._append = append = (newOp, c) ->
+  return if c.i == '' or c.d == ''
+  if newOp.length == 0
+    newOp.push c
+  else
+    last = newOp[newOp.length - 1]
+
+    # Compose the insert into the previous insert if possible
+    if last.i? && c.i? and last.p <= c.p <= (last.p + last.i.length)
+      newOp[newOp.length - 1] = {i:strInject(last.i, c.p - last.p, c.i), p:last.p}
+    else if last.d? && c.d? and c.p <= last.p <= (c.p + c.d.length)
+      newOp[newOp.length - 1] = {d:strInject(c.d, last.p - c.p, last.d), p:c.p}
+    else
+      newOp.push c
+
+text.compose = (op1, op2) ->
+  checkValidOp op1
+  checkValidOp op2
+
+  newOp = op1.slice()
+  append newOp, c for c in op2
+
+  newOp
+
+# Attempt to compress the op components together 'as much as possible'.
+# This implementation preserves order and preserves create/delete pairs.
+text.compress = (op) -> text.compose [], op
+
+text.normalize = (op) ->
+  newOp = []
+  
+  # Normalize should allow ops which are a single (unwrapped) component:
+  # {i:'asdf', p:23}.
+  # There's no good way to test if something is an array:
+  # http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/
+  # so this is probably the least bad solution.
+  op = [op] if op.i? or op.p?
+
+  for c in op
+    c.p ?= 0
+    append newOp, c
+  
+  newOp
+
+# This helper method transforms a position by an op component.
+#
+# If c is an insert, insertAfter specifies whether the transform
+# is pushed after the insert (true) or before it (false).
+#
+# insertAfter is optional for deletes.
+transformPosition = (pos, c, insertAfter) ->
+  if c.i?
+    if c.p < pos || (c.p == pos && insertAfter)
+      pos + c.i.length
+    else
+      pos
+  else
+    # I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length))
+    # but I think its harder to read that way, and it compiles using ternary operators anyway
+    # so its no slower written like this.
+    if pos <= c.p
+      pos
+    else if pos <= c.p + c.d.length
+      c.p
+    else
+      pos - c.d.length
+
+# Helper method to transform a cursor position as a result of an op.
+#
+# Like transformPosition above, if c is an insert, insertAfter specifies whether the cursor position
+# is pushed after an insert (true) or before it (false).
+text.transformCursor = (position, op, side) ->
+  insertAfter = side == 'right'
+  position = transformPosition position, c, insertAfter for c in op
+  position
+
+# Transform an op component by another op component. Asymmetric.
+# The result will be appended to destination.
+#
+# exported for use in JSON type
+text._tc = transformComponent = (dest, c, otherC, side) ->
+  checkValidOp [c]
+  checkValidOp [otherC]
+
+  if c.i?
+    append dest, {i:c.i, p:transformPosition(c.p, otherC, side == 'right')}
+
+  else # Delete
+    if otherC.i? # delete vs insert
+      s = c.d
+      if c.p < otherC.p
+        append dest, {d:s[...otherC.p - c.p], p:c.p}
+        s = s[(otherC.p - c.p)..]
+      if s != ''
+        append dest, {d:s, p:c.p + otherC.i.length}
+
+    else # Delete vs delete
+      if c.p >= otherC.p + otherC.d.length
+        append dest, {d:c.d, p:c.p - otherC.d.length}
+      else if c.p + c.d.length <= otherC.p
+        append dest, c
+      else
+        # They overlap somewhere.
+        newC = {d:'', p:c.p}
+        if c.p < otherC.p
+          newC.d = c.d[...(otherC.p - c.p)]
+        if c.p + c.d.length > otherC.p + otherC.d.length
+          newC.d += c.d[(otherC.p + otherC.d.length - c.p)..]
+
+        # This is entirely optional - just for a check that the deleted
+        # text in the two ops matches
+        intersectStart = Math.max c.p, otherC.p
+        intersectEnd = Math.min c.p + c.d.length, otherC.p + otherC.d.length
+        cIntersect = c.d[intersectStart - c.p...intersectEnd - c.p]
+        otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p]
+        throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect
+
+        if newC.d != ''
+          # This could be rewritten similarly to insert v delete, above.
+          newC.p = transformPosition newC.p, otherC
+          append dest, newC
+  
+  dest
+
+invertComponent = (c) ->
+  if c.i?
+    {d:c.i, p:c.p}
+  else
+    {i:c.d, p:c.p}
+
+# No need to use append for invert, because the components won't be able to
+# cancel with one another.
+text.invert = (op) -> (invertComponent c for c in op.slice().reverse())
+
+
+if WEB?
+  exports.types ||= {}
+
+  # This is kind of awful - come up with a better way to hook this helper code up.
+  bootstrapTransform(text, transformComponent, checkValidOp, append)
+
+  # [] is used to prevent closure from renaming types.text
+  exports.types.text = text
+else
+  module.exports = text
+
+  # The text type really shouldn't need this - it should be possible to define
+  # an efficient transform function by making a sort of transform map and passing each
+  # op component through it.
+  require('./helpers').bootstrapTransform(text, transformComponent, checkValidOp, append)
+

services/document-updater/app/coffee/sharejs/types/web-prelude.coffee

@@ -0,0 +1,11 @@
+# This is included at the top of each compiled type file for the web.
+
+`/**
+   @const
+   @type {boolean}
+*/
+var WEB = true;
+`
+
+exports = window['sharejs']
+

services/document-updater/app/coffee/sharejs/web-prelude.coffee

@@ -0,0 +1,11 @@
+# This is included at the top of each compiled type file for the web.
+
+`/**
+   @const
+   @type {boolean}
+*/
+var WEB = true;
+`
+
+exports = window['sharejs']
+

services/document-updater/app/lib/diff_match_patch.js

@@ -0,0 +1,2193 @@
+/**
+ * Diff Match and Patch
+ *
+ * Copyright 2006 Google Inc.
+ * http://code.google.com/p/google-diff-match-patch/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @fileoverview Computes the difference between two texts to create a patch.
+ * Applies the patch onto another text, allowing for errors.
+ * @author fraser@google.com (Neil Fraser)
+ */
+
+/**
+ * Class containing the diff, match and patch methods.
+ * @constructor
+ */
+function diff_match_patch() {
+
+  // Defaults.
+  // Redefine these in your program to override the defaults.
+
+  // Number of seconds to map a diff before giving up (0 for infinity).
+  this.Diff_Timeout = 1.0;
+  // Cost of an empty edit operation in terms of edit characters.
+  this.Diff_EditCost = 4;
+  // At what point is no match declared (0.0 = perfection, 1.0 = very loose).
+  this.Match_Threshold = 0.5;
+  // How far to search for a match (0 = exact location, 1000+ = broad match).
+  // A match this many characters away from the expected location will add
+  // 1.0 to the score (0.0 is a perfect match).
+  this.Match_Distance = 1000;
+  // When deleting a large block of text (over ~64 characters), how close do
+  // the contents have to be to match the expected contents. (0.0 = perfection,
+  // 1.0 = very loose).  Note that Match_Threshold controls how closely the
+  // end points of a delete need to match.
+  this.Patch_DeleteThreshold = 0.5;
+  // Chunk size for context length.
+  this.Patch_Margin = 4;
+
+  // The number of bits in an int.
+  this.Match_MaxBits = 32;
+}
+
+
+//  DIFF FUNCTIONS
+
+
+/**
+ * The data structure representing a diff is an array of tuples:
+ * [[DIFF_DELETE, 'Hello'], [DIFF_INSERT, 'Goodbye'], [DIFF_EQUAL, ' world.']]
+ * which means: delete 'Hello', add 'Goodbye' and keep ' world.'
+ */
+var DIFF_DELETE = -1;
+var DIFF_INSERT = 1;
+var DIFF_EQUAL = 0;
+
+/** @typedef {{0: number, 1: string}} */
+diff_match_patch.Diff;
+
+
+/**
+ * Find the differences between two texts.  Simplifies the problem by stripping
+ * any common prefix or suffix off the texts before diffing.
+ * @param {string} text1 Old string to be diffed.
+ * @param {string} text2 New string to be diffed.
+ * @param {boolean=} opt_checklines Optional speedup flag. If present and false,
+ *     then don't run a line-level diff first to identify the changed areas.
+ *     Defaults to true, which does a faster, slightly less optimal diff.
+ * @param {number} opt_deadline Optional time when the diff should be complete
+ *     by.  Used internally for recursive calls.  Users should set DiffTimeout
+ *     instead.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ */
+diff_match_patch.prototype.diff_main = function(text1, text2, opt_checklines,
+    opt_deadline) {
+  // Set a deadline by which time the diff must be complete.
+  if (typeof opt_deadline == 'undefined') {
+    if (this.Diff_Timeout <= 0) {
+      opt_deadline = Number.MAX_VALUE;
+    } else {
+      opt_deadline = (new Date).getTime() + this.Diff_Timeout * 1000;
+    }
+  }
+  var deadline = opt_deadline;
+
+  // Check for null inputs.
+  if (text1 == null || text2 == null) {
+    throw new Error('Null input. (diff_main)');
+  }
+
+  // Check for equality (speedup).
+  if (text1 == text2) {
+    if (text1) {
+      return [[DIFF_EQUAL, text1]];
+    }
+    return [];
+  }
+
+  if (typeof opt_checklines == 'undefined') {
+    opt_checklines = true;
+  }
+  var checklines = opt_checklines;
+
+  // Trim off common prefix (speedup).
+  var commonlength = this.diff_commonPrefix(text1, text2);
+  var commonprefix = text1.substring(0, commonlength);
+  text1 = text1.substring(commonlength);
+  text2 = text2.substring(commonlength);
+
+  // Trim off common suffix (speedup).
+  commonlength = this.diff_commonSuffix(text1, text2);
+  var commonsuffix = text1.substring(text1.length - commonlength);
+  text1 = text1.substring(0, text1.length - commonlength);
+  text2 = text2.substring(0, text2.length - commonlength);
+
+  // Compute the diff on the middle block.
+  var diffs = this.diff_compute_(text1, text2, checklines, deadline);
+
+  // Restore the prefix and suffix.
+  if (commonprefix) {
+    diffs.unshift([DIFF_EQUAL, commonprefix]);
+  }
+  if (commonsuffix) {
+    diffs.push([DIFF_EQUAL, commonsuffix]);
+  }
+  this.diff_cleanupMerge(diffs);
+  return diffs;
+};
+
+
+/**
+ * Find the differences between two texts.  Assumes that the texts do not
+ * have any common prefix or suffix.
+ * @param {string} text1 Old string to be diffed.
+ * @param {string} text2 New string to be diffed.
+ * @param {boolean} checklines Speedup flag.  If false, then don't run a
+ *     line-level diff first to identify the changed areas.
+ *     If true, then run a faster, slightly less optimal diff.
+ * @param {number} deadline Time when the diff should be complete by.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ * @private
+ */
+diff_match_patch.prototype.diff_compute_ = function(text1, text2, checklines,
+    deadline) {
+  var diffs;
+
+  if (!text1) {
+    // Just add some text (speedup).
+    return [[DIFF_INSERT, text2]];
+  }
+
+  if (!text2) {
+    // Just delete some text (speedup).
+    return [[DIFF_DELETE, text1]];
+  }
+
+  var longtext = text1.length > text2.length ? text1 : text2;
+  var shorttext = text1.length > text2.length ? text2 : text1;
+  var i = longtext.indexOf(shorttext);
+  if (i != -1) {
+    // Shorter text is inside the longer text (speedup).
+    diffs = [[DIFF_INSERT, longtext.substring(0, i)],
+             [DIFF_EQUAL, shorttext],
+             [DIFF_INSERT, longtext.substring(i + shorttext.length)]];
+    // Swap insertions for deletions if diff is reversed.
+    if (text1.length > text2.length) {
+      diffs[0][0] = diffs[2][0] = DIFF_DELETE;
+    }
+    return diffs;
+  }
+
+  if (shorttext.length == 1) {
+    // Single character string.
+    // After the previous speedup, the character can't be an equality.
+    return [[DIFF_DELETE, text1], [DIFF_INSERT, text2]];
+  }
+
+  // Check to see if the problem can be split in two.
+  var hm = this.diff_halfMatch_(text1, text2);
+  if (hm) {
+    // A half-match was found, sort out the return data.
+    var text1_a = hm[0];
+    var text1_b = hm[1];
+    var text2_a = hm[2];
+    var text2_b = hm[3];
+    var mid_common = hm[4];
+    // Send both pairs off for separate processing.
+    var diffs_a = this.diff_main(text1_a, text2_a, checklines, deadline);
+    var diffs_b = this.diff_main(text1_b, text2_b, checklines, deadline);
+    // Merge the results.
+    return diffs_a.concat([[DIFF_EQUAL, mid_common]], diffs_b);
+  }
+
+  if (checklines && text1.length > 100 && text2.length > 100) {
+    return this.diff_lineMode_(text1, text2, deadline);
+  }
+
+  return this.diff_bisect_(text1, text2, deadline);
+};
+
+
+/**
+ * Do a quick line-level diff on both strings, then rediff the parts for
+ * greater accuracy.
+ * This speedup can produce non-minimal diffs.
+ * @param {string} text1 Old string to be diffed.
+ * @param {string} text2 New string to be diffed.
+ * @param {number} deadline Time when the diff should be complete by.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ * @private
+ */
+diff_match_patch.prototype.diff_lineMode_ = function(text1, text2, deadline) {
+  // Scan the text on a line-by-line basis first.
+  var a = this.diff_linesToChars_(text1, text2);
+  text1 = a.chars1;
+  text2 = a.chars2;
+  var linearray = a.lineArray;
+
+  var diffs = this.diff_main(text1, text2, false, deadline);
+
+  // Convert the diff back to original text.
+  this.diff_charsToLines_(diffs, linearray);
+  // Eliminate freak matches (e.g. blank lines)
+  this.diff_cleanupSemantic(diffs);
+
+  // Rediff any replacement blocks, this time character-by-character.
+  // Add a dummy entry at the end.
+  diffs.push([DIFF_EQUAL, '']);
+  var pointer = 0;
+  var count_delete = 0;
+  var count_insert = 0;
+  var text_delete = '';
+  var text_insert = '';
+  while (pointer < diffs.length) {
+    switch (diffs[pointer][0]) {
+      case DIFF_INSERT:
+        count_insert++;
+        text_insert += diffs[pointer][1];
+        break;
+      case DIFF_DELETE:
+        count_delete++;
+        text_delete += diffs[pointer][1];
+        break;
+      case DIFF_EQUAL:
+        // Upon reaching an equality, check for prior redundancies.
+        if (count_delete >= 1 && count_insert >= 1) {
+          // Delete the offending records and add the merged ones.
+          diffs.splice(pointer - count_delete - count_insert,
+                       count_delete + count_insert);
+          pointer = pointer - count_delete - count_insert;
+          var a = this.diff_main(text_delete, text_insert, false, deadline);
+          for (var j = a.length - 1; j >= 0; j--) {
+            diffs.splice(pointer, 0, a[j]);
+          }
+          pointer = pointer + a.length;
+        }
+        count_insert = 0;
+        count_delete = 0;
+        text_delete = '';
+        text_insert = '';
+        break;
+    }
+    pointer++;
+  }
+  diffs.pop();  // Remove the dummy entry at the end.
+
+  return diffs;
+};
+
+
+/**
+ * Find the 'middle snake' of a diff, split the problem in two
+ * and return the recursively constructed diff.
+ * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.
+ * @param {string} text1 Old string to be diffed.
+ * @param {string} text2 New string to be diffed.
+ * @param {number} deadline Time at which to bail if not yet complete.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ * @private
+ */
+diff_match_patch.prototype.diff_bisect_ = function(text1, text2, deadline) {
+  // Cache the text lengths to prevent multiple calls.
+  var text1_length = text1.length;
+  var text2_length = text2.length;
+  var max_d = Math.ceil((text1_length + text2_length) / 2);
+  var v_offset = max_d;
+  var v_length = 2 * max_d;
+  var v1 = new Array(v_length);
+  var v2 = new Array(v_length);
+  // Setting all elements to -1 is faster in Chrome & Firefox than mixing
+  // integers and undefined.
+  for (var x = 0; x < v_length; x++) {
+    v1[x] = -1;
+    v2[x] = -1;
+  }
+  v1[v_offset + 1] = 0;
+  v2[v_offset + 1] = 0;
+  var delta = text1_length - text2_length;
+  // If the total number of characters is odd, then the front path will collide
+  // with the reverse path.
+  var front = (delta % 2 != 0);
+  // Offsets for start and end of k loop.
+  // Prevents mapping of space beyond the grid.
+  var k1start = 0;
+  var k1end = 0;
+  var k2start = 0;
+  var k2end = 0;
+  for (var d = 0; d < max_d; d++) {
+    // Bail out if deadline is reached.
+    if ((new Date()).getTime() > deadline) {
+      break;
+    }
+
+    // Walk the front path one step.
+    for (var k1 = -d + k1start; k1 <= d - k1end; k1 += 2) {
+      var k1_offset = v_offset + k1;
+      var x1;
+      if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) {
+        x1 = v1[k1_offset + 1];
+      } else {
+        x1 = v1[k1_offset - 1] + 1;
+      }
+      var y1 = x1 - k1;
+      while (x1 < text1_length && y1 < text2_length &&
+             text1.charAt(x1) == text2.charAt(y1)) {
+        x1++;
+        y1++;
+      }
+      v1[k1_offset] = x1;
+      if (x1 > text1_length) {
+        // Ran off the right of the graph.
+        k1end += 2;
+      } else if (y1 > text2_length) {
+        // Ran off the bottom of the graph.
+        k1start += 2;
+      } else if (front) {
+        var k2_offset = v_offset + delta - k1;
+        if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) {
+          // Mirror x2 onto top-left coordinate system.
+          var x2 = text1_length - v2[k2_offset];
+          if (x1 >= x2) {
+            // Overlap detected.
+            return this.diff_bisectSplit_(text1, text2, x1, y1, deadline);
+          }
+        }
+      }
+    }
+
+    // Walk the reverse path one step.
+    for (var k2 = -d + k2start; k2 <= d - k2end; k2 += 2) {
+      var k2_offset = v_offset + k2;
+      var x2;
+      if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) {
+        x2 = v2[k2_offset + 1];
+      } else {
+        x2 = v2[k2_offset - 1] + 1;
+      }
+      var y2 = x2 - k2;
+      while (x2 < text1_length && y2 < text2_length &&
+             text1.charAt(text1_length - x2 - 1) ==
+             text2.charAt(text2_length - y2 - 1)) {
+        x2++;
+        y2++;
+      }
+      v2[k2_offset] = x2;
+      if (x2 > text1_length) {
+        // Ran off the left of the graph.
+        k2end += 2;
+      } else if (y2 > text2_length) {
+        // Ran off the top of the graph.
+        k2start += 2;
+      } else if (!front) {
+        var k1_offset = v_offset + delta - k2;
+        if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) {
+          var x1 = v1[k1_offset];
+          var y1 = v_offset + x1 - k1_offset;
+          // Mirror x2 onto top-left coordinate system.
+          x2 = text1_length - x2;
+          if (x1 >= x2) {
+            // Overlap detected.
+            return this.diff_bisectSplit_(text1, text2, x1, y1, deadline);
+          }
+        }
+      }
+    }
+  }
+  // Diff took too long and hit the deadline or
+  // number of diffs equals number of characters, no commonality at all.
+  return [[DIFF_DELETE, text1], [DIFF_INSERT, text2]];
+};
+
+
+/**
+ * Given the location of the 'middle snake', split the diff in two parts
+ * and recurse.
+ * @param {string} text1 Old string to be diffed.
+ * @param {string} text2 New string to be diffed.
+ * @param {number} x Index of split point in text1.
+ * @param {number} y Index of split point in text2.
+ * @param {number} deadline Time at which to bail if not yet complete.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ * @private
+ */
+diff_match_patch.prototype.diff_bisectSplit_ = function(text1, text2, x, y,
+    deadline) {
+  var text1a = text1.substring(0, x);
+  var text2a = text2.substring(0, y);
+  var text1b = text1.substring(x);
+  var text2b = text2.substring(y);
+
+  // Compute both diffs serially.
+  var diffs = this.diff_main(text1a, text2a, false, deadline);
+  var diffsb = this.diff_main(text1b, text2b, false, deadline);
+
+  return diffs.concat(diffsb);
+};
+
+
+/**
+ * Split two texts into an array of strings.  Reduce the texts to a string of
+ * hashes where each Unicode character represents one line.
+ * @param {string} text1 First string.
+ * @param {string} text2 Second string.
+ * @return {{chars1: string, chars2: string, lineArray: !Array.<string>}}
+ *     An object containing the encoded text1, the encoded text2 and
+ *     the array of unique strings.
+ *     The zeroth element of the array of unique strings is intentionally blank.
+ * @private
+ */
+diff_match_patch.prototype.diff_linesToChars_ = function(text1, text2) {
+  var lineArray = [];  // e.g. lineArray[4] == 'Hello\n'
+  var lineHash = {};   // e.g. lineHash['Hello\n'] == 4
+
+  // '\x00' is a valid character, but various debuggers don't like it.
+  // So we'll insert a junk entry to avoid generating a null character.
+  lineArray[0] = '';
+
+  /**
+   * Split a text into an array of strings.  Reduce the texts to a string of
+   * hashes where each Unicode character represents one line.
+   * Modifies linearray and linehash through being a closure.
+   * @param {string} text String to encode.
+   * @return {string} Encoded string.
+   * @private
+   */
+  function diff_linesToCharsMunge_(text) {
+    var chars = '';
+    // Walk the text, pulling out a substring for each line.
+    // text.split('\n') would would temporarily double our memory footprint.
+    // Modifying text would create many large strings to garbage collect.
+    var lineStart = 0;
+    var lineEnd = -1;
+    // Keeping our own length variable is faster than looking it up.
+    var lineArrayLength = lineArray.length;
+    while (lineEnd < text.length - 1) {
+      lineEnd = text.indexOf('\n', lineStart);
+      if (lineEnd == -1) {
+        lineEnd = text.length - 1;
+      }
+      var line = text.substring(lineStart, lineEnd + 1);
+      lineStart = lineEnd + 1;
+
+      if (lineHash.hasOwnProperty ? lineHash.hasOwnProperty(line) :
+          (lineHash[line] !== undefined)) {
+        chars += String.fromCharCode(lineHash[line]);
+      } else {
+        chars += String.fromCharCode(lineArrayLength);
+        lineHash[line] = lineArrayLength;
+        lineArray[lineArrayLength++] = line;
+      }
+    }
+    return chars;
+  }
+
+  var chars1 = diff_linesToCharsMunge_(text1);
+  var chars2 = diff_linesToCharsMunge_(text2);
+  return {chars1: chars1, chars2: chars2, lineArray: lineArray};
+};
+
+
+/**
+ * Rehydrate the text in a diff from a string of line hashes to real lines of
+ * text.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @param {!Array.<string>} lineArray Array of unique strings.
+ * @private
+ */
+diff_match_patch.prototype.diff_charsToLines_ = function(diffs, lineArray) {
+  for (var x = 0; x < diffs.length; x++) {
+    var chars = diffs[x][1];
+    var text = [];
+    for (var y = 0; y < chars.length; y++) {
+      text[y] = lineArray[chars.charCodeAt(y)];
+    }
+    diffs[x][1] = text.join('');
+  }
+};
+
+
+/**
+ * Determine the common prefix of two strings.
+ * @param {string} text1 First string.
+ * @param {string} text2 Second string.
+ * @return {number} The number of characters common to the start of each
+ *     string.
+ */
+diff_match_patch.prototype.diff_commonPrefix = function(text1, text2) {
+  // Quick check for common null cases.
+  if (!text1 || !text2 || text1.charAt(0) != text2.charAt(0)) {
+    return 0;
+  }
+  // Binary search.
+  // Performance analysis: http://neil.fraser.name/news/2007/10/09/
+  var pointermin = 0;
+  var pointermax = Math.min(text1.length, text2.length);
+  var pointermid = pointermax;
+  var pointerstart = 0;
+  while (pointermin < pointermid) {
+    if (text1.substring(pointerstart, pointermid) ==
+        text2.substring(pointerstart, pointermid)) {
+      pointermin = pointermid;
+      pointerstart = pointermin;
+    } else {
+      pointermax = pointermid;
+    }
+    pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin);
+  }
+  return pointermid;
+};
+
+
+/**
+ * Determine the common suffix of two strings.
+ * @param {string} text1 First string.
+ * @param {string} text2 Second string.
+ * @return {number} The number of characters common to the end of each string.
+ */
+diff_match_patch.prototype.diff_commonSuffix = function(text1, text2) {
+  // Quick check for common null cases.
+  if (!text1 || !text2 ||
+      text1.charAt(text1.length - 1) != text2.charAt(text2.length - 1)) {
+    return 0;
+  }
+  // Binary search.
+  // Performance analysis: http://neil.fraser.name/news/2007/10/09/
+  var pointermin = 0;
+  var pointermax = Math.min(text1.length, text2.length);
+  var pointermid = pointermax;
+  var pointerend = 0;
+  while (pointermin < pointermid) {
+    if (text1.substring(text1.length - pointermid, text1.length - pointerend) ==
+        text2.substring(text2.length - pointermid, text2.length - pointerend)) {
+      pointermin = pointermid;
+      pointerend = pointermin;
+    } else {
+      pointermax = pointermid;
+    }
+    pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin);
+  }
+  return pointermid;
+};
+
+
+/**
+ * Determine if the suffix of one string is the prefix of another.
+ * @param {string} text1 First string.
+ * @param {string} text2 Second string.
+ * @return {number} The number of characters common to the end of the first
+ *     string and the start of the second string.
+ * @private
+ */
+diff_match_patch.prototype.diff_commonOverlap_ = function(text1, text2) {
+  // Cache the text lengths to prevent multiple calls.
+  var text1_length = text1.length;
+  var text2_length = text2.length;
+  // Eliminate the null case.
+  if (text1_length == 0 || text2_length == 0) {
+    return 0;
+  }
+  // Truncate the longer string.
+  if (text1_length > text2_length) {
+    text1 = text1.substring(text1_length - text2_length);
+  } else if (text1_length < text2_length) {
+    text2 = text2.substring(0, text1_length);
+  }
+  var text_length = Math.min(text1_length, text2_length);
+  // Quick check for the worst case.
+  if (text1 == text2) {
+    return text_length;
+  }
+
+  // Start by looking for a single character match
+  // and increase length until no match is found.
+  // Performance analysis: http://neil.fraser.name/news/2010/11/04/
+  var best = 0;
+  var length = 1;
+  while (true) {
+    var pattern = text1.substring(text_length - length);
+    var found = text2.indexOf(pattern);
+    if (found == -1) {
+      return best;
+    }
+    length += found;
+    if (found == 0 || text1.substring(text_length - length) ==
+        text2.substring(0, length)) {
+      best = length;
+      length++;
+    }
+  }
+};
+
+
+/**
+ * Do the two texts share a substring which is at least half the length of the
+ * longer text?
+ * This speedup can produce non-minimal diffs.
+ * @param {string} text1 First string.
+ * @param {string} text2 Second string.
+ * @return {Array.<string>} Five element Array, containing the prefix of
+ *     text1, the suffix of text1, the prefix of text2, the suffix of
+ *     text2 and the common middle.  Or null if there was no match.
+ * @private
+ */
+diff_match_patch.prototype.diff_halfMatch_ = function(text1, text2) {
+  if (this.Diff_Timeout <= 0) {
+    // Don't risk returning a non-optimal diff if we have unlimited time.
+    return null;
+  }
+  var longtext = text1.length > text2.length ? text1 : text2;
+  var shorttext = text1.length > text2.length ? text2 : text1;
+  if (longtext.length < 4 || shorttext.length * 2 < longtext.length) {
+    return null;  // Pointless.
+  }
+  var dmp = this;  // 'this' becomes 'window' in a closure.
+
+  /**
+   * Does a substring of shorttext exist within longtext such that the substring
+   * is at least half the length of longtext?
+   * Closure, but does not reference any external variables.
+   * @param {string} longtext Longer string.
+   * @param {string} shorttext Shorter string.
+   * @param {number} i Start index of quarter length substring within longtext.
+   * @return {Array.<string>} Five element Array, containing the prefix of
+   *     longtext, the suffix of longtext, the prefix of shorttext, the suffix
+   *     of shorttext and the common middle.  Or null if there was no match.
+   * @private
+   */
+  function diff_halfMatchI_(longtext, shorttext, i) {
+    // Start with a 1/4 length substring at position i as a seed.
+    var seed = longtext.substring(i, i + Math.floor(longtext.length / 4));
+    var j = -1;
+    var best_common = '';
+    var best_longtext_a, best_longtext_b, best_shorttext_a, best_shorttext_b;
+    while ((j = shorttext.indexOf(seed, j + 1)) != -1) {
+      var prefixLength = dmp.diff_commonPrefix(longtext.substring(i),
+                                               shorttext.substring(j));
+      var suffixLength = dmp.diff_commonSuffix(longtext.substring(0, i),
+                                               shorttext.substring(0, j));
+      if (best_common.length < suffixLength + prefixLength) {
+        best_common = shorttext.substring(j - suffixLength, j) +
+            shorttext.substring(j, j + prefixLength);
+        best_longtext_a = longtext.substring(0, i - suffixLength);
+        best_longtext_b = longtext.substring(i + prefixLength);
+        best_shorttext_a = shorttext.substring(0, j - suffixLength);
+        best_shorttext_b = shorttext.substring(j + prefixLength);
+      }
+    }
+    if (best_common.length * 2 >= longtext.length) {
+      return [best_longtext_a, best_longtext_b,
+              best_shorttext_a, best_shorttext_b, best_common];
+    } else {
+      return null;
+    }
+  }
+
+  // First check if the second quarter is the seed for a half-match.
+  var hm1 = diff_halfMatchI_(longtext, shorttext,
+                             Math.ceil(longtext.length / 4));
+  // Check again based on the third quarter.
+  var hm2 = diff_halfMatchI_(longtext, shorttext,
+                             Math.ceil(longtext.length / 2));
+  var hm;
+  if (!hm1 && !hm2) {
+    return null;
+  } else if (!hm2) {
+    hm = hm1;
+  } else if (!hm1) {
+    hm = hm2;
+  } else {
+    // Both matched.  Select the longest.
+    hm = hm1[4].length > hm2[4].length ? hm1 : hm2;
+  }
+
+  // A half-match was found, sort out the return data.
+  var text1_a, text1_b, text2_a, text2_b;
+  if (text1.length > text2.length) {
+    text1_a = hm[0];
+    text1_b = hm[1];
+    text2_a = hm[2];
+    text2_b = hm[3];
+  } else {
+    text2_a = hm[0];
+    text2_b = hm[1];
+    text1_a = hm[2];
+    text1_b = hm[3];
+  }
+  var mid_common = hm[4];
+  return [text1_a, text1_b, text2_a, text2_b, mid_common];
+};
+
+
+/**
+ * Reduce the number of edits by eliminating semantically trivial equalities.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ */
+diff_match_patch.prototype.diff_cleanupSemantic = function(diffs) {
+  var changes = false;
+  var equalities = [];  // Stack of indices where equalities are found.
+  var equalitiesLength = 0;  // Keeping our own length var is faster in JS.
+  /** @type {?string} */
+  var lastequality = null;
+  // Always equal to diffs[equalities[equalitiesLength - 1]][1]
+  var pointer = 0;  // Index of current position.
+  // Number of characters that changed prior to the equality.
+  var length_insertions1 = 0;
+  var length_deletions1 = 0;
+  // Number of characters that changed after the equality.
+  var length_insertions2 = 0;
+  var length_deletions2 = 0;
+  while (pointer < diffs.length) {
+    if (diffs[pointer][0] == DIFF_EQUAL) {  // Equality found.
+      equalities[equalitiesLength++] = pointer;
+      length_insertions1 = length_insertions2;
+      length_deletions1 = length_deletions2;
+      length_insertions2 = 0;
+      length_deletions2 = 0;
+      lastequality = diffs[pointer][1];
+    } else {  // An insertion or deletion.
+      if (diffs[pointer][0] == DIFF_INSERT) {
+        length_insertions2 += diffs[pointer][1].length;
+      } else {
+        length_deletions2 += diffs[pointer][1].length;
+      }
+      // Eliminate an equality that is smaller or equal to the edits on both
+      // sides of it.
+      if (lastequality && (lastequality.length <=
+          Math.max(length_insertions1, length_deletions1)) &&
+          (lastequality.length <= Math.max(length_insertions2,
+                                           length_deletions2))) {
+        // Duplicate record.
+        diffs.splice(equalities[equalitiesLength - 1], 0,
+                     [DIFF_DELETE, lastequality]);
+        // Change second copy to insert.
+        diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT;
+        // Throw away the equality we just deleted.
+        equalitiesLength--;
+        // Throw away the previous equality (it needs to be reevaluated).
+        equalitiesLength--;
+        pointer = equalitiesLength > 0 ? equalities[equalitiesLength - 1] : -1;
+        length_insertions1 = 0;  // Reset the counters.
+        length_deletions1 = 0;
+        length_insertions2 = 0;
+        length_deletions2 = 0;
+        lastequality = null;
+        changes = true;
+      }
+    }
+    pointer++;
+  }
+
+  // Normalize the diff.
+  if (changes) {
+    this.diff_cleanupMerge(diffs);
+  }
+  this.diff_cleanupSemanticLossless(diffs);
+
+  // Find any overlaps between deletions and insertions.
+  // e.g: <del>abcxxx</del><ins>xxxdef</ins>
+  //   -> <del>abc</del>xxx<ins>def</ins>
+  // e.g: <del>xxxabc</del><ins>defxxx</ins>
+  //   -> <ins>def</ins>xxx<del>abc</del>
+  // Only extract an overlap if it is as big as the edit ahead or behind it.
+  pointer = 1;
+  while (pointer < diffs.length) {
+    if (diffs[pointer - 1][0] == DIFF_DELETE &&
+        diffs[pointer][0] == DIFF_INSERT) {
+      var deletion = diffs[pointer - 1][1];
+      var insertion = diffs[pointer][1];
+      var overlap_length1 = this.diff_commonOverlap_(deletion, insertion);
+      var overlap_length2 = this.diff_commonOverlap_(insertion, deletion);
+      if (overlap_length1 >= overlap_length2) {
+        if (overlap_length1 >= deletion.length / 2 ||
+            overlap_length1 >= insertion.length / 2) {
+          // Overlap found.  Insert an equality and trim the surrounding edits.
+          diffs.splice(pointer, 0,
+              [DIFF_EQUAL, insertion.substring(0, overlap_length1)]);
+          diffs[pointer - 1][1] =
+              deletion.substring(0, deletion.length - overlap_length1);
+          diffs[pointer + 1][1] = insertion.substring(overlap_length1);
+          pointer++;
+        }
+      } else {
+        if (overlap_length2 >= deletion.length / 2 ||
+            overlap_length2 >= insertion.length / 2) {
+          // Reverse overlap found.
+          // Insert an equality and swap and trim the surrounding edits.
+          diffs.splice(pointer, 0,
+              [DIFF_EQUAL, deletion.substring(0, overlap_length2)]);
+          diffs[pointer - 1][0] = DIFF_INSERT;
+          diffs[pointer - 1][1] =
+              insertion.substring(0, insertion.length - overlap_length2);
+          diffs[pointer + 1][0] = DIFF_DELETE;
+          diffs[pointer + 1][1] =
+              deletion.substring(overlap_length2);
+          pointer++;
+        }
+      }
+      pointer++;
+    }
+    pointer++;
+  }
+};
+
+
+/**
+ * Look for single edits surrounded on both sides by equalities
+ * which can be shifted sideways to align the edit to a word boundary.
+ * e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ */
+diff_match_patch.prototype.diff_cleanupSemanticLossless = function(diffs) {
+  /**
+   * Given two strings, compute a score representing whether the internal
+   * boundary falls on logical boundaries.
+   * Scores range from 6 (best) to 0 (worst).
+   * Closure, but does not reference any external variables.
+   * @param {string} one First string.
+   * @param {string} two Second string.
+   * @return {number} The score.
+   * @private
+   */
+  function diff_cleanupSemanticScore_(one, two) {
+    if (!one || !two) {
+      // Edges are the best.
+      return 6;
+    }
+
+    // Each port of this function behaves slightly differently due to
+    // subtle differences in each language's definition of things like
+    // 'whitespace'.  Since this function's purpose is largely cosmetic,
+    // the choice has been made to use each language's native features
+    // rather than force total conformity.
+    var char1 = one.charAt(one.length - 1);
+    var char2 = two.charAt(0);
+    var nonAlphaNumeric1 = char1.match(diff_match_patch.nonAlphaNumericRegex_);
+    var nonAlphaNumeric2 = char2.match(diff_match_patch.nonAlphaNumericRegex_);
+    var whitespace1 = nonAlphaNumeric1 &&
+        char1.match(diff_match_patch.whitespaceRegex_);
+    var whitespace2 = nonAlphaNumeric2 &&
+        char2.match(diff_match_patch.whitespaceRegex_);
+    var lineBreak1 = whitespace1 &&
+        char1.match(diff_match_patch.linebreakRegex_);
+    var lineBreak2 = whitespace2 &&
+        char2.match(diff_match_patch.linebreakRegex_);
+    var blankLine1 = lineBreak1 &&
+        one.match(diff_match_patch.blanklineEndRegex_);
+    var blankLine2 = lineBreak2 &&
+        two.match(diff_match_patch.blanklineStartRegex_);
+
+    if (blankLine1 || blankLine2) {
+      // Five points for blank lines.
+      return 5;
+    } else if (lineBreak1 || lineBreak2) {
+      // Four points for line breaks.
+      return 4;
+    } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) {
+      // Three points for end of sentences.
+      return 3;
+    } else if (whitespace1 || whitespace2) {
+      // Two points for whitespace.
+      return 2;
+    } else if (nonAlphaNumeric1 || nonAlphaNumeric2) {
+      // One point for non-alphanumeric.
+      return 1;
+    }
+    return 0;
+  }
+
+  var pointer = 1;
+  // Intentionally ignore the first and last element (don't need checking).
+  while (pointer < diffs.length - 1) {
+    if (diffs[pointer - 1][0] == DIFF_EQUAL &&
+        diffs[pointer + 1][0] == DIFF_EQUAL) {
+      // This is a single edit surrounded by equalities.
+      var equality1 = diffs[pointer - 1][1];
+      var edit = diffs[pointer][1];
+      var equality2 = diffs[pointer + 1][1];
+
+      // First, shift the edit as far left as possible.
+      var commonOffset = this.diff_commonSuffix(equality1, edit);
+      if (commonOffset) {
+        var commonString = edit.substring(edit.length - commonOffset);
+        equality1 = equality1.substring(0, equality1.length - commonOffset);
+        edit = commonString + edit.substring(0, edit.length - commonOffset);
+        equality2 = commonString + equality2;
+      }
+
+      // Second, step character by character right, looking for the best fit.
+      var bestEquality1 = equality1;
+      var bestEdit = edit;
+      var bestEquality2 = equality2;
+      var bestScore = diff_cleanupSemanticScore_(equality1, edit) +
+          diff_cleanupSemanticScore_(edit, equality2);
+      while (edit.charAt(0) === equality2.charAt(0)) {
+        equality1 += edit.charAt(0);
+        edit = edit.substring(1) + equality2.charAt(0);
+        equality2 = equality2.substring(1);
+        var score = diff_cleanupSemanticScore_(equality1, edit) +
+            diff_cleanupSemanticScore_(edit, equality2);
+        // The >= encourages trailing rather than leading whitespace on edits.
+        if (score >= bestScore) {
+          bestScore = score;
+          bestEquality1 = equality1;
+          bestEdit = edit;
+          bestEquality2 = equality2;
+        }
+      }
+
+      if (diffs[pointer - 1][1] != bestEquality1) {
+        // We have an improvement, save it back to the diff.
+        if (bestEquality1) {
+          diffs[pointer - 1][1] = bestEquality1;
+        } else {
+          diffs.splice(pointer - 1, 1);
+          pointer--;
+        }
+        diffs[pointer][1] = bestEdit;
+        if (bestEquality2) {
+          diffs[pointer + 1][1] = bestEquality2;
+        } else {
+          diffs.splice(pointer + 1, 1);
+          pointer--;
+        }
+      }
+    }
+    pointer++;
+  }
+};
+
+// Define some regex patterns for matching boundaries.
+diff_match_patch.nonAlphaNumericRegex_ = /[^a-zA-Z0-9]/;
+diff_match_patch.whitespaceRegex_ = /\s/;
+diff_match_patch.linebreakRegex_ = /[\r\n]/;
+diff_match_patch.blanklineEndRegex_ = /\n\r?\n$/;
+diff_match_patch.blanklineStartRegex_ = /^\r?\n\r?\n/;
+
+/**
+ * Reduce the number of edits by eliminating operationally trivial equalities.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ */
+diff_match_patch.prototype.diff_cleanupEfficiency = function(diffs) {
+  var changes = false;
+  var equalities = [];  // Stack of indices where equalities are found.
+  var equalitiesLength = 0;  // Keeping our own length var is faster in JS.
+  /** @type {?string} */
+  var lastequality = null;
+  // Always equal to diffs[equalities[equalitiesLength - 1]][1]
+  var pointer = 0;  // Index of current position.
+  // Is there an insertion operation before the last equality.
+  var pre_ins = false;
+  // Is there a deletion operation before the last equality.
+  var pre_del = false;
+  // Is there an insertion operation after the last equality.
+  var post_ins = false;
+  // Is there a deletion operation after the last equality.
+  var post_del = false;
+  while (pointer < diffs.length) {
+    if (diffs[pointer][0] == DIFF_EQUAL) {  // Equality found.
+      if (diffs[pointer][1].length < this.Diff_EditCost &&
+          (post_ins || post_del)) {
+        // Candidate found.
+        equalities[equalitiesLength++] = pointer;
+        pre_ins = post_ins;
+        pre_del = post_del;
+        lastequality = diffs[pointer][1];
+      } else {
+        // Not a candidate, and can never become one.
+        equalitiesLength = 0;
+        lastequality = null;
+      }
+      post_ins = post_del = false;
+    } else {  // An insertion or deletion.
+      if (diffs[pointer][0] == DIFF_DELETE) {
+        post_del = true;
+      } else {
+        post_ins = true;
+      }
+      /*
+       * Five types to be split:
+       * <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
+       * <ins>A</ins>X<ins>C</ins><del>D</del>
+       * <ins>A</ins><del>B</del>X<ins>C</ins>
+       * <ins>A</del>X<ins>C</ins><del>D</del>
+       * <ins>A</ins><del>B</del>X<del>C</del>
+       */
+      if (lastequality && ((pre_ins && pre_del && post_ins && post_del) ||
+                           ((lastequality.length < this.Diff_EditCost / 2) &&
+                            (pre_ins + pre_del + post_ins + post_del) == 3))) {
+        // Duplicate record.
+        diffs.splice(equalities[equalitiesLength - 1], 0,
+                     [DIFF_DELETE, lastequality]);
+        // Change second copy to insert.
+        diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT;
+        equalitiesLength--;  // Throw away the equality we just deleted;
+        lastequality = null;
+        if (pre_ins && pre_del) {
+          // No changes made which could affect previous entry, keep going.
+          post_ins = post_del = true;
+          equalitiesLength = 0;
+        } else {
+          equalitiesLength--;  // Throw away the previous equality.
+          pointer = equalitiesLength > 0 ?
+              equalities[equalitiesLength - 1] : -1;
+          post_ins = post_del = false;
+        }
+        changes = true;
+      }
+    }
+    pointer++;
+  }
+
+  if (changes) {
+    this.diff_cleanupMerge(diffs);
+  }
+};
+
+
+/**
+ * Reorder and merge like edit sections.  Merge equalities.
+ * Any edit section can move as long as it doesn't cross an equality.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ */
+diff_match_patch.prototype.diff_cleanupMerge = function(diffs) {
+  diffs.push([DIFF_EQUAL, '']);  // Add a dummy entry at the end.
+  var pointer = 0;
+  var count_delete = 0;
+  var count_insert = 0;
+  var text_delete = '';
+  var text_insert = '';
+  var commonlength;
+  while (pointer < diffs.length) {
+    switch (diffs[pointer][0]) {
+      case DIFF_INSERT:
+        count_insert++;
+        text_insert += diffs[pointer][1];
+        pointer++;
+        break;
+      case DIFF_DELETE:
+        count_delete++;
+        text_delete += diffs[pointer][1];
+        pointer++;
+        break;
+      case DIFF_EQUAL:
+        // Upon reaching an equality, check for prior redundancies.
+        if (count_delete + count_insert > 1) {
+          if (count_delete !== 0 && count_insert !== 0) {
+            // Factor out any common prefixies.
+            commonlength = this.diff_commonPrefix(text_insert, text_delete);
+            if (commonlength !== 0) {
+              if ((pointer - count_delete - count_insert) > 0 &&
+                  diffs[pointer - count_delete - count_insert - 1][0] ==
+                  DIFF_EQUAL) {
+                diffs[pointer - count_delete - count_insert - 1][1] +=
+                    text_insert.substring(0, commonlength);
+              } else {
+                diffs.splice(0, 0, [DIFF_EQUAL,
+                                    text_insert.substring(0, commonlength)]);
+                pointer++;
+              }
+              text_insert = text_insert.substring(commonlength);
+              text_delete = text_delete.substring(commonlength);
+            }
+            // Factor out any common suffixies.
+            commonlength = this.diff_commonSuffix(text_insert, text_delete);
+            if (commonlength !== 0) {
+              diffs[pointer][1] = text_insert.substring(text_insert.length -
+                  commonlength) + diffs[pointer][1];
+              text_insert = text_insert.substring(0, text_insert.length -
+                  commonlength);
+              text_delete = text_delete.substring(0, text_delete.length -
+                  commonlength);
+            }
+          }
+          // Delete the offending records and add the merged ones.
+          if (count_delete === 0) {
+            diffs.splice(pointer - count_insert,
+                count_delete + count_insert, [DIFF_INSERT, text_insert]);
+          } else if (count_insert === 0) {
+            diffs.splice(pointer - count_delete,
+                count_delete + count_insert, [DIFF_DELETE, text_delete]);
+          } else {
+            diffs.splice(pointer - count_delete - count_insert,
+                count_delete + count_insert, [DIFF_DELETE, text_delete],
+                [DIFF_INSERT, text_insert]);
+          }
+          pointer = pointer - count_delete - count_insert +
+                    (count_delete ? 1 : 0) + (count_insert ? 1 : 0) + 1;
+        } else if (pointer !== 0 && diffs[pointer - 1][0] == DIFF_EQUAL) {
+          // Merge this equality with the previous one.
+          diffs[pointer - 1][1] += diffs[pointer][1];
+          diffs.splice(pointer, 1);
+        } else {
+          pointer++;
+        }
+        count_insert = 0;
+        count_delete = 0;
+        text_delete = '';
+        text_insert = '';
+        break;
+    }
+  }
+  if (diffs[diffs.length - 1][1] === '') {
+    diffs.pop();  // Remove the dummy entry at the end.
+  }
+
+  // Second pass: look for single edits surrounded on both sides by equalities
+  // which can be shifted sideways to eliminate an equality.
+  // e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
+  var changes = false;
+  pointer = 1;
+  // Intentionally ignore the first and last element (don't need checking).
+  while (pointer < diffs.length - 1) {
+    if (diffs[pointer - 1][0] == DIFF_EQUAL &&
+        diffs[pointer + 1][0] == DIFF_EQUAL) {
+      // This is a single edit surrounded by equalities.
+      if (diffs[pointer][1].substring(diffs[pointer][1].length -
+          diffs[pointer - 1][1].length) == diffs[pointer - 1][1]) {
+        // Shift the edit over the previous equality.
+        diffs[pointer][1] = diffs[pointer - 1][1] +
+            diffs[pointer][1].substring(0, diffs[pointer][1].length -
+                                        diffs[pointer - 1][1].length);
+        diffs[pointer + 1][1] = diffs[pointer - 1][1] + diffs[pointer + 1][1];
+        diffs.splice(pointer - 1, 1);
+        changes = true;
+      } else if (diffs[pointer][1].substring(0, diffs[pointer + 1][1].length) ==
+          diffs[pointer + 1][1]) {
+        // Shift the edit over the next equality.
+        diffs[pointer - 1][1] += diffs[pointer + 1][1];
+        diffs[pointer][1] =
+            diffs[pointer][1].substring(diffs[pointer + 1][1].length) +
+            diffs[pointer + 1][1];
+        diffs.splice(pointer + 1, 1);
+        changes = true;
+      }
+    }
+    pointer++;
+  }
+  // If shifts were made, the diff needs reordering and another shift sweep.
+  if (changes) {
+    this.diff_cleanupMerge(diffs);
+  }
+};
+
+
+/**
+ * loc is a location in text1, compute and return the equivalent location in
+ * text2.
+ * e.g. 'The cat' vs 'The big cat', 1->1, 5->8
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @param {number} loc Location within text1.
+ * @return {number} Location within text2.
+ */
+diff_match_patch.prototype.diff_xIndex = function(diffs, loc) {
+  var chars1 = 0;
+  var chars2 = 0;
+  var last_chars1 = 0;
+  var last_chars2 = 0;
+  var x;
+  for (x = 0; x < diffs.length; x++) {
+    if (diffs[x][0] !== DIFF_INSERT) {  // Equality or deletion.
+      chars1 += diffs[x][1].length;
+    }
+    if (diffs[x][0] !== DIFF_DELETE) {  // Equality or insertion.
+      chars2 += diffs[x][1].length;
+    }
+    if (chars1 > loc) {  // Overshot the location.
+      break;
+    }
+    last_chars1 = chars1;
+    last_chars2 = chars2;
+  }
+  // Was the location was deleted?
+  if (diffs.length != x && diffs[x][0] === DIFF_DELETE) {
+    return last_chars2;
+  }
+  // Add the remaining character length.
+  return last_chars2 + (loc - last_chars1);
+};
+
+
+/**
+ * Convert a diff array into a pretty HTML report.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @return {string} HTML representation.
+ */
+diff_match_patch.prototype.diff_prettyHtml = function(diffs) {
+  var html = [];
+  var pattern_amp = /&/g;
+  var pattern_lt = /</g;
+  var pattern_gt = />/g;
+  var pattern_para = /\n/g;
+  for (var x = 0; x < diffs.length; x++) {
+    var op = diffs[x][0];    // Operation (insert, delete, equal)
+    var data = diffs[x][1];  // Text of change.
+    var text = data.replace(pattern_amp, '&amp;').replace(pattern_lt, '&lt;')
+        .replace(pattern_gt, '&gt;').replace(pattern_para, '&para;<br>');
+    switch (op) {
+      case DIFF_INSERT:
+        html[x] = '<ins style="background:#e6ffe6;">' + text + '</ins>';
+        break;
+      case DIFF_DELETE:
+        html[x] = '<del style="background:#ffe6e6;">' + text + '</del>';
+        break;
+      case DIFF_EQUAL:
+        html[x] = '<span>' + text + '</span>';
+        break;
+    }
+  }
+  return html.join('');
+};
+
+
+/**
+ * Compute and return the source text (all equalities and deletions).
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @return {string} Source text.
+ */
+diff_match_patch.prototype.diff_text1 = function(diffs) {
+  var text = [];
+  for (var x = 0; x < diffs.length; x++) {
+    if (diffs[x][0] !== DIFF_INSERT) {
+      text[x] = diffs[x][1];
+    }
+  }
+  return text.join('');
+};
+
+
+/**
+ * Compute and return the destination text (all equalities and insertions).
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @return {string} Destination text.
+ */
+diff_match_patch.prototype.diff_text2 = function(diffs) {
+  var text = [];
+  for (var x = 0; x < diffs.length; x++) {
+    if (diffs[x][0] !== DIFF_DELETE) {
+      text[x] = diffs[x][1];
+    }
+  }
+  return text.join('');
+};
+
+
+/**
+ * Compute the Levenshtein distance; the number of inserted, deleted or
+ * substituted characters.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @return {number} Number of changes.
+ */
+diff_match_patch.prototype.diff_levenshtein = function(diffs) {
+  var levenshtein = 0;
+  var insertions = 0;
+  var deletions = 0;
+  for (var x = 0; x < diffs.length; x++) {
+    var op = diffs[x][0];
+    var data = diffs[x][1];
+    switch (op) {
+      case DIFF_INSERT:
+        insertions += data.length;
+        break;
+      case DIFF_DELETE:
+        deletions += data.length;
+        break;
+      case DIFF_EQUAL:
+        // A deletion and an insertion is one substitution.
+        levenshtein += Math.max(insertions, deletions);
+        insertions = 0;
+        deletions = 0;
+        break;
+    }
+  }
+  levenshtein += Math.max(insertions, deletions);
+  return levenshtein;
+};
+
+
+/**
+ * Crush the diff into an encoded string which describes the operations
+ * required to transform text1 into text2.
+ * E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
+ * Operations are tab-separated.  Inserted text is escaped using %xx notation.
+ * @param {!Array.<!diff_match_patch.Diff>} diffs Array of diff tuples.
+ * @return {string} Delta text.
+ */
+diff_match_patch.prototype.diff_toDelta = function(diffs) {
+  var text = [];
+  for (var x = 0; x < diffs.length; x++) {
+    switch (diffs[x][0]) {
+      case DIFF_INSERT:
+        text[x] = '+' + encodeURI(diffs[x][1]);
+        break;
+      case DIFF_DELETE:
+        text[x] = '-' + diffs[x][1].length;
+        break;
+      case DIFF_EQUAL:
+        text[x] = '=' + diffs[x][1].length;
+        break;
+    }
+  }
+  return text.join('\t').replace(/%20/g, ' ');
+};
+
+
+/**
+ * Given the original text1, and an encoded string which describes the
+ * operations required to transform text1 into text2, compute the full diff.
+ * @param {string} text1 Source string for the diff.
+ * @param {string} delta Delta text.
+ * @return {!Array.<!diff_match_patch.Diff>} Array of diff tuples.
+ * @throws {!Error} If invalid input.
+ */
+diff_match_patch.prototype.diff_fromDelta = function(text1, delta) {
+  var diffs = [];
+  var diffsLength = 0;  // Keeping our own length var is faster in JS.
+  var pointer = 0;  // Cursor in text1
+  var tokens = delta.split(/\t/g);
+  for (var x = 0; x < tokens.length; x++) {
+    // Each token begins with a one character parameter which specifies the
+    // operation of this token (delete, insert, equality).
+    var param = tokens[x].substring(1);
+    switch (tokens[x].charAt(0)) {
+      case '+':
+        try {
+          diffs[diffsLength++] = [DIFF_INSERT, decodeURI(param)];
+        } catch (ex) {
+          // Malformed URI sequence.
+          throw new Error('Illegal escape in diff_fromDelta: ' + param);
+        }
+        break;
+      case '-':
+        // Fall through.
+      case '=':
+        var n = parseInt(param, 10);
+        if (isNaN(n) || n < 0) {
+          throw new Error('Invalid number in diff_fromDelta: ' + param);
+        }
+        var text = text1.substring(pointer, pointer += n);
+        if (tokens[x].charAt(0) == '=') {
+          diffs[diffsLength++] = [DIFF_EQUAL, text];
+        } else {
+          diffs[diffsLength++] = [DIFF_DELETE, text];
+        }
+        break;
+      default:
+        // Blank tokens are ok (from a trailing \t).
+        // Anything else is an error.
+        if (tokens[x]) {
+          throw new Error('Invalid diff operation in diff_fromDelta: ' +
+                          tokens[x]);
+        }
+    }
+  }
+  if (pointer != text1.length) {
+    throw new Error('Delta length (' + pointer +
+        ') does not equal source text length (' + text1.length + ').');
+  }
+  return diffs;
+};
+
+
+//  MATCH FUNCTIONS
+
+
+/**
+ * Locate the best instance of 'pattern' in 'text' near 'loc'.
+ * @param {string} text The text to search.
+ * @param {string} pattern The pattern to search for.
+ * @param {number} loc The location to search around.
+ * @return {number} Best match index or -1.
+ */
+diff_match_patch.prototype.match_main = function(text, pattern, loc) {
+  // Check for null inputs.
+  if (text == null || pattern == null || loc == null) {
+    throw new Error('Null input. (match_main)');
+  }
+
+  loc = Math.max(0, Math.min(loc, text.length));
+  if (text == pattern) {
+    // Shortcut (potentially not guaranteed by the algorithm)
+    return 0;
+  } else if (!text.length) {
+    // Nothing to match.
+    return -1;
+  } else if (text.substring(loc, loc + pattern.length) == pattern) {
+    // Perfect match at the perfect spot!  (Includes case of null pattern)
+    return loc;
+  } else {
+    // Do a fuzzy compare.
+    return this.match_bitap_(text, pattern, loc);
+  }
+};
+
+
+/**
+ * Locate the best instance of 'pattern' in 'text' near 'loc' using the
+ * Bitap algorithm.
+ * @param {string} text The text to search.
+ * @param {string} pattern The pattern to search for.
+ * @param {number} loc The location to search around.
+ * @return {number} Best match index or -1.
+ * @private
+ */
+diff_match_patch.prototype.match_bitap_ = function(text, pattern, loc) {
+  if (pattern.length > this.Match_MaxBits) {
+    throw new Error('Pattern too long for this browser.');
+  }
+
+  // Initialise the alphabet.
+  var s = this.match_alphabet_(pattern);
+
+  var dmp = this;  // 'this' becomes 'window' in a closure.
+
+  /**
+   * Compute and return the score for a match with e errors and x location.
+   * Accesses loc and pattern through being a closure.
+   * @param {number} e Number of errors in match.
+   * @param {number} x Location of match.
+   * @return {number} Overall score for match (0.0 = good, 1.0 = bad).
+   * @private
+   */
+  function match_bitapScore_(e, x) {
+    var accuracy = e / pattern.length;
+    var proximity = Math.abs(loc - x);
+    if (!dmp.Match_Distance) {
+      // Dodge divide by zero error.
+      return proximity ? 1.0 : accuracy;
+    }
+    return accuracy + (proximity / dmp.Match_Distance);
+  }
+
+  // Highest score beyond which we give up.
+  var score_threshold = this.Match_Threshold;
+  // Is there a nearby exact match? (speedup)
+  var best_loc = text.indexOf(pattern, loc);
+  if (best_loc != -1) {
+    score_threshold = Math.min(match_bitapScore_(0, best_loc), score_threshold);
+    // What about in the other direction? (speedup)
+    best_loc = text.lastIndexOf(pattern, loc + pattern.length);
+    if (best_loc != -1) {
+      score_threshold =
+          Math.min(match_bitapScore_(0, best_loc), score_threshold);
+    }
+  }
+
+  // Initialise the bit arrays.
+  var matchmask = 1 << (pattern.length - 1);
+  best_loc = -1;
+
+  var bin_min, bin_mid;
+  var bin_max = pattern.length + text.length;
+  var last_rd;
+  for (var d = 0; d < pattern.length; d++) {
+    // Scan for the best match; each iteration allows for one more error.
+    // Run a binary search to determine how far from 'loc' we can stray at this
+    // error level.
+    bin_min = 0;
+    bin_mid = bin_max;
+    while (bin_min < bin_mid) {
+      if (match_bitapScore_(d, loc + bin_mid) <= score_threshold) {
+        bin_min = bin_mid;
+      } else {
+        bin_max = bin_mid;
+      }
+      bin_mid = Math.floor((bin_max - bin_min) / 2 + bin_min);
+    }
+    // Use the result from this iteration as the maximum for the next.
+    bin_max = bin_mid;
+    var start = Math.max(1, loc - bin_mid + 1);
+    var finish = Math.min(loc + bin_mid, text.length) + pattern.length;
+
+    var rd = Array(finish + 2);
+    rd[finish + 1] = (1 << d) - 1;
+    for (var j = finish; j >= start; j--) {
+      // The alphabet (s) is a sparse hash, so the following line generates
+      // warnings.
+      var charMatch = s[text.charAt(j - 1)];
+      if (d === 0) {  // First pass: exact match.
+        rd[j] = ((rd[j + 1] << 1) | 1) & charMatch;
+      } else {  // Subsequent passes: fuzzy match.
+        rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) |
+                (((last_rd[j + 1] | last_rd[j]) << 1) | 1) |
+                last_rd[j + 1];
+      }
+      if (rd[j] & matchmask) {
+        var score = match_bitapScore_(d, j - 1);
+        // This match will almost certainly be better than any existing match.
+        // But check anyway.
+        if (score <= score_threshold) {
+          // Told you so.
+          score_threshold = score;
+          best_loc = j - 1;
+          if (best_loc > loc) {
+            // When passing loc, don't exceed our current distance from loc.
+            start = Math.max(1, 2 * loc - best_loc);
+          } else {
+            // Already passed loc, downhill from here on in.
+            break;
+          }
+        }
+      }
+    }
+    // No hope for a (better) match at greater error levels.
+    if (match_bitapScore_(d + 1, loc) > score_threshold) {
+      break;
+    }
+    last_rd = rd;
+  }
+  return best_loc;
+};
+
+
+/**
+ * Initialise the alphabet for the Bitap algorithm.
+ * @param {string} pattern The text to encode.
+ * @return {!Object} Hash of character locations.
+ * @private
+ */
+diff_match_patch.prototype.match_alphabet_ = function(pattern) {
+  var s = {};
+  for (var i = 0; i < pattern.length; i++) {
+    s[pattern.charAt(i)] = 0;
+  }
+  for (var i = 0; i < pattern.length; i++) {
+    s[pattern.charAt(i)] |= 1 << (pattern.length - i - 1);
+  }
+  return s;
+};
+
+
+//  PATCH FUNCTIONS
+
+
+/**
+ * Increase the context until it is unique,
+ * but don't let the pattern expand beyond Match_MaxBits.
+ * @param {!diff_match_patch.patch_obj} patch The patch to grow.
+ * @param {string} text Source text.
+ * @private
+ */
+diff_match_patch.prototype.patch_addContext_ = function(patch, text) {
+  if (text.length == 0) {
+    return;
+  }
+  var pattern = text.substring(patch.start2, patch.start2 + patch.length1);
+  var padding = 0;
+
+  // Look for the first and last matches of pattern in text.  If two different
+  // matches are found, increase the pattern length.
+  while (text.indexOf(pattern) != text.lastIndexOf(pattern) &&
+         pattern.length < this.Match_MaxBits - this.Patch_Margin -
+         this.Patch_Margin) {
+    padding += this.Patch_Margin;
+    pattern = text.substring(patch.start2 - padding,
+                             patch.start2 + patch.length1 + padding);
+  }
+  // Add one chunk for good luck.
+  padding += this.Patch_Margin;
+
+  // Add the prefix.
+  var prefix = text.substring(patch.start2 - padding, patch.start2);
+  if (prefix) {
+    patch.diffs.unshift([DIFF_EQUAL, prefix]);
+  }
+  // Add the suffix.
+  var suffix = text.substring(patch.start2 + patch.length1,
+                              patch.start2 + patch.length1 + padding);
+  if (suffix) {
+    patch.diffs.push([DIFF_EQUAL, suffix]);
+  }
+
+  // Roll back the start points.
+  patch.start1 -= prefix.length;
+  patch.start2 -= prefix.length;
+  // Extend the lengths.
+  patch.length1 += prefix.length + suffix.length;
+  patch.length2 += prefix.length + suffix.length;
+};
+
+
+/**
+ * Compute a list of patches to turn text1 into text2.
+ * Use diffs if provided, otherwise compute it ourselves.
+ * There are four ways to call this function, depending on what data is
+ * available to the caller:
+ * Method 1:
+ * a = text1, b = text2
+ * Method 2:
+ * a = diffs
+ * Method 3 (optimal):
+ * a = text1, b = diffs
+ * Method 4 (deprecated, use method 3):
+ * a = text1, b = text2, c = diffs
+ *
+ * @param {string|!Array.<!diff_match_patch.Diff>} a text1 (methods 1,3,4) or
+ * Array of diff tuples for text1 to text2 (method 2).
+ * @param {string|!Array.<!diff_match_patch.Diff>} opt_b text2 (methods 1,4) or
+ * Array of diff tuples for text1 to text2 (method 3) or undefined (method 2).
+ * @param {string|!Array.<!diff_match_patch.Diff>} opt_c Array of diff tuples
+ * for text1 to text2 (method 4) or undefined (methods 1,2,3).
+ * @return {!Array.<!diff_match_patch.patch_obj>} Array of Patch objects.
+ */
+diff_match_patch.prototype.patch_make = function(a, opt_b, opt_c) {
+  var text1, diffs;
+  if (typeof a == 'string' && typeof opt_b == 'string' &&
+      typeof opt_c == 'undefined') {
+    // Method 1: text1, text2
+    // Compute diffs from text1 and text2.
+    text1 = /** @type {string} */(a);
+    diffs = this.diff_main(text1, /** @type {string} */(opt_b), true);
+    if (diffs.length > 2) {
+      this.diff_cleanupSemantic(diffs);
+      this.diff_cleanupEfficiency(diffs);
+    }
+  } else if (a && typeof a == 'object' && typeof opt_b == 'undefined' &&
+      typeof opt_c == 'undefined') {
+    // Method 2: diffs
+    // Compute text1 from diffs.
+    diffs = /** @type {!Array.<!diff_match_patch.Diff>} */(a);
+    text1 = this.diff_text1(diffs);
+  } else if (typeof a == 'string' && opt_b && typeof opt_b == 'object' &&
+      typeof opt_c == 'undefined') {
+    // Method 3: text1, diffs
+    text1 = /** @type {string} */(a);
+    diffs = /** @type {!Array.<!diff_match_patch.Diff>} */(opt_b);
+  } else if (typeof a == 'string' && typeof opt_b == 'string' &&
+      opt_c && typeof opt_c == 'object') {
+    // Method 4: text1, text2, diffs
+    // text2 is not used.
+    text1 = /** @type {string} */(a);
+    diffs = /** @type {!Array.<!diff_match_patch.Diff>} */(opt_c);
+  } else {
+    throw new Error('Unknown call format to patch_make.');
+  }
+
+  if (diffs.length === 0) {
+    return [];  // Get rid of the null case.
+  }
+  var patches = [];
+  var patch = new diff_match_patch.patch_obj();
+  var patchDiffLength = 0;  // Keeping our own length var is faster in JS.
+  var char_count1 = 0;  // Number of characters into the text1 string.
+  var char_count2 = 0;  // Number of characters into the text2 string.
+  // Start with text1 (prepatch_text) and apply the diffs until we arrive at
+  // text2 (postpatch_text).  We recreate the patches one by one to determine
+  // context info.
+  var prepatch_text = text1;
+  var postpatch_text = text1;
+  for (var x = 0; x < diffs.length; x++) {
+    var diff_type = diffs[x][0];
+    var diff_text = diffs[x][1];
+
+    if (!patchDiffLength && diff_type !== DIFF_EQUAL) {
+      // A new patch starts here.
+      patch.start1 = char_count1;
+      patch.start2 = char_count2;
+    }
+
+    switch (diff_type) {
+      case DIFF_INSERT:
+        patch.diffs[patchDiffLength++] = diffs[x];
+        patch.length2 += diff_text.length;
+        postpatch_text = postpatch_text.substring(0, char_count2) + diff_text +
+                         postpatch_text.substring(char_count2);
+        break;
+      case DIFF_DELETE:
+        patch.length1 += diff_text.length;
+        patch.diffs[patchDiffLength++] = diffs[x];
+        postpatch_text = postpatch_text.substring(0, char_count2) +
+                         postpatch_text.substring(char_count2 +
+                             diff_text.length);
+        break;
+      case DIFF_EQUAL:
+        if (diff_text.length <= 2 * this.Patch_Margin &&
+            patchDiffLength && diffs.length != x + 1) {
+          // Small equality inside a patch.
+          patch.diffs[patchDiffLength++] = diffs[x];
+          patch.length1 += diff_text.length;
+          patch.length2 += diff_text.length;
+        } else if (diff_text.length >= 2 * this.Patch_Margin) {
+          // Time for a new patch.
+          if (patchDiffLength) {
+            this.patch_addContext_(patch, prepatch_text);
+            patches.push(patch);
+            patch = new diff_match_patch.patch_obj();
+            patchDiffLength = 0;
+            // Unlike Unidiff, our patch lists have a rolling context.
+            // http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
+            // Update prepatch text & pos to reflect the application of the
+            // just completed patch.
+            prepatch_text = postpatch_text;
+            char_count1 = char_count2;
+          }
+        }
+        break;
+    }
+
+    // Update the current character count.
+    if (diff_type !== DIFF_INSERT) {
+      char_count1 += diff_text.length;
+    }
+    if (diff_type !== DIFF_DELETE) {
+      char_count2 += diff_text.length;
+    }
+  }
+  // Pick up the leftover patch if not empty.
+  if (patchDiffLength) {
+    this.patch_addContext_(patch, prepatch_text);
+    patches.push(patch);
+  }
+
+  return patches;
+};
+
+
+/**
+ * Given an array of patches, return another array that is identical.
+ * @param {!Array.<!diff_match_patch.patch_obj>} patches Array of Patch objects.
+ * @return {!Array.<!diff_match_patch.patch_obj>} Array of Patch objects.
+ */
+diff_match_patch.prototype.patch_deepCopy = function(patches) {
+  // Making deep copies is hard in JavaScript.
+  var patchesCopy = [];
+  for (var x = 0; x < patches.length; x++) {
+    var patch = patches[x];
+    var patchCopy = new diff_match_patch.patch_obj();
+    patchCopy.diffs = [];
+    for (var y = 0; y < patch.diffs.length; y++) {
+      patchCopy.diffs[y] = patch.diffs[y].slice();
+    }
+    patchCopy.start1 = patch.start1;
+    patchCopy.start2 = patch.start2;
+    patchCopy.length1 = patch.length1;
+    patchCopy.length2 = patch.length2;
+    patchesCopy[x] = patchCopy;
+  }
+  return patchesCopy;
+};
+
+
+/**
+ * Merge a set of patches onto the text.  Return a patched text, as well
+ * as a list of true/false values indicating which patches were applied.
+ * @param {!Array.<!diff_match_patch.patch_obj>} patches Array of Patch objects.
+ * @param {string} text Old text.
+ * @return {!Array.<string|!Array.<boolean>>} Two element Array, containing the
+ *      new text and an array of boolean values.
+ */
+diff_match_patch.prototype.patch_apply = function(patches, text) {
+  if (patches.length == 0) {
+    return [text, []];
+  }
+
+  // Deep copy the patches so that no changes are made to originals.
+  patches = this.patch_deepCopy(patches);
+
+  var nullPadding = this.patch_addPadding(patches);
+  text = nullPadding + text + nullPadding;
+
+  this.patch_splitMax(patches);
+  // delta keeps track of the offset between the expected and actual location
+  // of the previous patch.  If there are patches expected at positions 10 and
+  // 20, but the first patch was found at 12, delta is 2 and the second patch
+  // has an effective expected position of 22.
+  var delta = 0;
+  var results = [];
+  for (var x = 0; x < patches.length; x++) {
+    var expected_loc = patches[x].start2 + delta;
+    var text1 = this.diff_text1(patches[x].diffs);
+    var start_loc;
+    var end_loc = -1;
+    if (text1.length > this.Match_MaxBits) {
+      // patch_splitMax will only provide an oversized pattern in the case of
+      // a monster delete.
+      start_loc = this.match_main(text, text1.substring(0, this.Match_MaxBits),
+                                  expected_loc);
+      if (start_loc != -1) {
+        end_loc = this.match_main(text,
+            text1.substring(text1.length - this.Match_MaxBits),
+            expected_loc + text1.length - this.Match_MaxBits);
+        if (end_loc == -1 || start_loc >= end_loc) {
+          // Can't find valid trailing context.  Drop this patch.
+          start_loc = -1;
+        }
+      }
+    } else {
+      start_loc = this.match_main(text, text1, expected_loc);
+    }
+    if (start_loc == -1) {
+      // No match found.  :(
+      results[x] = false;
+      // Subtract the delta for this failed patch from subsequent patches.
+      delta -= patches[x].length2 - patches[x].length1;
+    } else {
+      // Found a match.  :)
+      results[x] = true;
+      delta = start_loc - expected_loc;
+      var text2;
+      if (end_loc == -1) {
+        text2 = text.substring(start_loc, start_loc + text1.length);
+      } else {
+        text2 = text.substring(start_loc, end_loc + this.Match_MaxBits);
+      }
+      if (text1 == text2) {
+        // Perfect match, just shove the replacement text in.
+        text = text.substring(0, start_loc) +
+               this.diff_text2(patches[x].diffs) +
+               text.substring(start_loc + text1.length);
+      } else {
+        // Imperfect match.  Run a diff to get a framework of equivalent
+        // indices.
+        var diffs = this.diff_main(text1, text2, false);
+        if (text1.length > this.Match_MaxBits &&
+            this.diff_levenshtein(diffs) / text1.length >
+            this.Patch_DeleteThreshold) {
+          // The end points match, but the content is unacceptably bad.
+          results[x] = false;
+        } else {
+          this.diff_cleanupSemanticLossless(diffs);
+          var index1 = 0;
+          var index2;
+          for (var y = 0; y < patches[x].diffs.length; y++) {
+            var mod = patches[x].diffs[y];
+            if (mod[0] !== DIFF_EQUAL) {
+              index2 = this.diff_xIndex(diffs, index1);
+            }
+            if (mod[0] === DIFF_INSERT) {  // Insertion
+              text = text.substring(0, start_loc + index2) + mod[1] +
+                     text.substring(start_loc + index2);
+            } else if (mod[0] === DIFF_DELETE) {  // Deletion
+              text = text.substring(0, start_loc + index2) +
+                     text.substring(start_loc + this.diff_xIndex(diffs,
+                         index1 + mod[1].length));
+            }
+            if (mod[0] !== DIFF_DELETE) {
+              index1 += mod[1].length;
+            }
+          }
+        }
+      }
+    }
+  }
+  // Strip the padding off.
+  text = text.substring(nullPadding.length, text.length - nullPadding.length);
+  return [text, results];
+};
+
+
+/**
+ * Add some padding on text start and end so that edges can match something.
+ * Intended to be called only from within patch_apply.
+ * @param {!Array.<!diff_match_patch.patch_obj>} patches Array of Patch objects.
+ * @return {string} The padding string added to each side.
+ */
+diff_match_patch.prototype.patch_addPadding = function(patches) {
+  var paddingLength = this.Patch_Margin;
+  var nullPadding = '';
+  for (var x = 1; x <= paddingLength; x++) {
+    nullPadding += String.fromCharCode(x);
+  }
+
+  // Bump all the patches forward.
+  for (var x = 0; x < patches.length; x++) {
+    patches[x].start1 += paddingLength;
+    patches[x].start2 += paddingLength;
+  }
+
+  // Add some padding on start of first diff.
+  var patch = patches[0];
+  var diffs = patch.diffs;
+  if (diffs.length == 0 || diffs[0][0] != DIFF_EQUAL) {
+    // Add nullPadding equality.
+    diffs.unshift([DIFF_EQUAL, nullPadding]);
+    patch.start1 -= paddingLength;  // Should be 0.
+    patch.start2 -= paddingLength;  // Should be 0.
+    patch.length1 += paddingLength;
+    patch.length2 += paddingLength;
+  } else if (paddingLength > diffs[0][1].length) {
+    // Grow first equality.
+    var extraLength = paddingLength - diffs[0][1].length;
+    diffs[0][1] = nullPadding.substring(diffs[0][1].length) + diffs[0][1];
+    patch.start1 -= extraLength;
+    patch.start2 -= extraLength;
+    patch.length1 += extraLength;
+    patch.length2 += extraLength;
+  }
+
+  // Add some padding on end of last diff.
+  patch = patches[patches.length - 1];
+  diffs = patch.diffs;
+  if (diffs.length == 0 || diffs[diffs.length - 1][0] != DIFF_EQUAL) {
+    // Add nullPadding equality.
+    diffs.push([DIFF_EQUAL, nullPadding]);
+    patch.length1 += paddingLength;
+    patch.length2 += paddingLength;
+  } else if (paddingLength > diffs[diffs.length - 1][1].length) {
+    // Grow last equality.
+    var extraLength = paddingLength - diffs[diffs.length - 1][1].length;
+    diffs[diffs.length - 1][1] += nullPadding.substring(0, extraLength);
+    patch.length1 += extraLength;
+    patch.length2 += extraLength;
+  }
+
+  return nullPadding;
+};
+
+
+/**
+ * Look through the patches and break up any which are longer than the maximum
+ * limit of the match algorithm.
+ * Intended to be called only from within patch_apply.
+ * @param {!Array.<!diff_match_patch.patch_obj>} patches Array of Patch objects.
+ */
+diff_match_patch.prototype.patch_splitMax = function(patches) {
+  var patch_size = this.Match_MaxBits;
+  for (var x = 0; x < patches.length; x++) {
+    if (patches[x].length1 <= patch_size) {
+      continue;
+    }
+    var bigpatch = patches[x];
+    // Remove the big old patch.
+    patches.splice(x--, 1);
+    var start1 = bigpatch.start1;
+    var start2 = bigpatch.start2;
+    var precontext = '';
+    while (bigpatch.diffs.length !== 0) {
+      // Create one of several smaller patches.
+      var patch = new diff_match_patch.patch_obj();
+      var empty = true;
+      patch.start1 = start1 - precontext.length;
+      patch.start2 = start2 - precontext.length;
+      if (precontext !== '') {
+        patch.length1 = patch.length2 = precontext.length;
+        patch.diffs.push([DIFF_EQUAL, precontext]);
+      }
+      while (bigpatch.diffs.length !== 0 &&
+             patch.length1 < patch_size - this.Patch_Margin) {
+        var diff_type = bigpatch.diffs[0][0];
+        var diff_text = bigpatch.diffs[0][1];
+        if (diff_type === DIFF_INSERT) {
+          // Insertions are harmless.
+          patch.length2 += diff_text.length;
+          start2 += diff_text.length;
+          patch.diffs.push(bigpatch.diffs.shift());
+          empty = false;
+        } else if (diff_type === DIFF_DELETE && patch.diffs.length == 1 &&
+                   patch.diffs[0][0] == DIFF_EQUAL &&
+                   diff_text.length > 2 * patch_size) {
+          // This is a large deletion.  Let it pass in one chunk.
+          patch.length1 += diff_text.length;
+          start1 += diff_text.length;
+          empty = false;
+          patch.diffs.push([diff_type, diff_text]);
+          bigpatch.diffs.shift();
+        } else {
+          // Deletion or equality.  Only take as much as we can stomach.
+          diff_text = diff_text.substring(0,
+              patch_size - patch.length1 - this.Patch_Margin);
+          patch.length1 += diff_text.length;
+          start1 += diff_text.length;
+          if (diff_type === DIFF_EQUAL) {
+            patch.length2 += diff_text.length;
+            start2 += diff_text.length;
+          } else {
+            empty = false;
+          }
+          patch.diffs.push([diff_type, diff_text]);
+          if (diff_text == bigpatch.diffs[0][1]) {
+            bigpatch.diffs.shift();
+          } else {
+            bigpatch.diffs[0][1] =
+                bigpatch.diffs[0][1].substring(diff_text.length);
+          }
+        }
+      }
+      // Compute the head context for the next patch.
+      precontext = this.diff_text2(patch.diffs);
+      precontext =
+          precontext.substring(precontext.length - this.Patch_Margin);
+      // Append the end context for this patch.
+      var postcontext = this.diff_text1(bigpatch.diffs)
+                            .substring(0, this.Patch_Margin);
+      if (postcontext !== '') {
+        patch.length1 += postcontext.length;
+        patch.length2 += postcontext.length;
+        if (patch.diffs.length !== 0 &&
+            patch.diffs[patch.diffs.length - 1][0] === DIFF_EQUAL) {
+          patch.diffs[patch.diffs.length - 1][1] += postcontext;
+        } else {
+          patch.diffs.push([DIFF_EQUAL, postcontext]);
+        }
+      }
+      if (!empty) {
+        patches.splice(++x, 0, patch);
+      }
+    }
+  }
+};
+
+
+/**
+ * Take a list of patches and return a textual representation.
+ * @param {!Array.<!diff_match_patch.patch_obj>} patches Array of Patch objects.
+ * @return {string} Text representation of patches.
+ */
+diff_match_patch.prototype.patch_toText = function(patches) {
+  var text = [];
+  for (var x = 0; x < patches.length; x++) {
+    text[x] = patches[x];
+  }
+  return text.join('');
+};
+
+
+/**
+ * Parse a textual representation of patches and return a list of Patch objects.
+ * @param {string} textline Text representation of patches.
+ * @return {!Array.<!diff_match_patch.patch_obj>} Array of Patch objects.
+ * @throws {!Error} If invalid input.
+ */
+diff_match_patch.prototype.patch_fromText = function(textline) {
+  var patches = [];
+  if (!textline) {
+    return patches;
+  }
+  var text = textline.split('\n');
+  var textPointer = 0;
+  var patchHeader = /^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$/;
+  while (textPointer < text.length) {
+    var m = text[textPointer].match(patchHeader);
+    if (!m) {
+      throw new Error('Invalid patch string: ' + text[textPointer]);
+    }
+    var patch = new diff_match_patch.patch_obj();
+    patches.push(patch);
+    patch.start1 = parseInt(m[1], 10);
+    if (m[2] === '') {
+      patch.start1--;
+      patch.length1 = 1;
+    } else if (m[2] == '0') {
+      patch.length1 = 0;
+    } else {
+      patch.start1--;
+      patch.length1 = parseInt(m[2], 10);
+    }
+
+    patch.start2 = parseInt(m[3], 10);
+    if (m[4] === '') {
+      patch.start2--;
+      patch.length2 = 1;
+    } else if (m[4] == '0') {
+      patch.length2 = 0;
+    } else {
+      patch.start2--;
+      patch.length2 = parseInt(m[4], 10);
+    }
+    textPointer++;
+
+    while (textPointer < text.length) {
+      var sign = text[textPointer].charAt(0);
+      try {
+        var line = decodeURI(text[textPointer].substring(1));
+      } catch (ex) {
+        // Malformed URI sequence.
+        throw new Error('Illegal escape in patch_fromText: ' + line);
+      }
+      if (sign == '-') {
+        // Deletion.
+        patch.diffs.push([DIFF_DELETE, line]);
+      } else if (sign == '+') {
+        // Insertion.
+        patch.diffs.push([DIFF_INSERT, line]);
+      } else if (sign == ' ') {
+        // Minor equality.
+        patch.diffs.push([DIFF_EQUAL, line]);
+      } else if (sign == '@') {
+        // Start of next patch.
+        break;
+      } else if (sign === '') {
+        // Blank line?  Whatever.
+      } else {
+        // WTF?
+        throw new Error('Invalid patch mode "' + sign + '" in: ' + line);
+      }
+      textPointer++;
+    }
+  }
+  return patches;
+};
+
+
+/**
+ * Class representing one patch operation.
+ * @constructor
+ */
+diff_match_patch.patch_obj = function() {
+  /** @type {!Array.<!diff_match_patch.Diff>} */
+  this.diffs = [];
+  /** @type {?number} */
+  this.start1 = null;
+  /** @type {?number} */
+  this.start2 = null;
+  /** @type {number} */
+  this.length1 = 0;
+  /** @type {number} */
+  this.length2 = 0;
+};
+
+
+/**
+ * Emmulate GNU diff's format.
+ * Header: @@ -382,8 +481,9 @@
+ * Indicies are printed as 1-based, not 0-based.
+ * @return {string} The GNU diff string.
+ */
+diff_match_patch.patch_obj.prototype.toString = function() {
+  var coords1, coords2;
+  if (this.length1 === 0) {
+    coords1 = this.start1 + ',0';
+  } else if (this.length1 == 1) {
+    coords1 = this.start1 + 1;
+  } else {
+    coords1 = (this.start1 + 1) + ',' + this.length1;
+  }
+  if (this.length2 === 0) {
+    coords2 = this.start2 + ',0';
+  } else if (this.length2 == 1) {
+    coords2 = this.start2 + 1;
+  } else {
+    coords2 = (this.start2 + 1) + ',' + this.length2;
+  }
+  var text = ['@@ -' + coords1 + ' +' + coords2 + ' @@\n'];
+  var op;
+  // Escape the body of the patch with %xx notation.
+  for (var x = 0; x < this.diffs.length; x++) {
+    switch (this.diffs[x][0]) {
+      case DIFF_INSERT:
+        op = '+';
+        break;
+      case DIFF_DELETE:
+        op = '-';
+        break;
+      case DIFF_EQUAL:
+        op = ' ';
+        break;
+    }
+    text[x + 1] = op + encodeURI(this.diffs[x][1]) + '\n';
+  }
+  return text.join('').replace(/%20/g, ' ');
+};
+
+
+// Export these global variables so that they survive Google's JS compiler.
+// In a browser, 'this' will be 'window'.
+// Users of node.js should 'require' the uncompressed version since Google's
+// JS compiler may break the following exports for non-browser environments.
+this['diff_match_patch'] = diff_match_patch;
+this['DIFF_DELETE'] = DIFF_DELETE;
+this['DIFF_INSERT'] = DIFF_INSERT;
+this['DIFF_EQUAL'] = DIFF_EQUAL;

services/document-updater/config/settings.development.coffee

@@ -0,0 +1,23 @@
+Path = require('path')
+http = require('http')
+http.globalAgent.maxSockets = 300
+
+module.exports =
+	internal:
+		documentupdater:
+			port: 3003
+
+	apis:
+		web:
+			url: "http://localhost:3000"
+			user: "sharelatex"
+			pass: "password"
+
+	redis:
+		web:
+			port:"6379"
+			host:"localhost"
+			password:""
+
+	mongo:
+		url: 'mongodb://127.0.0.1/sharelatex'

services/document-updater/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "document-updater-sharelatex",
+  "version": "0.0.1",
+  "dependencies": {
+    "express": "3.3.4",
+    "underscore": "1.2.2",
+    "redis": "0.7.2",
+    "chai": "",
+    "request": "2.25.0",
+    "sandboxed-module": "~0.2.0",
+    "chai-spies": "",
+    "async": "",
+    "lynx": "0.0.11",
+    "coffee-script": "1.4.0",
+    "settings-sharelatex": "git+ssh://git@bitbucket.org:sharelatex/settings-sharelatex.git#master",
+    "logger-sharelatex": "git+ssh://git@bitbucket.org:sharelatex/logger-sharelatex.git#bunyan",
+    "sinon": "~1.5.2",
+    "mongojs": "0.9.11"
+  },
+  "devDependencies": {
+    "grunt-execute": "~0.1.5",
+    "grunt-contrib-clean": "~0.5.0",
+    "grunt-mocha-test": "~0.9.0",
+    "grunt": "~0.4.2",
+    "grunt-available-tasks": "~0.4.1",
+    "grunt-contrib-coffee": "~0.10.0",
+    "bunyan": "~0.22.1",
+    "grunt-bunyan": "~0.5.0"
+  }
+}

services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee

@@ -0,0 +1,215 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+async = require "async"
+mongojs = require "../../../app/js/mongojs"
+db = mongojs.db
+ObjectId = mongojs.ObjectId
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Applying updates to a doc", ->
+	before ->
+		@lines = ["one", "two", "three"]
+		@update =
+			doc: @doc_id
+			op: [{
+				i: "one and a half\n"
+				p: 4
+			}]
+			v: 0
+		@result = ["one", "one and a half", "two", "three"]
+
+	describe "when the document is not loaded", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			sinon.spy MockWebApi, "getDocument"
+			DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
+				throw error if error?
+				setTimeout done, 200
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should load the document from the web API", ->
+			MockWebApi.getDocument
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should update the doc", (done) ->
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				doc.lines.should.deep.equal @result
+				done()
+
+	describe "when the document is loaded", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
+				throw error if error?
+				sinon.spy MockWebApi, "getDocument"
+				DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
+					throw error if error?
+					setTimeout done, 200
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should not need to call the web api", ->
+			MockWebApi.getDocument.called.should.equal false
+
+		it "should update the doc", (done) ->
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				doc.lines.should.deep.equal @result
+				done()
+
+	describe "when the document has been deleted", ->
+		describe "when the ops come in a single linear order", ->
+			before ->
+				[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+				@lines = ["", "", ""]
+				MockWebApi.insertDoc @project_id, @doc_id, {
+					lines: @lines
+				}
+
+				@updates = [
+					{ doc_id: @doc_id, v: 0,  op: [i: "h", p: 0 ] }
+					{ doc_id: @doc_id, v: 1,  op: [i: "e", p: 1 ] }
+					{ doc_id: @doc_id, v: 2,  op: [i: "l", p: 2 ] }
+					{ doc_id: @doc_id, v: 3,  op: [i: "l", p: 3 ] }
+					{ doc_id: @doc_id, v: 4,  op: [i: "o", p: 4 ] }
+					{ doc_id: @doc_id, v: 5,  op: [i: " ", p: 5 ] }
+					{ doc_id: @doc_id, v: 6,  op: [i: "w", p: 6 ] }
+					{ doc_id: @doc_id, v: 7,  op: [i: "o", p: 7 ] }
+					{ doc_id: @doc_id, v: 8,  op: [i: "r", p: 8 ] }
+					{ doc_id: @doc_id, v: 9,  op: [i: "l", p: 9 ] }
+					{ doc_id: @doc_id, v: 10, op: [i: "d", p: 10] }
+				]
+				@result = ["hello world", "", ""]
+				MockWebApi.insertDoc @project_id, @doc_id, {
+					lines: @lines
+				}
+
+			it "should be able to continue applying updates when the project has been deleted", (done) ->
+				actions = []
+				for update in @updates.slice(0,6)
+					do (update) =>
+						actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+				actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback
+				for update in @updates.slice(6)
+					do (update) =>
+						actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+
+				async.series actions, (error) =>
+					throw error if error?
+					DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+						doc.lines.should.deep.equal @result
+						done()
+
+		describe "when older ops come in after the delete", ->
+			before ->
+				[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+				@lines = ["", "", ""]
+				MockWebApi.insertDoc @project_id, @doc_id, {
+					lines: @lines
+				}
+
+				@updates = [
+					{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
+					{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
+					{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
+					{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
+					{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
+					{ doc_id: @doc_id, v: 0, op: [i: "world", p: 1 ] }
+				]
+				@result = ["hello", "world", ""]
+				MockWebApi.insertDoc @project_id, @doc_id, {
+					lines: @lines
+				}
+
+			it "should be able to continue applying updates when the project has been deleted", (done) ->
+				actions = []
+				for update in @updates.slice(0,5)
+					do (update) =>
+						actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+				actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback
+				for update in @updates.slice(5)
+					do (update) =>
+						actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+
+				async.series actions, (error) =>
+					throw error if error?
+					DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+						doc.lines.should.deep.equal @result
+						done()
+		
+	describe "when the mongo array has been trimmed", ->
+		before ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			@lines = ["", "", ""]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+
+			@updates = [
+				{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
+				{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
+				{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
+				{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
+				{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
+				{ doc_id: @doc_id, v: 3, op: [i: "world", p: 4 ] }
+			]
+			@result = ["hello", "world", ""]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+
+		it "should be able to reload the required ops from the trimmed mongo array", (done) ->
+			actions = []
+			# Apply first set of ops
+			for update in @updates.slice(0,5)
+				do (update) =>
+					actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+			# Delete doc from redis and trim ops back to version 3
+			actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback
+			actions.push (callback) =>
+				db.docOps.update({doc_id: ObjectId(@doc_id)}, {$push: docOps: { $each: [], $slice: -2 }}, callback)
+			# Apply older update back from version 3
+			for update in @updates.slice(5)
+				do (update) =>
+					actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback
+			# Flush ops to mongo
+			actions.push (callback) => DocUpdaterClient.flushDoc @project_id, @doc_id, callback
+
+			async.series actions, (error) =>
+				throw error if error?
+				DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+					db.docOps.find {doc_id: ObjectId(@doc_id)}, (error, docOps) =>
+						# Check mongo array has been trimmed
+						docOps = docOps[0]
+						docOps.docOps.length.should.equal 3
+						# Check ops have all be applied properly
+						doc.lines.should.deep.equal @result
+						done()
+
+	describe "with a broken update", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			DocUpdaterClient.sendUpdate @project_id, @doc_id, @undefined, (error) ->
+				throw error if error?
+				setTimeout done, 200
+
+		it "should not update the doc", (done) ->
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				doc.lines.should.deep.equal @lines
+				done()
+	

services/document-updater/test/acceptance/coffee/DeletingADocumentTests.coffee

@@ -0,0 +1,89 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Deleting a document", ->
+	before ->
+		@lines = ["one", "two", "three"]
+		@update =
+			doc: @doc_id
+			op: [{
+				i: "one and a half\n"
+				p: 4
+			}]
+			v: 0
+		@result = ["one", "one and a half", "two", "three"]
+
+	describe "when the updated doc exists in the doc updater", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			sinon.spy MockWebApi, "setDocumentLines"
+			sinon.spy MockWebApi, "getDocument"
+			DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
+				throw error if error?
+				DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
+					throw error if error?
+					setTimeout () =>
+						DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) =>
+							@statusCode = res.statusCode
+							done()
+					, 200
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+			MockWebApi.getDocument.restore()
+
+		it "should return a 204 status code", ->
+			@statusCode.should.equal 204
+
+		it "should send the updated document to the web api", ->
+			MockWebApi.setDocumentLines
+				.calledWith(@project_id, @doc_id, @result)
+				.should.equal true
+
+		it "should need to reload the doc if read again", (done) ->
+			MockWebApi.getDocument.called.should.equal.false
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				MockWebApi.getDocument
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+				done()
+
+	describe "when the doc is not in the doc updater", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			sinon.spy MockWebApi, "setDocumentLines"
+			sinon.spy MockWebApi, "getDocument"
+			DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) =>
+				@statusCode = res.statusCode
+				done()
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+			MockWebApi.getDocument.restore()
+
+		it "should return a 204 status code", ->
+			@statusCode.should.equal 204
+
+		it "should not need to send the updated document to the web api", ->
+			MockWebApi.setDocumentLines.called.should.equal false
+
+		it "should need to reload the doc if read again", (done) ->
+			MockWebApi.getDocument.called.should.equal.false
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				MockWebApi.getDocument
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+				done()
+
+		
+

services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee

@@ -0,0 +1,81 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+async = require "async"
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Deleting a project", ->
+	before ->
+		@project_id = DocUpdaterClient.randomId()
+		@docs = [{
+			id: doc_id0 = DocUpdaterClient.randomId()
+			lines: ["one", "two", "three"]
+			update:
+				doc: doc_id0
+				op: [{
+					i: "one and a half\n"
+					p: 4
+				}]
+				v: 0
+			updatedLines: ["one", "one and a half", "two", "three"]
+		}, {
+			id: doc_id1 = DocUpdaterClient.randomId()
+			lines: ["four", "five", "six"]
+			update:
+				doc: doc_id1
+				op: [{
+					i: "four and a half\n"
+					p: 5
+				}]
+				v: 0
+			updatedLines: ["four", "four and a half", "five", "six"]
+		}]
+		for doc in @docs
+			MockWebApi.insertDoc @project_id, doc.id, {
+				lines: doc.lines
+			}
+
+	describe "with documents which have been updated", ->
+		before (done) ->
+			sinon.spy MockWebApi, "setDocumentLines"
+			async.series @docs.map((doc) =>
+				(callback) =>
+					DocUpdaterClient.preloadDoc @project_id, doc.id, (error) =>
+						return callback(error) if error?
+						DocUpdaterClient.sendUpdate @project_id, doc.id, doc.update, (error) =>
+							callback(error)
+			), (error) =>
+				throw error if error?
+				setTimeout () =>
+					DocUpdaterClient.deleteProject @project_id, (error, res, body) =>
+						@statusCode = res.statusCode
+						done()
+				, 200
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+
+		it "should return a 204 status code", ->
+			@statusCode.should.equal 204
+
+		it "should send each document to the web api", ->
+			for doc in @docs
+				MockWebApi.setDocumentLines
+					.calledWith(@project_id, doc.id, doc.updatedLines)
+					.should.equal true
+
+		it "should need to reload the docs if read again", (done) ->
+			sinon.spy MockWebApi, "getDocument"
+			async.series @docs.map((doc) =>
+				(callback) =>
+					MockWebApi.getDocument.calledWith(@project_id, doc.id).should.equal false
+					DocUpdaterClient.getDoc @project_id, doc.id, (error, res, returnedDoc) =>
+						MockWebApi.getDocument.calledWith(@project_id, doc.id).should.equal true
+						callback()
+			), () ->
+				MockWebApi.getDocument.restore()
+				done()
+
+

services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee

@@ -0,0 +1,76 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+async = require "async"
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Flushing a project", ->
+	before ->
+		@project_id = DocUpdaterClient.randomId()
+		@docs = [{
+			id: doc_id0 = DocUpdaterClient.randomId()
+			lines: ["one", "two", "three"]
+			update:
+				doc: doc_id0
+				op: [{
+					i: "one and a half\n"
+					p: 4
+				}]
+				v: 0
+			updatedLines: ["one", "one and a half", "two", "three"]
+		}, {
+			id: doc_id1 = DocUpdaterClient.randomId()
+			lines: ["four", "five", "six"]
+			update:
+				doc: doc_id1
+				op: [{
+					i: "four and a half\n"
+					p: 5
+				}]
+				v: 0
+			updatedLines: ["four", "four and a half", "five", "six"]
+		}]
+		for doc in @docs
+			MockWebApi.insertDoc @project_id, doc.id, {
+				lines: doc.lines
+			}
+
+	describe "with documents which have been updated", ->
+		before (done) ->
+			sinon.spy MockWebApi, "setDocumentLines"
+			async.series @docs.map((doc) =>
+				(callback) =>
+					DocUpdaterClient.preloadDoc @project_id, doc.id, (error) =>
+						return callback(error) if error?
+						DocUpdaterClient.sendUpdate @project_id, doc.id, doc.update, (error) =>
+							callback(error)
+			), (error) =>
+				throw error if error?
+				setTimeout () =>
+					DocUpdaterClient.flushProject @project_id, (error, res, body) =>
+						@statusCode = res.statusCode
+						done()
+				, 200
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+
+		it "should return a 204 status code", ->
+			@statusCode.should.equal 204
+
+		it "should send each document to the web api", ->
+			for doc in @docs
+				MockWebApi.setDocumentLines
+					.calledWith(@project_id, doc.id, doc.updatedLines)
+					.should.equal true
+
+		it "should update the lines in the doc updater", (done) ->
+			async.series @docs.map((doc) =>
+				(callback) =>
+					DocUpdaterClient.getDoc @project_id, doc.id, (error, res, returnedDoc) =>
+						returnedDoc.lines.should.deep.equal doc.updatedLines
+						callback()
+			), done
+

services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee

@@ -0,0 +1,97 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+async = require "async"
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+mongojs = require "../../../app/js/mongojs"
+db = mongojs.db
+ObjectId = mongojs.ObjectId
+
+describe "Flushing a doc to Mongo", ->
+	before ->
+		@lines = ["one", "two", "three"]
+		@update =
+			doc: @doc_id
+			op: [{
+				i: "one and a half\n"
+				p: 4
+			}]
+			v: 0
+		@result = ["one", "one and a half", "two", "three"]
+		MockWebApi.insertDoc @project_id, @doc_id, {
+			lines: @lines
+		}
+
+	describe "when the updated doc exists in the doc updater", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			sinon.spy MockWebApi, "setDocumentLines"
+
+			DocUpdaterClient.sendUpdates @project_id, @doc_id, [@update], (error) =>
+				throw error if error?
+				setTimeout () =>
+					DocUpdaterClient.flushDoc @project_id, @doc_id, done
+				, 200
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+
+		it "should flush the updated document to the web api", ->
+			MockWebApi.setDocumentLines
+				.calledWith(@project_id, @doc_id, @result)
+				.should.equal true
+
+		it "should flush the doc ops to Mongo", (done) ->
+			db.docOps.find doc_id: ObjectId(@doc_id), (error, docs) =>
+				doc = docs[0]
+				doc.docOps[0].op.should.deep.equal @update.op
+				done()
+
+	describe "when the doc has a large number of ops to be flushed", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			@updates = []
+			for v in [0..999]
+				@updates.push
+					doc_id: @doc_id,
+					op: [i: v.toString(), p: 0]
+					v: v
+
+			DocUpdaterClient.sendUpdates @project_id, @doc_id, @updates, (error) =>
+				throw error if error?
+				setTimeout () =>
+					DocUpdaterClient.flushDoc @project_id, @doc_id, done
+				, 200
+
+		it "should flush the doc ops to Mongo in order", (done) ->
+			db.docOps.find doc_id: ObjectId(@doc_id), (error, docs) =>
+				doc = docs[0]
+				updates = @updates.slice(-100)
+				for update, i in doc.docOps
+					update.op.should.deep.equal updates[i].op
+				done()
+
+	describe "when the doc does not exist in the doc updater", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines
+			}
+			sinon.spy MockWebApi, "setDocumentLines"
+			DocUpdaterClient.flushDoc @project_id, @doc_id, done
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+
+		it "should not flush the doc to the web api", ->
+			MockWebApi.setDocumentLines.called.should.equal false
+			
+

services/document-updater/test/acceptance/coffee/GettingADocumentTests.coffee

@@ -0,0 +1,107 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Getting a document", ->
+	describe "when the document is not loaded", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines = ["one", "two", "three"]
+			}
+			sinon.spy MockWebApi, "getDocument"
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done()
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should load the document from the web API", ->
+			MockWebApi.getDocument
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+		
+		it "should return the document lines", ->
+			@returnedDoc.lines.should.deep.equal @lines
+
+		it "should return the document at version 0", ->
+			@returnedDoc.version.should.equal 0
+
+	describe "when the document is already loaded", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines = ["one", "two", "three"]
+			}
+
+			DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
+				throw error if error?
+				sinon.spy MockWebApi, "getDocument"
+				DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) =>	done()
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should not load the document from the web API", ->
+			MockWebApi.getDocument.called.should.equal false
+
+		it "should return the document lines", ->
+			@returnedDoc.lines.should.deep.equal @lines
+
+	describe "when the request asks for some recent ops", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			MockWebApi.insertDoc @project_id, @doc_id, {
+				lines: @lines = ["one", "two", "three"]
+			}
+
+			@updates = for v in [0..99]
+				doc_id: @doc_id,
+				op: [i: v.toString(), p: 0]
+				v: v
+
+			DocUpdaterClient.sendUpdates @project_id, @doc_id, @updates, (error) =>
+				throw error if error?
+				sinon.spy MockWebApi, "getDocument"
+				DocUpdaterClient.getDocAndRecentOps @project_id, @doc_id, 90, (error, res, @returnedDoc) => done()
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should return the recent ops", ->
+			@returnedDoc.ops.length.should.equal 10
+			for update, i in @updates.slice(90, -1)
+				@returnedDoc.ops[i].op.should.deep.equal update.op
+
+
+	describe "when the document does not exist", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				@statusCode = res.statusCode
+				done()
+
+		it "should return 404", ->
+			@statusCode.should.equal 404
+
+	describe "when the web api returns an error", ->
+		before (done) ->
+			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+			sinon.stub MockWebApi, "getDocument", (project_id, doc_id, callback = (error, doc) ->) ->
+				callback new Error("oops")
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				@statusCode = res.statusCode
+				done()
+
+		after ->
+			MockWebApi.getDocument.restore()
+
+		it "should return 500", ->
+			@statusCode.should.equal 500
+
+
+
+
+

services/document-updater/test/acceptance/coffee/SettingADocumentTests.coffee

@@ -0,0 +1,58 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+
+MockWebApi = require "./helpers/MockWebApi"
+DocUpdaterClient = require "./helpers/DocUpdaterClient"
+
+describe "Setting a document", ->
+	before ->
+		[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
+		@lines = ["one", "two", "three"]
+		@update =
+			doc: @doc_id
+			op: [{
+				i: "one and a half\n"
+				p: 4
+			}]
+			v: 0
+		@result = ["one", "one and a half", "two", "three"]
+		@newLines = ["these", "are", "the", "new", "lines"]
+		MockWebApi.insertDoc @project_id, @doc_id, {
+			lines: @lines
+		}
+
+	describe "when the updated doc exists in the doc updater", ->
+		before (done) ->
+			sinon.spy MockWebApi, "setDocumentLines"
+			DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
+				throw error if error?
+				DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
+					throw error if error?
+					setTimeout () =>
+						DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, (error, res, body) =>
+							@statusCode = res.statusCode
+							done()
+					, 200
+
+		after ->
+			MockWebApi.setDocumentLines.restore()
+
+		it "should return a 204 status code", ->
+			@statusCode.should.equal 204
+
+		it "should send the updated document to the web api", ->
+			MockWebApi.setDocumentLines
+				.calledWith(@project_id, @doc_id, @newLines)
+				.should.equal true
+
+		it "should update the lines in the doc updater", (done) ->
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				doc.lines.should.deep.equal @newLines
+				done()
+
+		it "should bump the version in the doc updater", (done) ->
+			DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
+				doc.version.should.equal 2
+				done()
+

services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee

@@ -0,0 +1,66 @@
+rclient = require("redis").createClient()
+request = require("request").defaults(jar: false)
+async = require "async"
+
+module.exports = DocUpdaterClient =
+	randomId: () ->
+		return require("../../../../app/js/mongojs").ObjectId().toString()
+
+	sendUpdate: (project_id, doc_id, update, callback = (error) ->) ->
+		rclient.rpush "PendingUpdates:#{doc_id}", JSON.stringify(update), (error)->
+			return callback(error) if error?
+			doc_key = "#{project_id}:#{doc_id}"
+			rclient.sadd "DocsWithPendingUpdates", doc_key, (error) ->
+				return callback(error) if error?
+				rclient.publish "pending-updates", doc_key, callback
+
+	sendUpdates: (project_id, doc_id, updates, callback = (error) ->) ->
+		DocUpdaterClient.preloadDoc project_id, doc_id, (error) ->
+			return callback(error) if error?
+			jobs = []
+			for update in updates
+				do (update) ->
+					jobs.push (callback) ->
+						DocUpdaterClient.sendUpdate project_id, doc_id, update, callback
+			async.series jobs, callback
+
+	getDoc: (project_id, doc_id, callback = (error, res, body) ->) ->
+		request.get "http://localhost:3003/project/#{project_id}/doc/#{doc_id}", (error, res, body) ->
+			if body? and res.statusCode >= 200 and res.statusCode < 300
+				body = JSON.parse(body)
+			callback error, res, body
+
+	getDocAndRecentOps: (project_id, doc_id, fromVersion, callback = (error, res, body) ->) ->
+		request.get "http://localhost:3003/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}", (error, res, body) ->
+			if body? and res.statusCode >= 200 and res.statusCode < 300
+				body = JSON.parse(body)
+			callback error, res, body
+
+	preloadDoc: (project_id, doc_id, callback = (error) ->) ->
+		DocUpdaterClient.getDoc project_id, doc_id, callback
+
+	flushDoc: (project_id, doc_id, callback = (error) ->) ->
+		request.post "http://localhost:3003/project/#{project_id}/doc/#{doc_id}/flush", (error, res, body) ->
+			callback error, res, body
+
+	setDocLines: (project_id, doc_id, lines, callback = (error) ->) ->
+		request.post {
+			url: "http://localhost:3003/project/#{project_id}/doc/#{doc_id}"
+			json:
+				lines: lines
+		}, (error, res, body) ->
+			callback error, res, body
+
+	deleteDoc: (project_id, doc_id, callback = (error) ->) ->
+		request.del "http://localhost:3003/project/#{project_id}/doc/#{doc_id}", (error, res, body) ->
+			callback error, res, body
+
+	flushProject: (project_id, callback = () ->) ->
+		request.post "http://localhost:3003/project/#{project_id}/flush", callback
+
+	deleteProject: (project_id, callback = () ->) ->
+		request.del "http://localhost:3003/project/#{project_id}", callback
+		
+		
+		
+	

services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee

@@ -0,0 +1,40 @@
+express = require("express")
+app = express()
+
+module.exports = MockWebApi =
+	docs: {}
+
+	clearDocs: () -> @docs = {}
+
+	insertDoc: (project_id, doc_id, doc) ->
+		@docs["#{project_id}:#{doc_id}"] = doc
+
+	setDocumentLines: (project_id, doc_id, lines, callback = (error) ->) ->
+		@docs["#{project_id}:#{doc_id}"] ||= {}
+		@docs["#{project_id}:#{doc_id}"].lines = lines
+		callback null
+
+	getDocument: (project_id, doc_id, callback = (error, doc) ->) ->
+		callback null, @docs["#{project_id}:#{doc_id}"]
+
+	run: () ->
+		app.get "/project/:project_id/doc/:doc_id", (req, res, next) =>
+			@getDocument req.params.project_id, req.params.doc_id, (error, doc) ->
+				if error?
+					res.send 500
+				else if doc?
+					res.send JSON.stringify doc
+				else
+					res.send 404
+
+		app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) =>
+			@setDocumentLines req.params.project_id, req.params.doc_id, req.body.lines, (error) ->
+				if error?
+					res.send 500
+				else
+					res.send 204
+
+		app.listen(3000)
+
+MockWebApi.run()
+

services/document-updater/test/unit/coffee/AddingDocsToMemory.coffee

@@ -0,0 +1,58 @@
+require('coffee-script')
+assert = require('assert')
+path = require('path')
+modulePath = path.join __dirname, '../../../app/js/RedisManager.js'
+keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js')
+project_id = 1234
+doc_id     = 5678
+loadModule = require('./module-loader').loadModule
+
+describe 'putting a doc into memory', ()->
+	lines = ["this is one line", "and another line"]
+	version = 42
+
+	potentialSets = {}
+	potentialSets[keys.docLines(doc_id:doc_id)] = lines
+	potentialSets[keys.projectKey(doc_id:doc_id)] = project_id
+	potentialSets[keys.docVersion(doc_id:doc_id)] = version
+
+	potentialSAdds = {}
+	potentialSAdds[keys.allDocs] = doc_id
+	potentialSAdds[keys.docsInProject(project_id:project_id)] = doc_id
+
+	potentialDels = {}
+	potentialDels[keys.docOps(doc_id:doc_id)] = true
+
+	mocks =
+		"logger-sharelatex": log:->
+		redis:
+			createClient : ()->
+				auth:->
+				multi: ()->
+					set:(key, value)->
+						result = potentialSets[key]
+						delete potentialSets[key]
+						if key == keys.docLines(doc_id:doc_id)
+							value = JSON.parse(value)
+						assert.deepEqual result, value
+					incr:()->
+					sadd:(key, value)->
+						result = potentialSAdds[key]
+						delete potentialSAdds[key]
+						assert.equal result, value
+					del: (key) ->
+						result = potentialDels[key]
+						delete potentialDels[key]
+						assert.equal result, true
+					exec:(callback)->
+						callback()
+	
+	redisManager = loadModule(modulePath, mocks).module.exports
+
+	it 'should put a all data into memory', (done)->
+		redisManager.putDocInMemory project_id, doc_id, lines, version, ()->
+			assert.deepEqual potentialSets, {}
+			assert.deepEqual potentialSAdds, {}
+			assert.deepEqual potentialDels, {}
+			done()
+

services/document-updater/test/unit/coffee/CheckingUpdatesLength.coffee

@@ -0,0 +1,27 @@
+assert = require('chai').assert
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../app/js/RedisManager.js"
+SandboxedModule = require('sandboxed-module')
+
+doc_id = "1234"
+
+describe 'Document Manager - getUpdatesLength ', ->
+
+	beforeEach ->
+
+		@llenStub = sinon.stub()
+		@redisManager = SandboxedModule.require modulePath, requires:
+			redis:
+				createClient:=>
+					auth:->
+					llen:@llenStub
+
+	it "should the number of things to process in the que", (done)->
+
+		@llenStub.callsArgWith(1, null, 3)
+		@redisManager.getUpdatesLength doc_id, (err, len)=>
+			@llenStub.calledWith("PendingUpdates:#{doc_id}").should.equal true
+			len.should.equal 3
+			done()

services/document-updater/test/unit/coffee/DiffCodec/DiffCodecTests.coffee

@@ -0,0 +1,56 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../../app/js/DiffCodec.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DiffCodec", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@DiffCodec = SandboxedModule.require modulePath
+
+	describe "diffAsShareJsOps", ->
+		it "should insert new text correctly", (done) ->
+			@before = ["hello world"]
+			@after  = ["hello beautiful world"]
+			@DiffCodec.diffAsShareJsOp @before, @after, (error, ops) ->
+				expect(ops).to.deep.equal [
+					i: "beautiful "
+					p: 6
+				]
+				done()
+
+		it "should shift later inserts by previous inserts", (done) ->
+			@before = ["the boy played with the ball"]
+			@after  = ["the tall boy played with the red ball"]
+			@DiffCodec.diffAsShareJsOp @before, @after, (error, ops) ->
+				expect(ops).to.deep.equal [
+					{ i: "tall ", p: 4 }
+					{ i: "red ", p: 29 }
+				]
+				done()
+
+		it "should delete text correctly", (done) ->
+			@before  = ["hello beautiful world"]
+			@after = ["hello world"]
+			@DiffCodec.diffAsShareJsOp @before, @after, (error, ops) ->
+				expect(ops).to.deep.equal [
+					d: "beautiful "
+					p: 6
+				]
+				done()
+
+
+		it "should shift later deletes by the first deletes", (done) ->
+			@before = ["the tall boy played with the red ball"]
+			@after  = ["the boy played with the ball"]
+			@DiffCodec.diffAsShareJsOp @before, @after, (error, ops) ->
+				expect(ops).to.deep.equal [
+					{ d: "tall ", p: 4 }
+					{ d: "red ", p: 24 }
+				]
+				done()
+			
+			
+

services/document-updater/test/unit/coffee/DocOpsManager/DocOpsManagerTests.coffee

@@ -0,0 +1,309 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocOpsManager.js"
+SandboxedModule = require('sandboxed-module')
+ObjectId = require("../../../../app/js/mongojs").ObjectId
+
+describe "DocOpsManager", ->
+	beforeEach ->
+		@doc_id = ObjectId().toString()
+		@project_id = "project-id"
+		@callback = sinon.stub()
+		@DocOpsManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./mongojs":
+				db: @db = { docOps: {} }
+				ObjectId: ObjectId
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+
+	describe "flushDocOpsToMongo", ->
+		describe "when versions are consistent", ->
+			beforeEach ->
+				@mongo_version = 40
+				@redis_version = 42
+				@ops = [ "mock-op-1", "mock-op-2" ]
+				@DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version)
+				@RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version)
+				@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
+				@DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3)
+				@DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback
+
+			it "should get the version from Mongo", ->
+				@DocOpsManager.getDocVersionInMongo
+					.calledWith(@doc_id)
+					.should.equal true
+
+			it "should get the version from REdis", ->
+				@RedisManager.getDocVersion
+					.calledWith(@doc_id)
+					.should.equal true
+
+			it "should get all doc ops since the version in Mongo", ->
+				@RedisManager.getPreviousDocOps
+					.calledWith(@doc_id, @mongo_version, -1)
+					.should.equal true
+
+			it "should update Mongo with the new ops", ->
+				@DocOpsManager._appendDocOpsInMongo
+					.calledWith(@doc_id, @ops, @redis_version)
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+			it "should time the execution", ->
+				@Metrics.Timer::done.called.should.equal true
+
+		describe "when the number of ops does not match the difference in versions", ->
+			beforeEach ->
+				@mongo_version = 40
+				@redis_version = 45
+				@ops = [ "mock-op-1", "mock-op-2" ]
+				@DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version)
+				@RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version)
+				@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
+				@DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3)
+				@DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error("inconsistet versions")).should.equal true
+
+			it "should log an error", ->
+				@logger.error
+					.calledWith(doc_id: @doc_id, mongoVersion: @mongo_version, redisVersion: @redis_version, opsLength: @ops.length, "version difference does not match ops length")
+					.should.equal true
+
+			it "should not modify mongo", ->
+				@DocOpsManager._appendDocOpsInMongo.called.should.equal false
+
+			it "should time the execution", ->
+				@Metrics.Timer::done.called.should.equal true
+
+		describe "when redis version is behind mongo version", ->
+			beforeEach ->
+				@mongo_version = 40
+				@redis_version = 30
+				@DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version)
+				@RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version)
+				@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
+				@DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3)
+				@DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback
+			
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error("inconsistet versions")).should.equal true
+
+			it "should log an error", ->
+				@logger.error
+					.calledWith(doc_id: @doc_id, mongoVersion: @mongo_version, redisVersion: @redis_version, "mongo version is ahead of redis")
+					.should.equal true
+
+			it "should not modify mongo", ->
+				@DocOpsManager._appendDocOpsInMongo.called.should.equal false
+
+			it "should time the execution", ->
+				@Metrics.Timer::done.called.should.equal true
+
+	describe "getPreviousDocOps", ->
+		beforeEach ->
+			@ops = [ "mock-op-1", "mock-op-2" ]
+			@start = 30
+			@end = 32
+			@RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops)
+			@DocOpsManager._ensureOpsAreLoaded = sinon.stub().callsArg(3)
+			@DocOpsManager.getPreviousDocOps @project_id, @doc_id, @start, @end, @callback
+
+		it "should ensure the ops are loaded back far enough", ->
+			@DocOpsManager._ensureOpsAreLoaded
+				.calledWith(@project_id, @doc_id, @start)
+				.should.equal true
+
+		it "should get the previous doc ops", ->
+			@RedisManager.getPreviousDocOps
+				.calledWith(@doc_id, @start, @end)
+				.should.equal true
+
+		it "should call the callback with the ops", ->
+			@callback.calledWith(null, @ops).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "_ensureOpsAreLoaded", ->
+		describe "when the ops are not loaded", ->
+			beforeEach ->
+				@redisVersion = 42
+				@redisOpsLength = 10
+				@backToVersion = 30
+				@ops = [ "mock-op-1", "mock-op-2" ]
+				@RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redisVersion)
+				@RedisManager.getDocOpsLength = sinon.stub().callsArgWith(1, null, @redisOpsLength)
+				@DocOpsManager._getDocOpsFromMongo = sinon.stub().callsArgWith(3, null, @ops)
+				@RedisManager.prependDocOps = sinon.stub().callsArgWith(2, null)
+				@DocOpsManager._ensureOpsAreLoaded @project_id, @doc_id, @backToVersion, @callback
+
+			it "should get the doc version from redis", ->
+				@RedisManager.getDocVersion
+					.calledWith(@doc_id)
+					.should.equal true
+
+			it "should get the doc ops length in redis", ->
+				@RedisManager.getDocOpsLength
+					.calledWith(@doc_id)
+					.should.equal true
+
+			it "should get the doc ops that need loading from Mongo", ->
+				@DocOpsManager._getDocOpsFromMongo
+					.calledWith(@doc_id, @backToVersion, @redisVersion - @redisOpsLength)
+					.should.equal true
+
+			it "should prepend the retrieved ops to redis", ->
+				@RedisManager.prependDocOps
+					.calledWith(@doc_id, @ops)
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "when the ops are loaded", ->
+			beforeEach ->
+				@redisVersion = 42
+				@redisOpsLength = 10
+				@backToVersion = 35
+				@RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redisVersion)
+				@RedisManager.getDocOpsLength = sinon.stub().callsArgWith(1, null, @redisOpsLength)
+				@DocOpsManager._getDocOpsFromMongo = sinon.stub().callsArgWith(3, null, @ops)
+				@RedisManager.prependDocOps = sinon.stub().callsArgWith(2, null)
+				@DocOpsManager._ensureOpsAreLoaded @project_id, @doc_id, @backToVersion, @callback
+
+			it "should not need to get the docs from Mongo or put any into redis", ->
+				@DocOpsManager._getDocOpsFromMongo.called.should.equal false
+				@RedisManager.prependDocOps.called.should.equal false
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+	describe "getDocVersionInMongo", ->
+		describe "when the doc exists", ->
+			beforeEach ->
+				@doc =
+					version: @version = 42
+				@db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc])
+				@DocOpsManager.getDocVersionInMongo @doc_id, @callback
+
+			it "should look for the doc in the database", ->
+				@db.docOps.find
+					.calledWith({ doc_id: ObjectId(@doc_id) }, {version: 1})
+					.should.equal true
+
+			it "should call the callback with the version", ->
+				@callback.calledWith(null, @version).should.equal true
+
+		describe "when the doc doesn't exist", ->
+			beforeEach ->
+				@db.docOps.find = sinon.stub().callsArgWith(2, null, [])
+				@DocOpsManager.getDocVersionInMongo @doc_id, @callback
+			
+			it "should call the callback with 0", ->
+				@callback.calledWith(null, 0).should.equal true
+
+	describe "_appendDocOpsInMongo", ->
+		describe "with a small set of updates", ->
+			beforeEach (done) ->
+				@ops = [ "mock-op-1", "mock-op-2" ]
+				@version = 42
+				@db.docOps.update = sinon.stub().callsArg(3)
+				@DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) =>
+					@callback(error)
+					done()
+
+			it "should update the database", ->
+				@db.docOps.update
+					.calledWith({
+						doc_id: ObjectId(@doc_id)
+					}, {
+						$push: docOps: { $each: @ops, $slice: -100 }
+						$set: version: @version
+					}, {
+						upsert: true
+					})
+					.should.equal true
+
+			it "should call the callbak", ->
+				@callback.called.should.equal true
+
+		describe "with a large set of updates", ->
+			beforeEach (done) ->
+				@ops = [ "mock-op-1", "mock-op-2", "mock-op-3", "mock-op-4", "mock-op-5" ]
+				@version = 42
+				@DocOpsManager.APPEND_OPS_BATCH_SIZE = 2
+				@db.docOps.update = sinon.stub().callsArg(3)
+				@DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) =>
+					@callback(error)
+					done()
+
+			it "should update the database in batches", ->
+				@db.docOps.update
+					.calledWith({ doc_id: ObjectId(@doc_id) }, {
+						$push: docOps: { $each: @ops.slice(0,2), $slice: -100 }
+						$set: version: @version - 3
+					}, { upsert: true })
+					.should.equal true
+				@db.docOps.update
+					.calledWith({ doc_id: ObjectId(@doc_id) }, {
+						$push: docOps: { $each: @ops.slice(2,4), $slice: -100 }
+						$set: version: @version - 1
+					}, { upsert: true })
+					.should.equal true
+				@db.docOps.update
+					.calledWith({ doc_id: ObjectId(@doc_id) }, {
+						$push: docOps: { $each: @ops.slice(4,5), $slice: -100 }
+						$set: version: @version
+					}, { upsert: true })
+					.should.equal true
+
+			it "should call the callbak", ->
+				@callback.called.should.equal true
+
+		describe "with no updates", ->
+			beforeEach (done) ->
+				@ops = []
+				@version = 42
+				@db.docOps.update = sinon.stub().callsArg(3)
+				@DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) =>
+					@callback(error)
+					done()
+
+			it "should not try to update the database", ->
+				@db.docOps.update.called.should.equal false
+
+	describe "_getDocsOpsFromMongo", ->
+		beforeEach ->
+			@version = 42
+			@start = 32
+			@limit = 5
+			@doc =
+				docOps: ["mock-ops"]
+			@DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @version)
+			@db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc])
+			@DocOpsManager._getDocOpsFromMongo @doc_id, @start, @start + @limit, @callback
+
+		it "should get the current version", ->
+			@DocOpsManager.getDocVersionInMongo
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should get the doc ops", ->
+			@db.docOps.find
+				.calledWith({ doc_id: ObjectId(@doc_id) }, {
+					docOps: $slice: [-(@version - @start), @limit]
+				})
+				.should.equal true
+
+		it "should return the ops", ->
+			@callback.calledWith(null, @doc.docOps).should.equal true
+
+	

services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee

@@ -0,0 +1,41 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocumentManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocumentUpdater - flushAndDeleteDoc", ->
+	beforeEach ->
+		@DocumentManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./PersistenceManager": @PersistenceManager = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"./DocOpsManager" :{}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@callback = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@RedisManager.removeDocFromMemory = sinon.stub().callsArg(2)
+			@DocumentManager.flushDocIfLoaded = sinon.stub().callsArgWith(2)
+			@DocumentManager.flushAndDeleteDoc @project_id, @doc_id, @callback
+		
+		it "should flush the doc", ->
+			@DocumentManager.flushDocIfLoaded
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should remove the doc from redis", ->
+			@RedisManager.removeDocFromMemory
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true

services/document-updater/test/unit/coffee/DocumentManager/flushDocTests.coffee

@@ -0,0 +1,73 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocumentManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocumentUpdater - flushDocIfLoaded", ->
+	beforeEach ->
+		@DocumentManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./PersistenceManager": @PersistenceManager = {}
+			"./DocOpsManager": @DocOpsManager = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@version = 42
+		@callback = sinon.stub()
+
+	describe "when the doc is in Redis", ->
+		beforeEach ->
+			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, @lines, @version)
+			@PersistenceManager.setDoc = sinon.stub().callsArgWith(3)
+			@DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2)
+			@DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback
+
+		it "should get the doc from redis", ->
+			@RedisManager.getDoc
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should write the doc lines to the persistence layer", ->
+			@PersistenceManager.setDoc
+				.calledWith(@project_id, @doc_id, @lines)
+				.should.equal true
+		
+		it "should write the doc ops to mongo", ->
+			@DocOpsManager.flushDocOpsToMongo
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the document is not in Redis", ->
+		beforeEach ->
+			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, null, null)
+			@PersistenceManager.setDoc = sinon.stub().callsArgWith(3)
+			@DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2)
+			@DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback
+
+		it "should get the doc from redis", ->
+			@RedisManager.getDoc
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should not write anything to the persistence layer", ->
+			@PersistenceManager.setDoc.called.should.equal false
+			@DocOpsManager.flushDocOpsToMongo.called.should.equal false
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+		
+

services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee

@@ -0,0 +1,67 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocumentManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocumentUpdater - getDocAndRecentOps", ->
+	beforeEach ->
+		@DocumentManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./PersistenceManager": @PersistenceManager = {}
+			"./DocOpsManager": @DocOpsManager = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@version = 42
+		@fromVersion = 40
+		@ops = ["mock-op-1", "mock-op-2"]
+		@callback = sinon.stub()
+
+	describe "with a previous version specified", ->
+		beforeEach ->
+			@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version)
+			@DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops)
+			@DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback
+
+		it "should get the doc", ->
+			@DocumentManager.getDoc
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should get the doc ops", ->
+			@DocOpsManager.getPreviousDocOps
+				.calledWith(@project_id, @doc_id, @fromVersion, @version)
+				.should.equal true
+
+		it "should call the callback with the doc info", ->
+			@callback.calledWith(null, @lines, @version, @ops).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "with no previous version specified", ->
+		beforeEach ->
+			@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version)
+			@DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops)
+			@DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback
+
+		it "should get the doc", ->
+			@DocumentManager.getDoc
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should not need to get the doc ops", ->
+			@DocOpsManager.getPreviousDocOps.called.should.equal false
+
+		it "should call the callback with the doc info", ->
+			@callback.calledWith(null, @lines, @version, []).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+

services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee

@@ -0,0 +1,75 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocumentManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocumentUpdater - getDoc", ->
+	beforeEach ->
+		@DocumentManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./PersistenceManager": @PersistenceManager = {}
+			"./DocOpsManager": @DocOpsManager = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@version = 42
+		@callback = sinon.stub()
+
+	describe "when the doc exists in Redis", ->
+		beforeEach ->
+			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, @lines, @version)
+			@DocumentManager.getDoc @project_id, @doc_id, @callback
+
+		it "should get the doc from Redis", ->
+			@RedisManager.getDoc
+				.calledWith(@doc_id)
+				.should.equal true
+		
+		it "should call the callback with the doc info", ->
+			@callback.calledWith(null, @lines, @version).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the doc does not exist in Redis", ->
+		beforeEach ->
+			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, null, null)
+			@PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines)
+			@DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @version)
+			@RedisManager.putDocInMemory = sinon.stub().callsArg(4)
+			@DocumentManager.getDoc @project_id, @doc_id, @callback
+
+		it "should try to get the doc from Redis", ->
+			@RedisManager.getDoc
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should get the doc version from Mongo", ->
+			@DocOpsManager.getDocVersionInMongo
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should get the doc from the PersistenceManager", ->
+			@PersistenceManager.getDoc
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should set the doc in Redis", ->
+			@RedisManager.putDocInMemory
+				.calledWith(@project_id, @doc_id, @lines, @version)
+				.should.equal true
+
+		it "should call the callback with the doc info", ->
+			@callback.calledWith(null, @lines, @version).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+		
+
+

services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee

@@ -0,0 +1,105 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/DocumentManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocumentManager - setDoc", ->
+	beforeEach ->
+		@DocumentManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./PersistenceManager": @PersistenceManager = {}
+			"./DiffCodec": @DiffCodec = {}
+			"./DocOpsManager":{}
+			"./UpdateManager": @UpdateManager = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@version = 42
+		@ops = ["mock-ops"]
+		@callback = sinon.stub()
+
+	describe "with plain tex lines", ->
+		beforeEach ->
+			@beforeLines = ["before", "lines"]
+			@afterLines = ["after", "lines"]
+
+		describe "successfully", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version)
+				@DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops)
+				@UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null)
+				@DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2)
+				@DocumentManager.setDoc @project_id, @doc_id, @afterLines, @callback
+
+			it "should get the current doc lines", ->
+				@DocumentManager.getDoc
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+
+			it "should return a diff of the old and new lines", ->
+				@DiffCodec.diffAsShareJsOp
+					.calledWith(@beforeLines, @afterLines)
+					.should.equal true
+
+			it "should apply the diff as a ShareJS op", ->
+				@UpdateManager.applyUpdates
+					.calledWith(@project_id, @doc_id, [doc: @doc_id, v: @version, op: @ops, meta: { type: "external" }])
+					.should.equal true
+
+			it "should flush the doc to Mongo", ->
+				@DocumentManager.flushDocIfLoaded
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+			
+			it "should call the callback", ->
+				@callback.calledWith(null).should.equal true
+
+			it "should time the execution", ->
+				@Metrics.Timer::done.called.should.equal true
+
+	describe "with json lines", ->
+		beforeEach ->
+			@beforeLines = [text: "before", text: "lines"]
+			@afterLines = ["after", "lines"]
+			
+		describe "successfully", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version)
+				@DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops)
+				@UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null)
+				@DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2)
+				@DocumentManager.setDoc @project_id, @doc_id, @afterLines, @callback
+
+			it "should get the current doc lines", ->
+				@DocumentManager.getDoc
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+
+			it "should return not try to get a diff", ->
+				@DiffCodec.diffAsShareJsOp.called.should.equal false
+
+			it "should call the callback", ->
+				@callback.calledWith(null).should.equal true
+
+	describe "without new lines", ->
+		beforeEach ->
+			@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version)
+			@DocumentManager.setDoc @project_id, @doc_id, null, @callback
+
+		it "should return teh callback with an error", ->
+			@callback.calledWith(new Error("No lines were passed to setDoc"))
+			
+		it "should not try to get the doc lines", ->
+			@DocumentManager.getDoc.called.should.equal false
+			
+		
+
+		
+
+
+

services/document-updater/test/unit/coffee/GettingDoc.coffee

@@ -0,0 +1,41 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../app/js/RedisManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe 'RedisManager - getDoc', ->
+	beforeEach ->
+		@rclient = {}
+		@rclient.auth = () ->
+		@rclient.multi = () => @rclient
+
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis": @redis =
+				createClient: () => @rclient
+
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@jsonlines = JSON.stringify @lines
+		@version = 42
+		@callback = sinon.stub()
+
+		@rclient.get = sinon.stub()
+		@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version])
+
+		@RedisManager.getDoc @doc_id, @callback
+
+	it "should get the lines from redis", ->
+		@rclient.get
+			.calledWith("doclines:#{@doc_id}")
+			.should.equal true
+	
+	it "should get the version from", ->
+		@rclient.get
+			.calledWith("DocVersion:#{@doc_id}")
+			.should.equal true
+
+	it 'should return the document', ->
+		@callback
+			.calledWith(null, @lines, @version)
+			.should.equal true

services/document-updater/test/unit/coffee/GettingListOfPendingUpdates.coffee

@@ -0,0 +1,42 @@
+assert = require('assert')
+should = require('chai').should()
+path = require('path')
+modulePath = path.join __dirname, '../../../app/js/RedisManager.js'
+_ = require('underscore')
+loadModule = require('./module-loader').loadModule
+keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js')
+
+describe 'getting entire list of pending updates', ()->
+
+	doc_id = 123
+	redisMemory = {}
+	correctUpdates = [{"update1"}, {"update2"}, {"update3"}]
+	jsonCorrectUpdates = _.map correctUpdates, (d)-> JSON.stringify d
+	redisMemory[keys.pendingUpdates(doc_id:doc_id)] = jsonCorrectUpdates
+	redisMemory[keys.pendingUpdates(doc_id:"notThis")] = JSON.stringify([{"updatex"}, {"updatez"}])
+
+	redisReturn = []
+
+	mocks =
+		redis:
+			createClient: ()->
+				auth:->
+				multi: ()->
+					lrange:(key, start, end)->
+						key.should.equal(keys.pendingUpdates(doc_id:doc_id))
+						start.should.equal(0)
+						end.should.equal(-1)
+						redisReturn.push(redisMemory[key])
+					del : (key)->
+						key.should.equal(keys.pendingUpdates(doc_id:doc_id))
+						redisReturn.push(1)
+					exec: (callback)->
+						callback(null, redisReturn)
+	
+	redisManager = loadModule(modulePath, mocks).module.exports
+
+	it 'should have 3 elements in array', (done)->
+		redisManager.getPendingUpdatesForDoc doc_id, (err, listOfUpdates)->
+			listOfUpdates.length.should.equal(3)
+			done()
+

services/document-updater/test/unit/coffee/GettingTotalNumberOfDocs.coffee

@@ -0,0 +1,47 @@
+require('coffee-script')
+assert = require('assert')
+should = require('chai').should()
+path = require('path')
+modulePath = path.join __dirname, '../../../app/js/RedisManager.js'
+keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js')
+loadModule = require('./module-loader').loadModule
+
+describe 'getting cound of docs from memory', ()->
+
+	project_id = "12345"
+	doc_id1     = "docid1"
+	doc_id2     = "docid2"
+	doc_id3     = "docid3"
+	
+	redisMemory = {}
+	redisManager = undefined
+
+	beforeEach (done)->
+		mocks =
+			"logger-sharelatex": log:->
+			redis:
+				createClient : ()->
+					auth:->
+					smembers:(key, callback)->
+						callback(null, redisMemory[key])
+					multi: ()->
+						set:(key, value)->
+							redisMemory[key] = value
+						sadd:(key, value)->
+							if !redisMemory[key]?
+								redisMemory[key] = []
+							redisMemory[key].push value
+						del:()->
+						exec:(callback)->
+							callback()
+		
+		redisManager = loadModule(modulePath, mocks).module.exports
+		redisManager.putDocInMemory project_id, doc_id1, 0, ["line"], ->
+			redisManager.putDocInMemory project_id, doc_id2, 0, ["ledf"], ->
+				redisManager.putDocInMemory project_id, doc_id3, 0, ["ledf"], ->
+					done()
+
+	it 'should return total', (done)->
+		redisManager.getCountOfDocsInMemory (err, count)->
+			assert.equal count, 3
+			done()

services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee

@@ -0,0 +1,63 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - deleteProject", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager": @ProjectManager = {}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+		@next = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1)
+			@HttpController.deleteProject(@req, @res, @next)
+
+		it "should delete the project", ->
+			@ProjectManager.flushAndDeleteProjectWithLocks
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should return a successful No Content response", ->
+			@res.send
+				.calledWith(204)
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(project_id: @project_id, "deleting project via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops"))
+			@HttpController.deleteProject(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+
+
+
+

services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee

@@ -0,0 +1,64 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - flushAndDeleteDoc", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager":{}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+				doc_id: @doc_id
+		@next = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2)
+			@HttpController.flushAndDeleteDoc(@req, @res, @next)
+
+		it "should flush and delete the doc", ->
+			@DocumentManager.flushAndDeleteDocWithLock
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should return a successful No Content response", ->
+			@res.send
+				.calledWith(204)
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(doc_id: @doc_id, project_id: @project_id, "deleting doc via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2, new Error("oops"))
+			@HttpController.flushAndDeleteDoc(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+
+
+

services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee

@@ -0,0 +1,65 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - flushDocIfLoaded", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager": {}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@version = 42
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+				doc_id: @doc_id
+		@next = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2)
+			@HttpController.flushDocIfLoaded(@req, @res, @next)
+
+		it "should flush the doc", ->
+			@DocumentManager.flushDocIfLoadedWithLock
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should return a successful No Content response", ->
+			@res.send
+				.calledWith(204)
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(doc_id: @doc_id, project_id: @project_id, "flushing doc via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2, new Error("oops"))
+			@HttpController.flushDocIfLoaded(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+
+

services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee

@@ -0,0 +1,62 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - flushProject", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager": @ProjectManager = {}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+		@next = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1)
+			@HttpController.flushProject(@req, @res, @next)
+
+		it "should flush the project", ->
+			@ProjectManager.flushProjectWithLocks
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should return a successful No Content response", ->
+			@res.send
+				.calledWith(204)
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(project_id: @project_id, "flushing project via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops"))
+			@HttpController.flushProject(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+
+
+

services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee

@@ -0,0 +1,110 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - getDoc", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager": {}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@ops = ["mock-op-1", "mock-op-2"]
+		@version = 42
+		@fromVersion = 42
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+				doc_id: @doc_id
+		@next = sinon.stub()
+
+	describe "when the document exists and no recent ops are requested", ->
+		beforeEach ->
+			@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [])
+			@HttpController.getDoc(@req, @res, @next)
+
+		it "should get the doc", ->
+			@DocumentManager.getDocAndRecentOpsWithLock
+				.calledWith(@project_id, @doc_id, -1)
+				.should.equal true
+
+		it "should return the doc as JSON", ->
+			@res.send
+				.calledWith(JSON.stringify({
+					id: @doc_id
+					lines: @lines
+					version: @version
+					ops: []
+				}))
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when recent ops are requested", ->
+		beforeEach ->
+			@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, @ops)
+			@req.query = fromVersion: "#{@fromVersion}"
+			@HttpController.getDoc(@req, @res, @next)
+
+		it "should get the doc", ->
+			@DocumentManager.getDocAndRecentOpsWithLock
+				.calledWith(@project_id, @doc_id, @fromVersion)
+				.should.equal true
+
+		it "should return the doc as JSON", ->
+			@res.send
+				.calledWith(JSON.stringify({
+					id: @doc_id
+					lines: @lines
+					version: @version
+					ops: @ops
+				}))
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the document does not exist", ->
+		beforeEach ->
+			@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, null, null)
+			@HttpController.getDoc(@req, @res, @next)
+
+		it "should call next with NotFoundError", ->
+			@next
+				.calledWith(new Errors.NotFoundError("not found"))
+				.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, new Error("oops"), null, null)
+			@HttpController.getDoc(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+

services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee

@@ -0,0 +1,67 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/HttpController.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors.js"
+
+describe "HttpController - setDoc", ->
+	beforeEach ->
+		@HttpController = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./ProjectManager": {}
+			"logger-sharelatex" : @logger = { log: sinon.stub() }
+			"./Metrics": @Metrics = {}
+
+		@Metrics.Timer = class Timer
+			done: sinon.stub()
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@res =
+			send: sinon.stub()
+		@req =
+			params:
+				project_id: @project_id
+				doc_id: @doc_id
+			body:
+				lines: @lines
+		@next = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@DocumentManager.setDocWithLock = sinon.stub().callsArgWith(3)
+			@HttpController.setDoc(@req, @res, @next)
+
+		it "should set the doc", ->
+			@DocumentManager.setDocWithLock
+				.calledWith(@project_id, @doc_id)
+				.should.equal true
+
+		it "should return a successful No Content response", ->
+			@res.send
+				.calledWith(204)
+				.should.equal true
+
+		it "should log the request", ->
+			@logger.log
+				.calledWith(doc_id: @doc_id, project_id: @project_id, lines: @lines, "setting doc via http")
+				.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when an errors occurs", ->
+		beforeEach ->
+			@DocumentManager.setDocWithLock = sinon.stub().callsArgWith(3, new Error("oops"))
+			@HttpController.setDoc(@req, @res, @next)
+
+		it "should call next with the error", ->
+			@next
+				.calledWith(new Error("oops"))
+				.should.equal true
+		
+
+
+

services/document-updater/test/unit/coffee/LockManager/CheckingTheLock.coffee

@@ -0,0 +1,50 @@
+require('coffee-script')
+sinon = require('sinon')
+assert = require('assert')
+path = require('path')
+modulePath = path.join __dirname, '../../../../app/js/LockManager.js'
+keys = require(path.join __dirname, '../../../../app/js/RedisKeyBuilder.js')
+project_id = 1234
+doc_id     = 5678
+blockingKey = "Blocking:#{doc_id}"
+loadModule = require('../module-loader').loadModule
+
+describe 'Lock Manager - checking the lock', ()->
+
+	existsStub = sinon.stub()
+	setStub = sinon.stub()
+	exireStub = sinon.stub()
+	execStub = sinon.stub()
+	
+	mocks =
+		"logger-sharelatex": log:->
+
+		redis:
+			createClient : ()->
+				auth:->
+				multi: ->
+					exists: existsStub
+					expire: exireStub
+					set: setStub
+					exec: execStub
+	LockManager = loadModule(modulePath, mocks).module.exports
+
+	it 'should check if lock exists but not set or expire', (done)->
+		execStub.callsArgWith(0, null, ["1"])
+		LockManager.checkLock doc_id, (err, docIsLocked)->
+			existsStub.calledWith(blockingKey).should.equal true
+			setStub.called.should.equal false
+			exireStub.called.should.equal false
+			done()
+
+	it 'should return true if the key does not exists', (done)->
+		execStub.callsArgWith(0, null, "0")
+		LockManager.checkLock doc_id, (err, free)->
+			free.should.equal true
+			done()
+
+	it 'should return false if the key does exists', (done)->
+		execStub.callsArgWith(0, null, "1")
+		LockManager.checkLock doc_id, (err, free)->
+			free.should.equal false
+			done()

services/document-updater/test/unit/coffee/LockManager/ReleasingTheLock.coffee

@@ -0,0 +1,28 @@
+require('coffee-script')
+sinon = require('sinon')
+assert = require('assert')
+path = require('path')
+modulePath = path.join __dirname, '../../../../app/js/LockManager.js'
+keys = require(path.join __dirname, '../../../../app/js/RedisKeyBuilder.js')
+project_id = 1234
+doc_id     = 5678
+loadModule = require('../module-loader').loadModule
+
+describe 'LockManager - releasing the lock', ()->
+
+	deleteStub = sinon.stub().callsArgWith(1)
+	mocks =
+		"logger-sharelatex": log:->
+
+		redis:
+			createClient : ()->
+				auth:->
+				del:deleteStub
+	
+	LockManager = loadModule(modulePath, mocks).module.exports
+
+	it 'should put a all data into memory', (done)->
+		LockManager.releaseLock doc_id, ->
+			deleteStub.calledWith("Blocking:#{doc_id}").should.equal true
+			done()
+

services/document-updater/test/unit/coffee/LockManager/getLockTests.coffee

@@ -0,0 +1,69 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/LockManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe 'LockManager - getting the lock', ->
+	beforeEach ->
+		@LockManager = SandboxedModule.require modulePath, requires:
+			"logger-sharelatex": log:->
+			redis:
+				createClient : () =>
+					auth:->
+		@callback = sinon.stub()
+		@doc_id = "doc-id-123"
+	
+	describe "when the lock is not set", ->
+		beforeEach (done) ->
+			@LockManager.tryLock = sinon.stub().callsArgWith(1, null, true)
+			@LockManager.getLock @doc_id, (args...) =>
+				@callback(args...)
+				done()
+
+		it "should try to get the lock", ->
+			@LockManager.tryLock
+				.calledWith(@doc_id)
+				.should.equal true
+
+		it "should only need to try once", ->
+			@LockManager.tryLock.callCount.should.equal 1
+
+		it "should return the callback", ->
+			@callback.calledWith(null).should.equal true
+
+	describe "when the lock is initially set", ->
+		beforeEach (done) ->
+			startTime = Date.now()
+			@LockManager.LOCK_TEST_INTERVAL = 5
+			@LockManager.tryLock = (doc_id, callback = (error, isFree) ->) ->
+				if Date.now() - startTime < 20
+					callback null, false
+				else
+					callback null, true
+			sinon.spy @LockManager, "tryLock"
+
+			@LockManager.getLock @doc_id, (args...) =>
+				@callback(args...)
+				done()
+
+		it "should call tryLock multiple times until free", ->
+			(@LockManager.tryLock.callCount > 1).should.equal true
+
+		it "should return the callback", ->
+			@callback.calledWith(null).should.equal true
+
+	describe "when the lock times out", ->
+		beforeEach (done) ->
+			time = Date.now()
+			@LockManager.MAX_LOCK_WAIT_TIME = 5
+			@LockManager.tryLock = sinon.stub().callsArgWith(1, null, false)
+			@LockManager.getLock @doc_id, (args...) =>
+				@callback(args...)
+				done()
+
+		it "should return the callback with an error", ->
+			@callback.calledWith(new Error("timeout")).should.equal true
+		
+
+

services/document-updater/test/unit/coffee/LockManager/tryLockTests.coffee

@@ -0,0 +1,37 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/LockManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe 'LockManager - trying the lock', ->
+	beforeEach ->
+		@LockManager = SandboxedModule.require modulePath, requires:
+			"logger-sharelatex": log:->
+			redis:
+				createClient : () =>
+					auth:->
+					set: @set = sinon.stub()
+		@callback = sinon.stub()
+		@doc_id = "doc-id-123"
+	
+	describe "when the lock is not set", ->
+		beforeEach ->
+			@set.callsArgWith(5, null, "OK")
+			@LockManager.tryLock @doc_id, @callback
+
+		it "should set the lock key with an expiry if it is not set", ->
+			@set.calledWith("Blocking:#{@doc_id}", "locked", "EX", 10, "NX")
+				.should.equal true
+
+		it "should return the callback with true", ->
+			@callback.calledWith(null, true).should.equal true
+
+	describe "when the lock is already set", ->
+		beforeEach ->
+			@set.callsArgWith(5, null, null)
+			@LockManager.tryLock @doc_id, @callback
+
+		it "should return the callback with false", ->
+			@callback.calledWith(null, false).should.equal true
+

services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee

@@ -0,0 +1,85 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/PersistenceManager.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors"
+
+describe "PersistenceManager.getDoc", ->
+	beforeEach ->
+		@PersistenceManager = SandboxedModule.require modulePath, requires:
+			"request": @request = sinon.stub()
+			"settings-sharelatex": @Settings = {}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@callback = sinon.stub()
+		@Settings.apis =
+			web:
+				url: @url = "www.example.com"
+				user: @user = "sharelatex"
+				pass: @pass = "password"
+
+	describe "with a successful response from the web api", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines))
+			@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
+
+		it "should call the web api", ->
+			@request
+				.calledWith({
+					url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
+					method: "GET"
+					headers:
+						"accept": "application/json"
+					auth:
+						user: @user
+						pass: @pass
+						sendImmediately: true
+					jar: false
+				})
+				.should.equal true
+
+		it "should call the callback with the doc lines", ->
+			@callback.calledWith(null, @lines).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when request returns an error", ->
+		beforeEach ->
+			@request.callsArgWith(1, @error = new Error("oops"), null, null)
+			@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
+
+		it "should return the error", ->
+			@callback.calledWith(@error).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the request returns 404", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 404}, "")
+			@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
+			
+		it "should return a NotFoundError", ->
+			@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the request returns an error status code", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 500}, "")
+			@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
+			
+		it "should return an error", ->
+			@callback.calledWith(new Error("web api error")).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+		
+		

services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee

@@ -0,0 +1,86 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/PersistenceManager.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors"
+
+describe "PersistenceManager.setDoc", ->
+	beforeEach ->
+		@PersistenceManager = SandboxedModule.require modulePath, requires:
+			"request": @request = sinon.stub()
+			"settings-sharelatex": @Settings = {}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@lines = ["one", "two", "three"]
+		@callback = sinon.stub()
+		@Settings.apis =
+			web:
+				url: @url = "www.example.com"
+				user: @user = "sharelatex"
+				pass: @pass = "password"
+
+	describe "with a successful response from the web api", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines))
+			@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback)
+
+		it "should call the web api", ->
+			@request
+				.calledWith({
+					url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
+					body: JSON.stringify
+						lines: @lines
+					method: "POST"
+					headers:
+						"content-type": "application/json"
+					auth:
+						user: @user
+						pass: @pass
+						sendImmediately: true
+					jar: false
+				})
+				.should.equal true
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when request returns an error", ->
+		beforeEach ->
+			@request.callsArgWith(1, @error = new Error("oops"), null, null)
+			@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback)
+
+		it "should return the error", ->
+			@callback.calledWith(@error).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the request returns 404", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 404}, "")
+			@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback)
+			
+		it "should return a NotFoundError", ->
+			@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when the request returns an error status code", ->
+		beforeEach ->
+			@request.callsArgWith(1, null, {statusCode: 500}, "")
+			@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback)
+			
+		it "should return an error", ->
+			@callback.calledWith(new Error("web api error")).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+

services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee

@@ -0,0 +1,75 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/ProjectManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ProjectManager - flushAndDeleteProject", ->
+	beforeEach ->
+		@ProjectManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./DocumentManager": @DocumentManager = {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@callback = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach (done) ->
+			@doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"]
+			@RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids)
+			@DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArg(2)
+			@ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) =>
+				@callback(error)
+				done()
+
+		it "should get the doc ids in the project", ->
+			@RedisManager.getDocIdsInProject
+				.calledWith(@project_id)
+				.should.equal true
+		
+		it "should delete each doc in the project", ->
+			for doc_id in @doc_ids
+				@DocumentManager.flushAndDeleteDocWithLock
+					.calledWith(@project_id, doc_id)
+					.should.equal true
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when a doc errors", ->
+		beforeEach (done) ->
+			@doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"]
+			@RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids)
+			@DocumentManager.flushAndDeleteDocWithLock = sinon.spy (project_id, doc_id, callback = (error) ->) =>
+				if doc_id == "doc-id-1"
+					callback(@error = new Error("oops, something went wrong"))
+				else
+					callback()
+			@ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) =>
+				@callback(error)
+				done()
+			
+		it "should still flush each doc in the project", ->
+			for doc_id in @doc_ids
+				@DocumentManager.flushAndDeleteDocWithLock
+					.calledWith(@project_id, doc_id)
+					.should.equal true
+
+		it "should record the error", ->
+			@logger.error
+				.calledWith(err: @error, project_id: @project_id, doc_id: "doc-id-1", "error deleting doc")
+				.should.equal true
+
+		it "should call the callback with an error", ->
+			@callback.calledWith(new Error()).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+

services/document-updater/test/unit/coffee/ProjectManager/flushProjectTests.coffee

@@ -0,0 +1,75 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/ProjectManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ProjectManager - flushProject", ->
+	beforeEach ->
+		@ProjectManager = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./DocumentManager": @DocumentManager = {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@callback = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach (done) ->
+			@doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"]
+			@RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids)
+			@DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArg(2)
+			@ProjectManager.flushProjectWithLocks @project_id, (error) =>
+				@callback(error)
+				done()
+
+		it "should get the doc ids in the project", ->
+			@RedisManager.getDocIdsInProject
+				.calledWith(@project_id)
+				.should.equal true
+		
+		it "should flush each doc in the project", ->
+			for doc_id in @doc_ids
+				@DocumentManager.flushDocIfLoadedWithLock
+					.calledWith(@project_id, doc_id)
+					.should.equal true
+
+		it "should call the callback without error", ->
+			@callback.calledWith(null).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "when a doc errors", ->
+		beforeEach (done) ->
+			@doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"]
+			@RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids)
+			@DocumentManager.flushDocIfLoadedWithLock = sinon.spy (project_id, doc_id, callback = (error) ->) =>
+				if doc_id == "doc-id-1"
+					callback(@error = new Error("oops, something went wrong"))
+				else
+					callback()
+			@ProjectManager.flushProjectWithLocks @project_id, (error) =>
+				@callback(error)
+				done()
+			
+		it "should still flush each doc in the project", ->
+			for doc_id in @doc_ids
+				@DocumentManager.flushDocIfLoadedWithLock
+					.calledWith(@project_id, doc_id)
+					.should.equal true
+
+		it "should record the error", ->
+			@logger.error
+				.calledWith(err: @error, project_id: @project_id, doc_id: "doc-id-1", "error flushing doc")
+				.should.equal true
+
+		it "should call the callback with an error", ->
+			@callback.calledWith(new Error()).should.equal true
+
+		it "should time the execution", ->
+			@Metrics.Timer::done.called.should.equal true
+
+

services/document-updater/test/unit/coffee/RedisManager/clearDocFromPendingUpdatesSetTests.coffee

@@ -0,0 +1,27 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.clearDocFromPendingUpdatesSet", ->
+	beforeEach ->
+		@project_id = "project-id"
+		@doc_id = "document-id"
+		@callback = sinon.stub()
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis" : createClient: () =>
+				@rclient = auth:->
+
+		@rclient.srem = sinon.stub().callsArg(2)
+		@RedisManager.clearDocFromPendingUpdatesSet(@project_id, @doc_id, @callback)
+
+	it "should get the docs with pending updates", ->
+		@rclient.srem
+			.calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}")
+			.should.equal true
+	
+	it "should return the callback", ->
+		@callback.called.should.equal true
+
+

services/document-updater/test/unit/coffee/RedisManager/getDocsWithPendingUpdatesTests.coffee

@@ -0,0 +1,33 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.getDocsWithPendingUpdates", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis" : createClient: () =>
+				@rclient = auth:->
+
+		@docs = [{
+			doc_id: "doc-id-1"
+			project_id: "project-id-1"
+		}, {
+			doc_id: "doc-id-2"
+			project_id: "project-id-2"
+		}]
+		@doc_keys = @docs.map (doc) -> "#{doc.project_id}:#{doc.doc_id}"
+
+		@rclient.smembers = sinon.stub().callsArgWith(1, null, @doc_keys)
+		@RedisManager.getDocsWithPendingUpdates(@callback)
+
+	it "should get the docs with pending updates", ->
+		@rclient.smembers
+			.calledWith("DocsWithPendingUpdates")
+			.should.equal true
+	
+	it "should return the docs with pending updates", ->
+		@callback.calledWith(null, @docs).should.equal true
+

services/document-updater/test/unit/coffee/RedisManager/getPendingUpdatesForDocTests.coffee

@@ -0,0 +1,56 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.getPendingUpdatesForDoc", ->
+	beforeEach ->
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis": createClient: () =>
+				@rclient =
+					auth: () ->
+					multi: () => @rclient
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@callback = sinon.stub()
+		@rclient.lrange = sinon.stub()
+		@rclient.del = sinon.stub()
+
+	describe "successfully", ->
+		beforeEach ->
+			@updates = [
+				{ op: [{ i: "foo", p: 4 }] }
+				{ op: [{ i: "foo", p: 4 }] }
+			]
+			@jsonUpdates = @updates.map (update) -> JSON.stringify update
+			@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
+			@RedisManager.getPendingUpdatesForDoc @doc_id, @callback
+		
+		it "should get the pending updates", ->
+			@rclient.lrange
+				.calledWith("PendingUpdates:#{@doc_id}", 0, -1)
+				.should.equal true
+
+		it "should delete the pending updates", ->
+			@rclient.del
+				.calledWith("PendingUpdates:#{@doc_id}")
+				.should.equal true
+
+		it "should call the callback with the updates", ->
+			@callback.calledWith(null, @updates).should.equal true
+
+	describe "when the JSON doesn't parse", ->
+		beforeEach ->
+			@jsonUpdates = [
+				JSON.stringify { op: [{ i: "foo", p: 4 }] }
+				"broken json"
+			]
+			@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
+			@RedisManager.getPendingUpdatesForDoc @doc_id, @callback
+
+		it "should return an error to the callback", ->
+			@callback.calledWith(new Error("JSON parse error")).should.equal true
+
+

services/document-updater/test/unit/coffee/RedisManager/getPreviousDocOpsTests.coffee

@@ -0,0 +1,99 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.getPreviousDocOpsTests", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis" : createClient: () =>
+				@rclient =
+					auth: ->
+					multi: => @rclient
+			"logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() }
+		@doc_id = "doc-id-123"
+
+	describe "with a start and an end value", ->
+		beforeEach ->
+			@first_version_in_redis = 30
+			@version = 70
+			@length = @version - @first_version_in_redis
+			@start  = 50
+			@end    = 60
+			@ops = [
+				{ "mock": "op-1" },
+				{ "mock": "op-2" }
+			]
+			@jsonOps = @ops.map (op) -> JSON.stringify op
+			@rclient.llen = sinon.stub().callsArgWith(1, null, @length)
+			@rclient.get = sinon.stub().callsArgWith(1, null, @version.toString())
+			@rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps)
+			@RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback)
+
+		it "should get the length of the existing doc ops", ->
+			@rclient.llen
+				.calledWith("DocOps:#{@doc_id}")
+				.should.equal true
+
+		it "should get the current version of the doc", ->
+			@rclient.get
+				.calledWith("DocVersion:#{@doc_id}")
+				.should.equal true
+
+		it "should get the appropriate docs ops", ->
+			@rclient.lrange
+				.calledWith("DocOps:#{@doc_id}", @start - @first_version_in_redis, @end - @first_version_in_redis)
+				.should.equal true
+
+		it "should return the docs with the doc ops deserialized", ->
+			@callback.calledWith(null, @ops).should.equal true
+
+	describe "with an end value of -1", ->
+		beforeEach ->
+			@first_version_in_redis = 30
+			@version = 70
+			@length = @version - @first_version_in_redis
+			@start  = 50
+			@end    = -1
+			@ops = [
+				{ "mock": "op-1" },
+				{ "mock": "op-2" }
+			]
+			@jsonOps = @ops.map (op) -> JSON.stringify op
+			@rclient.llen = sinon.stub().callsArgWith(1, null, @length)
+			@rclient.get = sinon.stub().callsArgWith(1, null, @version.toString())
+			@rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps)
+			@RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback)
+
+		it "should get the appropriate docs ops to the end of list", ->
+			@rclient.lrange
+				.calledWith("DocOps:#{@doc_id}", @start - @first_version_in_redis, -1)
+				.should.equal true
+
+		it "should return the docs with the doc ops deserialized", ->
+			@callback.calledWith(null, @ops).should.equal true
+
+	describe "when the requested range is not in Redis", ->
+		beforeEach ->
+			@first_version_in_redis = 30
+			@version = 70
+			@length = @version - @first_version_in_redis
+			@start  = 20
+			@end    = -1
+			@ops = [
+				{ "mock": "op-1" },
+				{ "mock": "op-2" }
+			]
+			@jsonOps = @ops.map (op) -> JSON.stringify op
+			@rclient.llen = sinon.stub().callsArgWith(1, null, @length)
+			@rclient.get = sinon.stub().callsArgWith(1, null, @version.toString())
+			@rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps)
+			@RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback)
+			
+		it "should return an error", ->
+			@callback.calledWith(new Error("range is not loaded in redis")).should.equal true
+
+		it "should log out the problem", ->
+			@logger.error.called.should.equal true

services/document-updater/test/unit/coffee/RedisManager/prependDocOpsTests.coffee

@@ -0,0 +1,32 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.clearDocFromPendingUpdatesSet", ->
+	beforeEach ->
+		@doc_id = "document-id"
+		@callback = sinon.stub()
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis" : createClient: () =>
+				@rclient = auth:->
+
+		@rclient.lpush = sinon.stub().callsArg(2)
+		@ops = [
+			{ "mock" : "op-1" },
+			{ "mock" : "op-2" }
+		]
+		@reversedJsonOps = @ops.map((op) -> JSON.stringify op).reverse()
+		@RedisManager.prependDocOps(@doc_id, @ops, @callback)
+
+	it "should push the reversed JSONed ops", ->
+		@rclient.lpush
+			.calledWith("DocOps:#{@doc_id}", @reversedJsonOps)
+			.should.equal true
+	
+	it "should return the callback", ->
+		@callback.called.should.equal true
+
+
+

services/document-updater/test/unit/coffee/RedisManager/pushDocOpTests.coffee

@@ -0,0 +1,37 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/RedisManager"
+SandboxedModule = require('sandboxed-module')
+
+describe "RedisManager.getPreviousDocOpsTests", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@RedisManager = SandboxedModule.require modulePath, requires:
+			"redis" : createClient: () =>
+				@rclient =
+					auth: ->
+					multi: => @rclient
+		@doc_id = "doc-id-123"
+
+	beforeEach ->
+		@version = 70
+		@op =
+			{ "mock": "op-1" }
+		@jsonOp = JSON.stringify @op
+		@rclient.rpush = sinon.stub().callsArgWith(2, null)
+		@rclient.incr = sinon.stub().callsArgWith(1, null, @version.toString())
+		@RedisManager.pushDocOp(@doc_id, @op, @callback)
+
+	it "should push the op into redis", ->
+		@rclient.rpush
+			.calledWith("DocOps:#{@doc_id}", @jsonOp)
+			.should.equal true
+
+	it "should increment the version number", ->
+		@rclient.incr
+			.calledWith("DocVersion:#{@doc_id}")
+			.should.equal true
+	
+	it "should call the callback with the new version", ->
+		@callback.calledWith(null, @version).should.equal true

services/document-updater/test/unit/coffee/RemovingSingleDocFromMemory.coffee

@@ -0,0 +1,73 @@
+require('coffee-script')
+_ = require("underscore")
+assert = require('assert')
+sinon = require('sinon')
+path = require('path')
+modulePath = path.join __dirname, '../../../app/js/RedisManager.js'
+keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js')
+loadModule = require('./module-loader').loadModule
+
+describe 'removing single doc from memory', ()->
+
+	project_id = "12345"
+	doc_id1     = "docid1"
+	doc_id2     = "docid2"
+	doc_id3     = "docid3"
+	
+	redisMemory = undefined
+	redisManager = undefined
+	self = @
+	beforeEach (done)->
+		redisMemory = {}
+
+		mocks =
+			"logger-sharelatex":
+				error:->
+				log:->
+			redis:
+				createClient : ->
+					auth:->
+					multi: ->
+						get:->
+						set:(key, value)->
+							redisMemory[key] = value
+						sadd:(key, value)->
+							if !redisMemory[key]?
+								redisMemory[key] = []
+							redisMemory[key].push value
+						del : (key)->
+							delete redisMemory[key]
+						srem : (key, member)->
+							index = redisMemory[key].indexOf(member)
+							redisMemory[key].splice(index, 1)
+						exec:(callback)->
+							callback(null, [])
+		
+		redisManager = loadModule(modulePath, mocks).module.exports
+		redisManager.putDocInMemory project_id, doc_id1, 0, ["line"], ->
+			redisManager.putDocInMemory project_id, doc_id2, 0, ["ledf"], ->
+				redisManager.putDocInMemory project_id, doc_id3, 0, ["ledf"], ->
+					done()
+
+	it 'should remove doc lines from memory', (done)->
+		keyExists = false
+		redisManager.removeDocFromMemory project_id, doc_id1, ()->
+			assert.equal redisMemory[keys.docLines(doc_id:doc_id1)], undefined
+			keys = _.keys(redisMemory)
+			containsKey(keys, doc_id1)
+			keys.forEach (sets)->
+				containsKey sets, doc_id1
+			_.each redisMemory, (value)->
+				if value.indexOf(doc_id1) != -1
+					assert.equal false, "#{doc_id1} found in value #{value}"
+			done()
+
+
+containsKey = (haystack, key)->
+	if haystack.forEach?
+		haystack.forEach (area)->
+			if area.indexOf(key) != -1
+				assert.equal false, "#{key} found in haystack in #{area}"
+
+
+

services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee

@@ -0,0 +1,54 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/ShareJsDB.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ShareJsDB.getOps", ->
+	beforeEach ->
+		@doc_id = "document-id"
+		@project_id = "project-id"
+		@doc_key = "#{@project_id}:#{@doc_id}"
+		@callback = sinon.stub()
+		@ops = [{p: 20, t: "foo"}]
+		@redis_ops = (JSON.stringify(op) for op in @ops)
+		@ShareJsDB = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./DocOpsManager": @DocOpsManager = {}
+			"./DocumentManager":{}
+
+	describe "with start == end", ->
+		beforeEach ->
+			@start = @end = 42
+			@ShareJsDB.getOps @doc_key, @start, @end, @callback
+
+		it "should return an empty array", ->
+			@callback.calledWith(null, []).should.equal true
+	
+	describe "with a non empty range", ->
+		beforeEach ->
+			@start = 35
+			@end = 42
+			@DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops)
+			@ShareJsDB.getOps @doc_key, @start, @end, @callback
+
+		it "should get the range from redis", ->
+			@DocOpsManager.getPreviousDocOps
+				.calledWith(@project_id, @doc_id, @start, @end-1)
+				.should.equal true
+
+		it "should return the ops", ->
+			@callback.calledWith(null, @ops).should.equal true
+
+	describe "with no specified end", ->
+		beforeEach ->
+			@start = 35
+			@end = null
+			@DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops)
+			@ShareJsDB.getOps @doc_key, @start, @end, @callback
+		
+		it "should get until the end of the list", ->
+			@DocOpsManager.getPreviousDocOps
+				.calledWith(@project_id, @doc_id, @start, -1)
+				.should.equal true
+

services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee

@@ -0,0 +1,85 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../../app/js/ShareJsDB.js"
+SandboxedModule = require('sandboxed-module')
+Errors = require "../../../../app/js/Errors"
+
+describe "ShareJsDB.getSnapshot", ->
+	beforeEach ->
+		@doc_id = "document-id"
+		@project_id = "project-id"
+		@doc_key = "#{@project_id}:#{@doc_id}"
+		@callback = sinon.stub()
+		@ShareJsDB = SandboxedModule.require modulePath, requires:
+			"./DocumentManager": @DocumentManager = {}
+			"./RedisManager": {}
+			"./DocOpsManager": {}
+
+		@version = 42
+
+	describe "with a text document", ->
+		beforeEach ->
+			@lines = ["one", "two", "three"]
+
+		describe "successfully", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version)
+				@ShareJsDB.getSnapshot @doc_key, @callback
+
+			it "should get the doc", ->
+				@DocumentManager.getDoc
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+
+			it "should return the doc lines", ->
+				@callback.args[0][1].snapshot.should.equal @lines.join("\n")
+
+			it "should return the doc version", ->
+				@callback.args[0][1].v.should.equal @version
+
+			it "should return the type as text", ->
+				@callback.args[0][1].type.should.equal "text"
+
+		describe "when the doclines do not exist", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, null, null)
+				@ShareJsDB.getSnapshot @doc_key, @callback
+
+			it "should return the callback with a NotFoundError", ->
+				@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
+
+		describe "when getDoc returns an error", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, @error = new Error("oops"), null, null)
+				@ShareJsDB.getSnapshot @doc_key, @callback
+
+			it "should return the callback with an error", ->
+				@callback.calledWith(@error).should.equal true
+
+	describe "with a JSON document", ->
+		beforeEach ->
+			@lines = [{text: "one"}, {text:"two"}, {text:"three"}]
+
+		describe "successfully", ->
+			beforeEach ->
+				@DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version)
+				@ShareJsDB.getSnapshot @doc_key, @callback
+
+			it "should get the doc", ->
+				@DocumentManager.getDoc
+					.calledWith(@project_id, @doc_id)
+					.should.equal true
+
+			it "should return the doc lines", ->
+				expect(@callback.args[0][1].snapshot).to.deep.equal lines: @lines
+
+			it "should return the doc version", ->
+				@callback.args[0][1].v.should.equal @version
+
+			it "should return the type as text", ->
+				@callback.args[0][1].type.should.equal "json"
+
+
+	

services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee

@@ -0,0 +1,53 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+modulePath = "../../../../app/js/ShareJsDB.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ShareJsDB.writeOps", ->
+	beforeEach ->
+		@project_id = "project-id"
+		@doc_id = "document-id"
+		@doc_key = "#{@project_id}:#{@doc_id}"
+		@callback = sinon.stub()
+		@opData =
+			op: {p: 20, t: "foo"}
+			meta: {source: "bar"}
+		@ShareJsDB = SandboxedModule.require modulePath, requires:
+			"./RedisManager": @RedisManager = {}
+			"./DocOpsManager": @DocOpsManager = {}
+			"./DocumentManager": {}
+
+	describe "writing an op", ->
+		beforeEach ->
+			@version = 42
+			@opData.v = @version
+			@DocOpsManager.pushDocOp = sinon.stub().callsArgWith(3, null, @version+1)
+			@ShareJsDB.writeOp @doc_key, @opData, @callback
+
+		it "should write the op to redis", ->
+			op =
+				op: @opData.op
+				meta: @opData.meta
+			@DocOpsManager.pushDocOp
+				.calledWith(@project_id, @doc_id, op)
+				.should.equal true
+
+		it "should call the callback without an error", ->
+			@callback.called.should.equal true
+			(@callback.args[0][0]?).should.equal false
+
+	describe "writing an op at the wrong version", ->
+		beforeEach ->
+			@version = 42
+			@mismatch = 5
+			@opData.v = @version
+			@DocOpsManager.pushDocOp = sinon.stub().callsArgWith(3, null, @version + @mismatch)
+			@ShareJsDB.writeOp @doc_key, @opData, @callback
+
+		it "should call the callback with an error", ->
+			@callback.calledWith(sinon.match.string).should.equal true
+
+
+	
+