Merge pull request #6 from heukirne/s3-archive

Add S3 archive track changes feature

services/track-changes/app.coffee

@@ -27,6 +27,9 @@ app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpContro
 
 app.post "/doc/:doc_id/pack", HttpController.packDoc
 
+app.get  '/project/:project_id/archive', HttpController.archiveProject
+app.get  '/project/:project_id/unarchive', HttpController.unArchiveProject
+
 packWorker = null # use a single packing worker
 
 app.post "/pack", (req, res, next) ->

services/track-changes/app/coffee/DocArchiveManager.coffee

@@ -0,0 +1,74 @@
+MongoManager = require "./MongoManager"
+MongoAWS = require "./MongoAWS"
+LockManager = require "./LockManager"
+DocstoreHandler = require "./DocstoreHandler"
+logger = require "logger-sharelatex"
+_ = require "underscore"
+async = require "async"
+settings = require("settings-sharelatex")
+
+# increase lock timeouts because archiving can be slow
+LockManager.LOCK_TEST_INTERVAL = 500 # 500ms between each test of the lock
+LockManager.MAX_LOCK_WAIT_TIME = 30000 # 30s maximum time to spend trying to get the lock
+LockManager.LOCK_TTL = 30 # seconds
+
+module.exports = DocArchiveManager =
+
+	archiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
+		DocstoreHandler.getAllDocs project_id, (error, docs) ->
+			if error?
+				return callback(error)
+			else if !docs?
+				return callback new Error("No docs for project #{project_id}")
+			jobs = _.map docs, (doc) ->
+				(cb)-> DocArchiveManager.archiveDocChangesWithLock project_id, doc._id, cb
+			async.series jobs, callback
+
+	archiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) ->
+		job = (releaseLock) ->
+			DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock
+		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
+
+	archiveDocChanges: (project_id, doc_id, callback)->
+		MongoManager.getDocChangesCount doc_id, (error, count) ->
+			return callback(error) if error?
+			if count == 0
+				logger.log {project_id, doc_id}, "document history is empty, not archiving"
+				return callback()
+			else
+				MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
+					return callback(error) if error?
+					MongoAWS.archiveDocHistory project_id, doc_id, (error) ->
+						return callback(error) if error?
+						logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
+						MongoManager.markDocHistoryAsArchived doc_id, update, (error) ->
+							return callback(error) if error?
+							callback()
+
+	unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
+		DocstoreHandler.getAllDocs project_id, (error, docs) ->
+			if error?
+				return callback(error)
+			else if !docs?
+				return callback new Error("No docs for project #{project_id}")
+			jobs = _.map docs, (doc) ->
+				(cb)-> DocArchiveManager.unArchiveDocChangesWithLock project_id, doc._id, cb
+			async.parallelLimit jobs, 4, callback
+
+	unArchiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) ->
+		job = (releaseLock) ->
+			DocArchiveManager.unArchiveDocChanges project_id, doc_id, releaseLock
+		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
+
+	unArchiveDocChanges: (project_id, doc_id, callback)->
+		MongoManager.getArchivedDocChanges doc_id, (error, count) ->
+			return callback(error) if error?
+			if count == 0
+				return callback()
+			else
+				MongoAWS.unArchiveDocHistory project_id, doc_id, (error) ->
+					return callback(error) if error?
+					logger.log doc_id:doc_id, project_id:project_id, "imported document from S3"
+					MongoManager.markDocHistoryAsUnarchived doc_id, (error) ->
+						return callback(error) if error?
+						callback()

services/track-changes/app/coffee/DocstoreHandler.coffee

@@ -0,0 +1,21 @@
+request = require("request").defaults(jar: false)
+logger = require "logger-sharelatex"
+settings = require "settings-sharelatex"
+
+module.exports = DocstoreHandler =
+
+	getAllDocs: (project_id, callback = (error) ->) ->
+		logger.log project_id: project_id, "getting all docs for project in docstore api"
+		url = "#{settings.apis.docstore.url}/project/#{project_id}/doc"
+		request.get {
+			url: url
+			json: true
+		}, (error, res, docs) ->
+			return callback(error) if error?
+			logger.log {error, res, docs: if docs?.length then docs.map (d) -> d._id else []}, "docstore response"
+			if 200 <= res.statusCode < 300
+				callback(null, docs)
+			else
+				error = new Error("docstore api responded with non-success code: #{res.statusCode}")
+				logger.error err: error, project_id: project_id, "error getting all docs from docstore"
+				callback(error)

services/track-changes/app/coffee/HttpController.coffee

@@ -3,6 +3,7 @@ DiffManager = require "./DiffManager"
 PackManager = require "./PackManager"
 RestoreManager = require "./RestoreManager"
 logger = require "logger-sharelatex"
+DocArchiveManager = require "./DocArchiveManager"
 
 module.exports = HttpController =
 	flushDoc: (req, res, next = (error) ->) ->
@@ -66,3 +67,17 @@ module.exports = HttpController =
 		RestoreManager.restoreToBeforeVersion project_id, doc_id, version, user_id, (error) ->
 			return next(error) if error?
 			res.send 204
+
+	archiveProject: (req, res, next = (error) ->) ->
+		project_id = req.params.project_id
+		logger.log project_id: project_id, "archiving all track changes to s3"
+		DocArchiveManager.archiveAllDocsChanges project_id, (error) ->
+			return next(error) if error?
+			res.send 204
+
+	unArchiveProject: (req, res, next = (error) ->) ->
+		project_id = req.params.project_id
+		logger.log project_id: project_id, "unarchiving all track changes from s3"
+		DocArchiveManager.unArchiveAllDocsChanges project_id, (error) ->
+			return next(error) if error?
+			res.send 204

services/track-changes/app/coffee/MongoAWS.coffee

@@ -0,0 +1,105 @@
+settings = require "settings-sharelatex"
+logger = require "logger-sharelatex"
+AWS = require 'aws-sdk'
+S3S = require 's3-streams'
+{db, ObjectId} = require "./mongojs"
+JSONStream = require "JSONStream"
+ReadlineStream = require "readline-stream"
+
+module.exports = MongoAWS =
+
+	MAX_SIZE:  1024*1024 # almost max size
+	MAX_COUNT: 1024 # almost max count
+
+	archiveDocHistory: (project_id, doc_id, _callback = (error) ->) ->
+
+		callback = (args...) ->
+			_callback(args...)
+			_callback = () ->
+
+		query = {
+			doc_id: ObjectId(doc_id)
+			expiresAt: {$exists : false}
+		}
+
+		AWS.config.update {
+			accessKeyId: settings.filestore.s3.key
+			secretAccessKey: settings.filestore.s3.secret
+		}
+		 
+		upload = S3S.WriteStream new AWS.S3(), {
+			"Bucket": settings.filestore.stores.user_files,
+			"Key": project_id+"/changes-"+doc_id
+		}
+
+		db.docHistory.find(query)
+			.on 'error', (err) ->
+				callback(err)
+			.pipe JSONStream.stringify()
+			.pipe upload
+			.on 'error', (err) ->
+				callback(err)
+			.on 'finish', () ->
+				return callback(null)
+
+	unArchiveDocHistory: (project_id, doc_id, _callback = (error) ->) ->
+
+		callback = (args...) ->
+			_callback(args...)
+			_callback = () ->
+
+		AWS.config.update {
+			accessKeyId: settings.filestore.s3.key
+			secretAccessKey: settings.filestore.s3.secret
+		}
+		 
+		download = S3S.ReadStream new AWS.S3(), {
+			"Bucket": settings.filestore.stores.user_files,
+			"Key": project_id+"/changes-"+doc_id
+		}, {
+			encoding: "utf8"
+		}
+
+		lineStream = new ReadlineStream();
+		ops = [] 
+		sz = 0
+
+		download
+			.on 'open', (obj) ->
+				return 1
+			.on 'error', (err) ->
+				callback(err)
+			.pipe lineStream
+			.on 'data', (line) ->
+				if line.length > 2
+					ops.push(JSON.parse(line))
+					sz += line.length 
+				if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE
+					download.pause()
+					MongoAWS.handleBulk ops.slice(0), sz, () ->
+						download.resume()
+					ops.splice(0,ops.length)
+					sz = 0
+			.on 'end', () ->
+				MongoAWS.handleBulk ops, sz, callback
+			.on 'error', (err) ->
+				return callback(err)
+
+	handleBulk: (ops, size, cb) ->
+		bulk = db.docHistory.initializeUnorderedBulkOp();
+
+		for op in ops
+			op._id = ObjectId(op._id)
+			op.doc_id = ObjectId(op.doc_id)
+			op.project_id = ObjectId(op.project_id)
+			bulk.find({_id:op._id}).upsert().updateOne(op)
+			
+		if ops.length > 0
+			bulk.execute (err, result) ->
+				if err?
+					logger.error err:err, "error bulking ReadlineStream"
+				else
+					logger.log count:ops.length, result:result, size: size, "bulked ReadlineStream"
+				cb(err)
+		else
+			cb()

services/track-changes/app/coffee/MongoAWSexternal.coffee

@@ -0,0 +1,123 @@
+settings = require "settings-sharelatex"
+child_process = require "child_process"
+mongoUri = require "mongo-uri";
+logger = require "logger-sharelatex"
+AWS = require 'aws-sdk'
+fs = require 'fs'
+S3S = require 's3-streams'
+
+module.exports = MongoAWSexternal =
+
+	archiveDocHistory: (project_id, doc_id, callback = (error) ->) ->
+		MongoAWS.mongoExportDocHistory doc_id, (error, filepath) ->
+			MongoAWS.s3upStream project_id, doc_id, filepath, callback
+				#delete temp file?
+
+
+	unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) ->
+		MongoAWS.s3downStream project_id, doc_id, (error, filepath) ->
+			if error == null
+				MongoAWS.mongoImportDocHistory filepath, callback
+					#delete temp file?
+			else
+				callback
+
+	mongoExportDocHistory: (doc_id, callback = (error, filepath) ->) ->
+		uriData = mongoUri.parse(settings.mongo.url);
+		filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonUp' 
+
+		args = []
+		args.push '-h' 
+		args.push uriData.hosts[0]
+		args.push '-d'
+		args.push uriData.database
+		args.push '-c' 
+		args.push 'docHistory'
+		args.push '-q'
+		args.push "{doc_id: ObjectId('#{doc_id}') , expiresAt: {$exists : false} }"
+		args.push '-o' 
+		args.push filepath
+
+		proc = child_process.spawn "mongoexport", args
+
+		proc.on "error", callback
+
+		stderr = ""
+		proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
+
+		proc.on "close", (code) ->
+			if code == 0
+				return callback(null,filepath)
+			else
+				return callback(new Error("mongodump failed: #{stderr}"),null)
+
+	mongoImportDocHistory: (filepath, callback = (error) ->) ->
+
+		uriData = mongoUri.parse(settings.mongo.url);
+
+		args = []
+		args.push '-h' 
+		args.push uriData.hosts[0]
+		args.push '-d'
+		args.push uriData.database
+		args.push '-c' 
+		args.push 'docHistory'
+		args.push '--file' 
+		args.push filepath
+
+		proc = child_process.spawn "mongoimport", args
+
+		proc.on "error", callback
+
+		stderr = ""
+		proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
+
+		proc.on "close", (code) ->
+			if code == 0
+				return callback(null,filepath)
+			else
+				return callback(new Error("mongodump failed: #{stderr}"),null)
+
+	s3upStream: (project_id, doc_id, filepath, callback = (error) ->) ->
+
+		AWS.config.update {
+			accessKeyId: settings.filestore.s3.key
+			secretAccessKey: settings.filestore.s3.secret
+		}
+		 
+		upload = S3S.WriteStream new AWS.S3(), {
+			"Bucket": settings.filestore.stores.user_files,
+			"Key": project_id+"/changes-"+doc_id
+		}
+
+		fs.createReadStream(filepath)
+			.on 'open', (obj) ->
+				return 1
+			.pipe(upload)
+			.on 'finish', () ->
+				return callback(null)
+			.on 'error', (err) ->
+				return callback(err)
+
+	s3downStream: (project_id, doc_id, callback = (error, filepath) ->) ->
+
+		filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' 
+
+		AWS.config.update {
+			accessKeyId: settings.filestore.s3.key
+			secretAccessKey: settings.filestore.s3.secret
+		}
+		 
+		download = S3S.ReadStream new AWS.S3(), {
+			"Bucket": settings.filestore.stores.user_files,
+			"Key": project_id+"/changes-"+doc_id
+		}
+
+		download
+			.on 'open', (obj) ->
+				return 1
+			.pipe(fs.createWriteStream(filepath))
+			.on 'finish', () ->
+				return callback(null, filepath)
+			.on 'error', (err) ->
+				return callback(err, null)

services/track-changes/app/coffee/MongoManager.coffee

@@ -1,6 +1,7 @@
 {db, ObjectId} = require "./mongojs"
 PackManager = require "./PackManager"
 async = require "async"
+logger = require "logger-sharelatex"
 
 module.exports = MongoManager =
 	getLastCompressedUpdate: (doc_id, callback = (error, update) ->) ->
@@ -47,6 +48,7 @@ module.exports = MongoManager =
 
 
 	insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) ->
+		inS3 = update.inS3?
 		update = {
 			doc_id: ObjectId(doc_id.toString())
 			project_id: ObjectId(project_id.toString())
@@ -54,6 +56,9 @@ module.exports = MongoManager =
 			meta:   update.meta
 			v:      update.v
 		}
+		if inS3
+			update.inS3 = true
+		
 		if temporary
 			seconds = 1000
 			minutes = 60 * seconds
@@ -126,3 +131,20 @@ module.exports = MongoManager =
 		# For finding documents which need packing
 		db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true }
 		db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true }
+
+	getDocChangesCount: (doc_id, callback)->
+		db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, {}, callback
+
+	getArchivedDocChanges: (doc_id, callback)->
+		db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback
+
+	markDocHistoryAsArchived: (doc_id, update, callback)->
+		db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)->
+			return callback(error) if error?
+			db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)->
+				return callback(error) if error?
+				callback(error)
+
+	markDocHistoryAsUnarchived: (doc_id, callback)->
+		db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)->
+			callback(error)

services/track-changes/app/coffee/UpdatesManager.coffee

@@ -6,6 +6,8 @@ WebApiManager = require "./WebApiManager"
 UpdateTrimmer = require "./UpdateTrimmer"
 logger = require "logger-sharelatex"
 async = require "async"
+DocArchiveManager = require "./DocArchiveManager"
+_ = require "underscore"
 
 module.exports = UpdatesManager =
 	compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) ->
@@ -32,6 +34,9 @@ module.exports = UpdatesManager =
 					return
 
 			compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates
+			if lastCompressedUpdate?.inS3? and not _.some(compressedUpdates, (update) -> update.inS3)
+				compressedUpdates[compressedUpdates.length-1].inS3 = lastCompressedUpdate.inS3
+
 			MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) ->
 				return callback(error) if error?
 				logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates"
@@ -94,7 +99,17 @@ module.exports = UpdatesManager =
 	getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
 		UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
 			return callback(error) if error?
-			MongoManager.getProjectUpdates project_id, options, callback
+			MongoManager.getProjectUpdates project_id, options, (error, updates) ->
+				jobs = []
+				for update in updates
+					if update.inS3?
+						do (update) ->
+							jobs.push (callback) -> DocArchiveManager.unArchiveDocChanges update.project_id, update.doc_id, callback
+				if jobs.length?
+					async.series jobs, (err) ->
+						MongoManager.getProjectUpdates project_id, options, callback
+				else
+					callback(error, updates)
 
 	getProjectUpdatesWithUserInfo: (project_id, options = {}, callback = (error, updates) ->) ->
 		UpdatesManager.getProjectUpdates project_id, options, (error, updates) ->

services/track-changes/config/settings.defaults.coffee

@@ -1,3 +1,6 @@
+Path = require('path')
+TMP_DIR = Path.resolve(Path.join(__dirname, "../../", "tmp"))
+
 module.exports =
 	mongo:
 		url: 'mongodb://127.0.0.1/sharelatex'
@@ -19,3 +22,14 @@ module.exports =
 			host: "localhost"
 			port: 6379
 			pass: ""
+
+	filestore:
+		backend: "s3"
+		stores:
+			user_files: ""
+		s3:
+			key: ""
+			secret: ""
+
+	path:
+		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")

services/track-changes/package.json

@@ -18,7 +18,11 @@
     "request": "~2.33.0",
     "redis-sharelatex": "~0.0.4",
     "redis": "~0.10.1",
-    "underscore": "~1.7.0"
+    "underscore": "~1.7.0",
+    "mongo-uri": "^0.1.2",
+    "s3-streams": "^0.3.0",
+    "JSONStream": "^1.0.4",
+    "readline-stream": "^1.0.1"
   },
   "devDependencies": {
     "chai": "~1.9.0",

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -0,0 +1,113 @@
+sinon = require "sinon"
+chai = require("chai")
+chai.should()
+expect = chai.expect
+mongojs = require "../../../app/js/mongojs"
+db = mongojs.db
+ObjectId = mongojs.ObjectId
+Settings = require "settings-sharelatex"
+request = require "request"
+rclient = require("redis").createClient() # Only works locally for now
+
+TrackChangesClient = require "./helpers/TrackChangesClient"
+MockDocStoreApi = require "./helpers/MockDocStoreApi"
+MockWebApi = require "./helpers/MockWebApi"
+
+describe "Archiving updates", ->
+	before (done) ->
+		@now = Date.now()
+		@to = @now
+		@user_id = ObjectId().toString()
+		@doc_id = ObjectId().toString()
+		@project_id = ObjectId().toString()
+
+		@minutes = 60 * 1000
+		@hours = 60 * @minutes
+
+		MockWebApi.projects[@project_id] =
+			features:
+				versioning: true
+		sinon.spy MockWebApi, "getProjectDetails"
+
+		MockWebApi.users[@user_id] = @user =
+			email: "user@sharelatex.com"
+			first_name: "Leo"
+			last_name: "Lion"
+			id: @user_id
+		sinon.spy MockWebApi, "getUserInfo"
+
+		MockDocStoreApi.docs[@doc_id] = @doc = 
+			_id: @doc_id
+			project_id: @project_id
+		sinon.spy MockDocStoreApi, "getAllDoc"
+
+		@updates = []
+		for i in [0..9]
+			@updates.push {
+				op: [{ i: "a", p: 0 }]
+				meta: { ts: @now - (9 - i) * @hours - 2 * @minutes, user_id: @user_id }
+				v: 2 * i + 1
+			}
+			@updates.push {
+				op: [{ i: "b", p: 0 }]
+				meta: { ts: @now - (9 - i) * @hours, user_id: @user_id }
+				v: 2 * i + 2
+			}
+
+		TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
+			throw error if error?
+			TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
+				throw error if error?
+				done()
+
+	after (done) ->
+		MockWebApi.getUserInfo.restore()
+		db.docHistory.remove {project_id: ObjectId(@project_id)}
+		TrackChangesClient.removeS3Doc @project_id, @doc_id, done
+
+	describe "archiving a doc's updates", ->
+		before (done) ->
+			TrackChangesClient.archiveProject @project_id, (error) ->
+				throw error if error?
+				done()
+
+		it "should remain one doc change", (done) ->
+			db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
+				throw error if error?
+				count.should.equal 1
+				done()
+
+		it "should remained doc marked as inS3", (done) ->
+			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+				throw error if error?
+				doc.inS3.should.equal true
+				done()
+
+		it "should remained doc have last version", (done) ->
+			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+				throw error if error?
+				doc.v.should.equal 20
+				done()
+
+		it "should store twenty doc changes in S3", (done) ->
+			TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) =>
+				doc.length.should.equal 20
+				done()
+
+	describe "unarchiving a doc's updates", ->
+		before (done) ->
+			TrackChangesClient.unarchiveProject @project_id, (error) ->
+				throw error if error?
+				done()
+
+		it "should restore doc changes", (done) ->
+			db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
+				throw error if error?
+				count.should.equal 20
+				done()
+
+		it "should remove doc marked as inS3", (done) ->
+			db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) ->
+				throw error if error?
+				count.should.equal 0
+				done()

services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee

@@ -28,7 +28,7 @@ describe "Getting a diff", ->
 			first_name: "Leo"
 			last_name: "Lion"
 			id: @user_id
-		sinon.spy MockWebApi, "getUser"
+		sinon.spy MockWebApi, "getUserInfo"
 
 		twoMinutes = 2 * 60 * 1000
 
@@ -68,7 +68,7 @@ describe "Getting a diff", ->
 
 	after () ->
 		MockDocUpdaterApi.getDoc.restore()
-		MockWebApi.getUser.restore()
+		MockWebApi.getUserInfo.restore()
 
 	it "should return the diff", ->
 		expect(@diff).to.deep.equal @expected_diff

services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee

@@ -31,7 +31,7 @@ describe "Getting updates", ->
 			first_name: "Leo"
 			last_name: "Lion"
 			id: @user_id
-		sinon.spy MockWebApi, "getUser"
+		sinon.spy MockWebApi, "getUserInfo"
 
 		@updates = []
 		for i in [0..9]
@@ -52,7 +52,7 @@ describe "Getting updates", ->
 			done()
 
 	after: () ->
-		MockWebApi.getUser.restore()
+		MockWebApi.getUserInfo.restore()
 
 	describe "getting updates up to the limit", ->
 		before (done) ->
@@ -62,7 +62,7 @@ describe "Getting updates", ->
 				done()
 
 		it "should fetch the user details from the web api", ->
-			MockWebApi.getUser
+			MockWebApi.getUserInfo
 				.calledWith(@user_id)
 				.should.equal true
 

services/track-changes/test/acceptance/coffee/RestoringVersions.coffee

@@ -20,7 +20,7 @@ describe "Restoring a version", ->
 		@doc_id = ObjectId().toString()
 		@project_id = ObjectId().toString()
 		MockWebApi.projects[@project_id] = features: versioning: true
-
+		
 		minutes = 60 * 1000
 
 		@updates = [{
@@ -49,6 +49,7 @@ describe "Restoring a version", ->
 			first_name: "Leo"
 			last_name: "Lion"
 			id: @user_id
+
 		MockDocUpdaterApi.docs[@doc_id] =
 			lines: @lines
 			version: 7

services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee

@@ -0,0 +1,24 @@
+express = require("express")
+app = express()
+
+module.exports = MockDocUpdaterApi =
+	docs: {}
+
+	getAllDoc: (project_id, callback = (error) ->) ->
+		callback null, @docs
+
+	run: () ->
+		app.get "/project/:project_id/doc", (req, res, next) =>
+			@getAllDoc req.params.project_id, (error, docs) ->
+				if error?
+					res.send 500
+				if !docs?
+					res.send 404
+				else
+					res.send JSON.stringify docs
+
+		app.listen 3016, (error) ->
+			throw error if error?
+
+MockDocUpdaterApi.run()
+

services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee

@@ -6,15 +6,15 @@ module.exports = MockWebApi =
 
 	projects: {}
 
-	getUser: (user_id, callback = (error) ->) ->
+	getUserInfo: (user_id, callback = (error) ->) ->
 		callback null, @users[user_id] or null
 
-	getProject: (project_id, callback = (error, project) ->) ->
+	getProjectDetails: (project_id, callback = (error, project) ->) ->
 		callback null, @projects[project_id]
 
 	run: () ->
 		app.get "/user/:user_id/personal_info", (req, res, next) =>
-			@getUser req.params.user_id, (error, user) ->
+			@getUserInfo req.params.user_id, (error, user) ->
 				if error?
 					res.send 500
 				if !user?
@@ -23,7 +23,7 @@ module.exports = MockWebApi =
 					res.send JSON.stringify user
 
 		app.get "/project/:project_id/details", (req, res, next) =>
-			@getProject req.params.project_id, (error, project) ->
+			@getProjectDetails req.params.project_id, (error, project) ->
 				if error?
 					res.send 500
 				if !project?

services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee

@@ -1,6 +1,7 @@
 request = require "request"
 rclient = require("redis").createClient() # Only works locally for now
 {db, ObjectId} = require "../../../../app/js/mongojs"
+Settings = require "settings-sharelatex"
 
 module.exports = TrackChangesClient =
 	flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) ->
@@ -71,4 +72,37 @@ module.exports = TrackChangesClient =
 				"X-User-Id": user_id
 		}, (error, response, body) =>
 			response.statusCode.should.equal 204
-			callback null
+			callback null
+
+	archiveProject: (project_id, callback = (error) ->) ->
+		request.get {
+			url: "http://localhost:3015/project/#{project_id}/archive"
+		}, (error, response, body) =>
+			response.statusCode.should.equal 204
+			callback(error)
+
+	unarchiveProject: (project_id, callback = (error) ->) ->
+		request.get {
+			url: "http://localhost:3015/project/#{project_id}/unarchive"
+		}, (error, response, body) =>
+			response.statusCode.should.equal 204
+			callback(error)
+
+	buildS3Options: (content, key)->
+		return {
+				aws:
+					key: Settings.filestore.s3.key
+					secret: Settings.filestore.s3.secret
+					bucket: Settings.filestore.stores.user_files
+				timeout: 30 * 1000
+				json: content
+				uri:"https://#{Settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}"
+		}
+
+	getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
+		options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id)
+		request.get options, callback	
+
+	removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
+		options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id)
+		request.del options, callback	

services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee

@@ -0,0 +1,120 @@
+chai = require('chai')
+sinon = require("sinon")
+should = chai.should()
+modulePath = "../../../../app/js/DocArchiveManager.js"
+SandboxedModule = require('sandboxed-module')
+ObjectId = require("mongojs").ObjectId
+
+describe "DocArchiveManager", ->
+	beforeEach ->
+		@DocArchiveManager = SandboxedModule.require modulePath, requires:
+			"./MongoManager" : @MongoManager = sinon.stub()
+			"./MongoAWS" : @MongoAWS = sinon.stub()
+			"./LockManager" : @LockManager = sinon.stub()
+			"./DocstoreHandler" : @DocstoreHandler = sinon.stub()
+			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->}
+			"settings-sharelatex": @settings =
+				filestore:
+					backend: 's3'
+
+		@mongoDocs = [{
+			_id: ObjectId()
+		}, {
+			_id: ObjectId()
+		}, {
+			_id: ObjectId()
+		}]
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@callback = sinon.stub()
+		
+	describe "archiveAllDocsChanges", ->
+		it "should archive all project docs change", (done)->
+			@DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs)
+			@DocArchiveManager.archiveDocChangesWithLock = sinon.stub().callsArgWith(2, null)
+			
+			@DocArchiveManager.archiveAllDocsChanges @project_id, (err)=>
+				@DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true
+				@DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true
+				@DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true
+				should.not.exist err
+				done()
+
+	describe "archiveDocChangesWithLock", ->
+		beforeEach ->
+			@DocArchiveManager.archiveDocChanges = sinon.stub().callsArg(2)
+			@LockManager.runWithLock = sinon.stub().callsArg(2)
+			@DocArchiveManager.archiveDocChangesWithLock @project_id, @doc_id, @callback
+
+		it "should run archiveDocChangesWithLock with the lock", ->
+			@LockManager.runWithLock
+				.calledWith(
+					"HistoryLock:#{@doc_id}"
+				)
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "archiveDocChanges", ->
+		beforeEach ->
+			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
+			@MongoManager.getDocChangesCount = sinon.stub().callsArg(1)
+			@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
+			@MongoAWS.archiveDocHistory = sinon.stub().callsArg(2)
+			@MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2)
+			@DocArchiveManager.archiveDocChanges @project_id, @doc_id, @callback
+
+		it "should run markDocHistoryAsArchived with doc_id and update", ->
+			@MongoManager.markDocHistoryAsArchived
+				.calledWith(
+					@doc_id, @update
+				)
+				.should.equal true
+		it "should call the callback", ->
+			@callback.called.should.equal true
+		
+	describe "unArchiveAllDocsChanges", ->
+		it "should unarchive all project docs change", (done)->
+			@DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs)
+			@DocArchiveManager.unArchiveDocChangesWithLock = sinon.stub().callsArgWith(2, null)
+			
+			@DocArchiveManager.unArchiveAllDocsChanges @project_id, (err)=>
+				@DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true
+				@DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true
+				@DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true
+				should.not.exist err
+				done()
+
+	describe "unArchiveDocChangesWithLock", ->
+		beforeEach ->
+			@DocArchiveManager.unArchiveDocChanges = sinon.stub().callsArg(2)
+			@LockManager.runWithLock = sinon.stub().callsArg(2)
+			@DocArchiveManager.unArchiveDocChangesWithLock @project_id, @doc_id, @callback
+
+		it "should run unArchiveDocChangesWithLock with the lock", ->
+			@LockManager.runWithLock
+				.calledWith(
+					"HistoryLock:#{@doc_id}"
+				)
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "unArchiveDocChanges", ->
+		beforeEach ->
+			@MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1)
+			@MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2)
+			@MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1)
+			@DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback
+
+		it "should run markDocHistoryAsUnarchived with doc_id", ->
+			@MongoManager.markDocHistoryAsUnarchived
+				.calledWith(
+					@doc_id
+				)
+				.should.equal true
+		it "should call the callback", ->
+			@callback.called.should.equal true

services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee

@@ -0,0 +1,55 @@
+chai = require('chai')
+chai.should()
+sinon = require("sinon")
+modulePath = "../../../../app/js/DocstoreHandler.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "DocstoreHandler", ->
+	beforeEach ->
+		@requestDefaults = sinon.stub().returns(@request = sinon.stub())
+		@DocstoreHandler = SandboxedModule.require modulePath, requires:
+			"request" : defaults: @requestDefaults
+			"settings-sharelatex": @settings =
+				apis:
+					docstore:
+						url: "docstore.sharelatex.com"
+			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->}
+
+		@requestDefaults.calledWith(jar: false).should.equal true
+
+		@project_id = "project-id-123"
+		@doc_id = "doc-id-123"
+		@callback = sinon.stub()
+		
+	describe "getAllDocs", ->
+		describe "with a successful response code", ->
+			beforeEach ->
+				@request.get = sinon.stub().callsArgWith(1, null, statusCode: 204, @docs = [{ _id: "mock-doc-id" }])
+				@DocstoreHandler.getAllDocs @project_id, @callback
+
+			it "should get all the project docs in the docstore api", ->
+				@request.get
+					.calledWith({
+						url: "#{@settings.apis.docstore.url}/project/#{@project_id}/doc"
+						json: true
+					})
+					.should.equal true
+
+			it "should call the callback with the docs", ->
+				@callback.calledWith(null, @docs).should.equal true
+
+		describe "with a failed response code", ->
+			beforeEach ->
+				@request.get = sinon.stub().callsArgWith(1, null, statusCode: 500, "")
+				@DocstoreHandler.getAllDocs @project_id, @callback
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error("docstore api responded with non-success code: 500")).should.equal true
+
+			it "should log the error", ->
+				@logger.error
+					.calledWith({
+						err: new Error("docstore api responded with a non-success code: 500")
+						project_id: @project_id
+					}, "error getting all docs from docstore")
+					.should.equal true

services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee

@@ -0,0 +1,113 @@
+chai = require('chai')
+chai.should()
+sinon = require("sinon")
+modulePath = "../../../../app/js/MongoAWS.js"
+SandboxedModule = require('sandboxed-module')
+{ObjectId} = require("mongojs")
+
+describe "MongoAWS", ->
+	beforeEach ->
+		@MongoAWS = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @settings =
+				filestore:
+					s3:
+						secret: "s3-secret"
+						key: "s3-key"
+					stores:
+						user_files: "s3-bucket"
+			"child_process": @child_process = {}
+			"mongo-uri": @mongouri = {}
+			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->}
+			"aws-sdk": @awssdk = {}
+			"fs": @fs = {}
+			"s3-streams": @s3streams = {}
+			"./mongojs" : { db: @db = {}, ObjectId: ObjectId }
+			"JSONStream": @JSONStream = {}
+			"readline-stream": @readline = sinon.stub()
+
+		@project_id = ObjectId().toString()
+		@doc_id = ObjectId().toString()
+		@callback = sinon.stub()
+		
+	describe "archiveDocHistory", ->
+
+		beforeEach ->
+			@awssdk.config = { update: sinon.stub() } 
+			@awssdk.S3 = sinon.stub()
+			@s3streams.WriteStream = sinon.stub()
+			@db.docHistory = {}
+			@db.docHistory.on = sinon.stub()
+			@db.docHistory.find = sinon.stub().returns @db.docHistory
+			@db.docHistory.on.returns
+				pipe:->
+					pipe:->
+						on: (type, cb)-> 
+							on: (type, cb)-> 
+								cb()
+			@JSONStream.stringify = sinon.stub()
+
+			@MongoAWS.archiveDocHistory @project_id, @doc_id, @callback
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "unArchiveDocHistory", ->
+
+		beforeEach ->
+			@awssdk.config = { update: sinon.stub() } 
+			@awssdk.S3 = sinon.stub()
+			@s3streams.ReadStream = sinon.stub()
+
+			@s3streams.ReadStream.returns
+				#describe on 'open' behavior
+				on: (type, cb)->
+					#describe on 'error' behavior
+					on: (type, cb)->
+						pipe:->
+							#describe on 'data' behavior
+							on: (type, cb)->
+								cb([])
+								#describe on 'end' behavior
+								on: (type, cb)-> 
+									cb()
+									#describe on 'error' behavior
+									on: sinon.stub()
+
+			@MongoAWS.handleBulk = sinon.stub()
+			@MongoAWS.unArchiveDocHistory @project_id, @doc_id, @callback
+
+		it "should call handleBulk", ->
+			@MongoAWS.handleBulk.called.should.equal true
+
+	describe "handleBulk", ->
+		beforeEach ->
+			@bulkOps = [{
+				_id: ObjectId()
+				doc_id: ObjectId()
+				project_id: ObjectId()
+			}, {
+				_id: ObjectId()
+				doc_id: ObjectId()
+				project_id: ObjectId()
+			}, {
+				_id: ObjectId()
+				doc_id: ObjectId()
+				project_id: ObjectId()
+			}]
+			@bulk =
+				find: sinon.stub().returns
+					upsert: sinon.stub().returns
+						updateOne: sinon.stub()
+				execute: sinon.stub().callsArgWith(0, null, {})
+			@db.docHistory = {}
+			@db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk
+			@MongoAWS.handleBulk @bulkOps, @bulkOps.length, @callback
+
+		it "should call updateOne for each operation", ->
+			@bulk.find.calledWith({_id:@bulkOps[0]._id}).should.equal true
+			@bulk.find.calledWith({_id:@bulkOps[1]._id}).should.equal true
+			@bulk.find.calledWith({_id:@bulkOps[2]._id}).should.equal true
+
+		it "should call the callback", ->
+			@callback.calledWith(null).should.equal true
+

services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee

@@ -13,6 +13,7 @@ describe "HttpController", ->
 			"./DiffManager": @DiffManager = {}
 			"./RestoreManager": @RestoreManager = {}
 			"./PackManager": @PackManager = {}
+			"./DocArchiveManager": @DocArchiveManager = {}
 		@doc_id = "doc-id-123"
 		@project_id = "project-id-123"
 		@next = sinon.stub()
@@ -130,3 +131,39 @@ describe "HttpController", ->
 
 		it "should return a success code", ->
 			@res.send.calledWith(204).should.equal true
+
+	describe "archiveProject", ->
+		beforeEach ->
+			@req =
+				params:
+					project_id: @project_id
+			@res =
+				send: sinon.stub()
+			@DocArchiveManager.archiveAllDocsChanges = sinon.stub().callsArg(1)
+			@HttpController.archiveProject @req, @res, @next
+
+		it "should process archive doc changes", ->
+			@DocArchiveManager.archiveAllDocsChanges
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should return a success code", ->
+			@res.send.calledWith(204).should.equal true
+
+	describe "unArchiveProject", ->
+		beforeEach ->
+			@req =
+				params:
+					project_id: @project_id
+			@res =
+				send: sinon.stub()
+			@DocArchiveManager.unArchiveAllDocsChanges = sinon.stub().callsArg(1)
+			@HttpController.unArchiveProject @req, @res, @next
+
+		it "should process unarchive doc changes", ->
+			@DocArchiveManager.unArchiveAllDocsChanges
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should return a success code", ->
+			@res.send.calledWith(204).should.equal true

services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee

@@ -398,3 +398,89 @@ describe "MongoManager", ->
 
 		it "should call the callback", ->
 			@callback.called.should.equal true
+
+	describe "getDocChangesCount", ->
+		beforeEach ->
+			@db.docHistory =
+				count: sinon.stub().callsArg(2)
+			@MongoManager.getDocChangesCount @doc_id, @callback
+
+		it "should return if there is any doc changes", ->
+			@db.docHistory.count
+				.calledWith({
+					doc_id: ObjectId(@doc_id)
+					inS3 : { $exists : false }
+				}, {
+				})
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "getArchivedDocChanges", ->
+		beforeEach ->
+			@db.docHistory =
+				count: sinon.stub().callsArg(2)
+			@MongoManager.getArchivedDocChanges @doc_id, @callback
+
+		it "should return if there is any archived doc changes", ->
+			@db.docHistory.count
+				.calledWith({
+					doc_id: ObjectId(@doc_id)
+					inS3 : true
+				}, {
+				})
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "markDocHistoryAsArchived", ->
+		beforeEach ->
+			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
+			@db.docHistory =
+				update: sinon.stub().callsArg(2)
+				remove: sinon.stub().callsArg(1)
+			@MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback
+
+		it "should update last doc change with inS3 flag", ->
+			@db.docHistory.update
+				.calledWith({
+					_id: ObjectId(@update._id)
+				},{
+					$set : { inS3 : true }
+				})
+				.should.equal true
+
+		it "should remove any other doc changes before last update", ->
+			@db.docHistory.remove
+				.calledWith({
+					doc_id: ObjectId(@doc_id)
+					inS3 : { $exists : false }
+					v: { $lt : @update.v }
+					expiresAt: {$exists : false}
+				})
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "markDocHistoryAsUnarchived", ->
+		beforeEach ->
+			@db.docHistory =
+				update: sinon.stub().callsArg(3)
+			@MongoManager.markDocHistoryAsUnarchived @doc_id, @callback
+
+		it "should remove any doc changes inS3 flag", ->
+			@db.docHistory.update
+				.calledWith({
+					doc_id: ObjectId(@doc_id)
+				},{
+					$unset : { inS3 : true }
+				},{
+					multi: true
+				})
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true