Merge pull request #11 from sharelatex/archive-all-data-in-history

Archive all data in history

services/track-changes/app/coffee/DocArchiveManager.coffee

@@ -25,39 +25,38 @@ module.exports = DocArchiveManager =
 		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
 
 	archiveDocChanges: (project_id, doc_id, callback)->
+		MongoManager.getArchivedDocStatus doc_id, (error, result) ->
+			return callback(error) if error?
+			if result?.inS3 is true
+				logger.log {project_id, doc_id}, "document history is already archived"
+				return callback()
+			if result?.inS3?
+				logger.log {project_id, doc_id}, "document history archive is already in progress"
+				return callback()
 		MongoManager.getDocChangesCount doc_id, (error, count) ->
 			return callback(error) if error?
 			if count == 0
 				logger.log {project_id, doc_id}, "document history is empty, not archiving"
 				return callback()
-			else if count == 1
-				logger.log {project_id, doc_id}, "document history only has one entry, not archiving"
-				return callback()
-			else
-				MongoManager.getArchivedDocChanges doc_id, (error, count) ->
+			MongoManager.peekLastCompressedUpdate doc_id, (error, update, lastVersion) ->
+				return callback(error) if error?
+				logger.log {doc_id, project_id}, "archiving got last compressed update"
+				MongoManager.markDocHistoryAsArchiveInProgress doc_id, lastVersion, (error) ->
 					return callback(error) if error?
-					if count != 0
-						logger.log {project_id, doc_id}, "document history contains archived entries, not archiving"
-						return callback()
-					MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
-						return callback(error) if error?
-						logger.log {doc_id, project_id}, "archiving got last compressed update"
-						MongoManager.markDocHistoryAsArchiveInProgress doc_id, update, (error) ->
-							return callback(error) if error?
-							logger.log {doc_id, project_id}, "marked doc history as archive in progress"
-							MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
-								if error?
-									logger.log {doc_id, project_id, error}, "error exporting document to S3"
-									MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
-										return callback(err) if err?
-										logger.log {doc_id, project_id}, "cleared archive in progress flag"
-										callback(error)
-								else
-									logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
-									MongoManager.markDocHistoryAsArchived doc_id, update, (error) ->
-										return callback(error) if error?
-										logger.log {doc_id, project_id}, "marked doc history as archived"
-										callback()
+					logger.log {doc_id, project_id}, "marked doc history as archive in progress"
+					MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
+						if error?
+							logger.log {doc_id, project_id, error}, "error exporting document to S3"
+							MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
+								return callback(err) if err?
+								logger.log {doc_id, project_id}, "cleared archive in progress flag"
+								callback(error)
+						else
+							logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
+							MongoManager.markDocHistoryAsArchived doc_id, lastVersion, (error) ->
+								return callback(error) if error?
+								logger.log {doc_id, project_id}, "marked doc history as archived"
+								callback()
 
 	unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
 		DocstoreHandler.getAllDocs project_id, (error, docs) ->
@@ -75,9 +74,9 @@ module.exports = DocArchiveManager =
 		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
 
 	unArchiveDocChanges: (project_id, doc_id, callback)->
-		MongoManager.getArchivedDocChanges doc_id, (error, count) ->
+		MongoManager.getArchivedDocStatus doc_id, (error, result) ->
 			return callback(error) if error?
-			if count == 0
+			if result?.inS3 isnt true
 				logger.log {project_id, doc_id}, "no changes marked as in s3, not unarchiving"
 				return callback()
 			else

services/track-changes/app/coffee/MongoAWS.coffee

@@ -19,7 +19,7 @@ module.exports = MongoAWS =
 
 		query = {
 			doc_id: ObjectId(doc_id)
-			v: {$lt: update.v}
+			v: {$lte: update.v}
 			expiresAt: {$exists : false}
 		}
 

services/track-changes/app/coffee/MongoManager.coffee

@@ -18,7 +18,7 @@ module.exports = MongoManager =
 
 	peekLastCompressedUpdate: (doc_id, callback = (error, update, version) ->) ->
 		# under normal use we pass back the last update as
-		# callback(null,update).
+		# callback(null,update,version).
 		#
 		# when we have an existing last update but want to force a new one
 		# to start, we pass it back as callback(null,null,version), just
@@ -26,17 +26,18 @@ module.exports = MongoManager =
 		MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
 			return callback(error) if error?
 			if update?
-				if update.inS3?
-					# we want to force a new update, but ensure that it is
-					# consistent with the version of the existing one in S3
-					return callback null, null, update.v
-				else if update.broken
+				if update.broken
 					# the update is marked as broken so we will force a new op
 					return callback null, null
+				else if update.pack?
+					return callback null, update, update.pack[0]?.v
 				else
-					return callback null, update
+					return callback null, update, update.v
 			else
-				callback null, null
+				MongoManager.getArchivedDocStatus doc_id, (error, status) ->
+					return callback(error) if error?
+					return callback(null, null, status.lastVersion) if status?.inS3? and status?.lastVersion?
+					callback null, null
 
 	insertCompressedUpdates: (project_id, doc_id, updates, temporary, callback = (error) ->) ->
 		jobs = []
@@ -152,26 +153,27 @@ module.exports = MongoManager =
 		db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true }
 		db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true }
 
-	getDocChangesCount: (doc_id, callback)->
-		db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, callback
+	getArchivedDocStatus: (doc_id, callback)->
+		db.docHistoryStats.findOne {doc_id: ObjectId(doc_id.toString()), inS3: {$exists:true}}, {inS3: true, lastVersion: true}, callback
 
-	getArchivedDocChanges: (doc_id, callback)->
-		db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: { $exists: true }}, callback
+	getDocChangesCount: (doc_id, callback)->
+		db.docHistory.count { doc_id : ObjectId(doc_id.toString())}, callback
 
-	markDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
-		db.docHistory.update { _id: update._id }, { $set : { inS3 : false } }, callback
+	markDocHistoryAsArchiveInProgress: (doc_id, lastVersion, callback) ->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: false, lastVersion: lastVersion}}, {upsert:true}, callback
 
 	clearDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
-		db.docHistory.update { _id: update._id }, { $unset : { inS3 : true } }, callback
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$unset : {inS3: true, lastVersion: true}}, callback
 
-	markDocHistoryAsArchived: (doc_id, update, callback)->
-		db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)->
+	markDocHistoryAsArchived: (doc_id, lastVersion, callback)->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: true}}, {upsert:true}, (error)->
 			return callback(error) if error?
-			db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)->
+			# clear the archived entries from the docHistory now we have finally succeeded
+			db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), v: {$lte : lastVersion}, expiresAt: {$exists : false} }, (error)->
 				return callback(error) if error?
 				callback(error)
 
 	markDocHistoryAsUnarchived: (doc_id, callback)->
 		# note this removes any inS3 field, regardless of its value (true/false/null)
-		db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, { $unset : { inS3: true, lastVersion: true} }, (error)->
 			callback(error)

services/track-changes/app/coffee/UpdatesManager.coffee

@@ -17,17 +17,13 @@ module.exports = UpdatesManager =
 			return callback()
 
 		MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
-			# lastCompressedUpdate is the most recent update in Mongo.
+			# lastCompressedUpdate is the most recent update in Mongo, and
+			# lastVersion is its sharejs version number.
 			#
-			# The peekLastCompressedUpdate method may pass it back as 'null'
-			# to force the start of a new compressed update, even when there
-			# was a previous compressed update in Mongo.  In this case it
-			# passes back the lastVersion from the update to check
-			# consistency.
-
-			# when lastVersion is not provided, default to lastCompressedUpdate.v
-			lastVersion ?= lastCompressedUpdate?.v
-
+			# The peekLastCompressedUpdate method may pass the update back
+			# as 'null' (for example if the previous compressed update has
+			# been archived).  In this case it can still pass back the
+			# lastVersion from the update to allow us to check consistency.
 			return callback(error) if error?
 
 			# Ensure that raw updates start where lastVersion left off

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -71,27 +71,27 @@ describe "Archiving updates", ->
 				throw error if error?
 				done()
 
-		it "should remain one doc change", (done) ->
+		it "should remain zero doc change", (done) ->
 			db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
 				throw error if error?
-				count.should.equal 1
+				count.should.equal 0
 				done()
 
-		it "should remained doc marked as inS3", (done) ->
-			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+		it "should have docHistoryStats marked as inS3", (done) ->
+			db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
 				throw error if error?
 				doc.inS3.should.equal true
 				done()
 
-		it "should remained doc have last version", (done) ->
-			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+		it "should have docHistoryStats with the last version", (done) ->
+			db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
 				throw error if error?
-				doc.v.should.equal 20
+				doc.lastVersion.should.equal 20
 				done()
 
-		it "should store nineteen doc changes in S3", (done) ->
+		it "should store twenty doc changes in S3", (done) ->
 			TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) =>
-				doc.length.should.equal 19
+				doc.length.should.equal 20
 				done()
 
 	describe "unarchiving a doc's updates", ->
@@ -107,7 +107,8 @@ describe "Archiving updates", ->
 				done()
 
 		it "should remove doc marked as inS3", (done) ->
-			db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) ->
+			db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) ->
 				throw error if error?
-				count.should.equal 0
+				doc.should.not.contain.key('inS3')
+				doc.should.not.contain.key('lastVersion')
 				done()

services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee

@@ -61,8 +61,8 @@ describe "DocArchiveManager", ->
 		beforeEach ->
 			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
 			@MongoManager.getDocChangesCount = sinon.stub().callsArg(1)
-			@MongoManager.getArchivedDocChanges = sinon.stub().callsArgWith(1, null, 0)
-			@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
+			@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, 0)
+			@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update, @update.v)
 			@MongoAWS.archiveDocHistory = sinon.stub().callsArg(3)
 			@MongoManager.markDocHistoryAsArchiveInProgress = sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2)
@@ -71,7 +71,7 @@ describe "DocArchiveManager", ->
 		it "should run markDocHistoryAsArchived with doc_id and update", ->
 			@MongoManager.markDocHistoryAsArchived
 				.calledWith(
-					@doc_id, @update
+					@doc_id, @update.v
 				)
 				.should.equal true
 		it "should call the callback", ->
@@ -107,7 +107,7 @@ describe "DocArchiveManager", ->
 
 	describe "unArchiveDocChanges", ->
 		beforeEach ->
-			@MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1)
+			@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, {inS3: true})
 			@MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1)
 			@DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback

services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee

@@ -57,6 +57,8 @@ describe "MongoManager", ->
 	describe "peekLastCompressedUpdate", ->
 		describe "when there is no last update", ->
 			beforeEach ->
+				@db.docHistoryStats = {}
+				@db.docHistoryStats.findOne = sinon.stub().callsArgWith(2, null, null)
 				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
 				@MongoManager.peekLastCompressedUpdate @doc_id, @callback
 
@@ -84,8 +86,10 @@ describe "MongoManager", ->
 
 		describe "when there is a last update in S3", ->
 			beforeEach ->
-				@update = { _id: Object(), v: 12345, inS3: true }
-				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
+				@update = { _id: Object(), v: 12345}
+				@db.docHistoryStats = {}
+				@db.docHistoryStats.findOne = sinon.stub().callsArgWith(2, null, {inS3:true, lastVersion: @update.v})
+				@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null)
 				@MongoManager.peekLastCompressedUpdate @doc_id, @callback
 
 			it "should get the last update", ->
@@ -402,21 +406,20 @@ describe "MongoManager", ->
 			@db.docHistory.count
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
-					inS3 : { $exists : false }
 				})
 				.should.equal true
 
 		it "should call the callback", ->
 			@callback.called.should.equal true
 
-	describe "getArchivedDocChanges", ->
+	describe "getArchivedDocStatus", ->
 		beforeEach ->
-			@db.docHistory =
-				count: sinon.stub().callsArg(1)
-			@MongoManager.getArchivedDocChanges @doc_id, @callback
+			@db.docHistoryStats =
+				findOne: sinon.stub().callsArg(2)
+			@MongoManager.getArchivedDocStatus @doc_id, @callback
 
 		it "should return if there is any archived doc changes", ->
-			@db.docHistory.count
+			@db.docHistoryStats.findOne
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
 					inS3: {$exists: true}
@@ -429,15 +432,16 @@ describe "MongoManager", ->
 	describe "markDocHistoryAsArchived", ->
 		beforeEach ->
 			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
+			@db.docHistoryStats =
+				update: sinon.stub().callsArg(3)
 			@db.docHistory =
-				update: sinon.stub().callsArg(2)
 				remove: sinon.stub().callsArg(1)
-			@MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback
+			@MongoManager.markDocHistoryAsArchived @doc_id, @update.v, @callback
 
-		it "should update last doc change with inS3 flag", ->
-			@db.docHistory.update
+		it "should update doc status with inS3 flag", ->
+			@db.docHistoryStats.update
 				.calledWith({
-					_id: ObjectId(@update._id)
+					doc_id: ObjectId(@doc_id)
 				},{
 					$set : { inS3 : true }
 				})
@@ -447,8 +451,7 @@ describe "MongoManager", ->
 			@db.docHistory.remove
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
-					inS3 : { $exists : false }
-					v: { $lt : @update.v }
+					v: { $lte : @update.v }
 					expiresAt: {$exists : false}
 				})
 				.should.equal true
@@ -458,18 +461,16 @@ describe "MongoManager", ->
 
 	describe "markDocHistoryAsUnarchived", ->
 		beforeEach ->
-			@db.docHistory =
-				update: sinon.stub().callsArg(3)
+			@db.docHistoryStats =
+				update: sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsUnarchived @doc_id, @callback
 
 		it "should remove any doc changes inS3 flag", ->
-			@db.docHistory.update
+			@db.docHistoryStats.update
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
 				},{
-					$unset : { inS3 : true }
-				},{
-					multi: true
+					$unset : { inS3 : true, lastVersion: true }
 				})
 				.should.equal true
 

services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee

@@ -69,7 +69,7 @@ describe "UpdatesManager", ->
 				@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
 				@compressedUpdates = [ { v: 12, op: "compressed-op-11+12" }, { v: 13, op: "compressed-op-12" } ]
 
-				@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
+				@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate, @lastCompressedUpdate.v)
 				@MongoManager.modifyCompressedUpdate = sinon.stub().callsArg(2)
 				@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(4)
 				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)