archiving document history now sends all changes to s3

services/track-changes/app/coffee/DocArchiveManager.coffee

@@ -25,39 +25,38 @@ module.exports = DocArchiveManager =
 		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
 
 	archiveDocChanges: (project_id, doc_id, callback)->
+		MongoManager.getArchivedDocStatus doc_id, (error, result) ->
+			return callback(error) if error?
+			if result?.inS3 is true
+				logger.log {project_id, doc_id}, "document history is already archived"
+				return callback()
+			if result?.inS3?
+				logger.log {project_id, doc_id}, "document history archive is already in progress"
+				return callback()
 		MongoManager.getDocChangesCount doc_id, (error, count) ->
 			return callback(error) if error?
 			if count == 0
 				logger.log {project_id, doc_id}, "document history is empty, not archiving"
 				return callback()
-			else if count == 1
-				logger.log {project_id, doc_id}, "document history only has one entry, not archiving"
-				return callback()
-			else
-				MongoManager.getArchivedDocChanges doc_id, (error, count) ->
+			MongoManager.peekLastCompressedUpdate doc_id, (error, update, lastVersion) ->
+				return callback(error) if error?
+				logger.log {doc_id, project_id}, "archiving got last compressed update"
+				MongoManager.markDocHistoryAsArchiveInProgress doc_id, lastVersion, (error) ->
 					return callback(error) if error?
-					if count != 0
-						logger.log {project_id, doc_id}, "document history contains archived entries, not archiving"
-						return callback()
-					MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
-						return callback(error) if error?
-						logger.log {doc_id, project_id}, "archiving got last compressed update"
-						MongoManager.markDocHistoryAsArchiveInProgress doc_id, update, (error) ->
-							return callback(error) if error?
-							logger.log {doc_id, project_id}, "marked doc history as archive in progress"
-							MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
-								if error?
-									logger.log {doc_id, project_id, error}, "error exporting document to S3"
-									MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
-										return callback(err) if err?
-										logger.log {doc_id, project_id}, "cleared archive in progress flag"
-										callback(error)
-								else
-									logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
-									MongoManager.markDocHistoryAsArchived doc_id, update, (error) ->
-										return callback(error) if error?
-										logger.log {doc_id, project_id}, "marked doc history as archived"
-										callback()
+					logger.log {doc_id, project_id}, "marked doc history as archive in progress"
+					MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
+						if error?
+							logger.log {doc_id, project_id, error}, "error exporting document to S3"
+							MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
+								return callback(err) if err?
+								logger.log {doc_id, project_id}, "cleared archive in progress flag"
+								callback(error)
+						else
+							logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
+							MongoManager.markDocHistoryAsArchived doc_id, lastVersion, (error) ->
+								return callback(error) if error?
+								logger.log {doc_id, project_id}, "marked doc history as archived"
+								callback()
 
 	unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
 		DocstoreHandler.getAllDocs project_id, (error, docs) ->
@@ -75,9 +74,9 @@ module.exports = DocArchiveManager =
 		LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
 
 	unArchiveDocChanges: (project_id, doc_id, callback)->
-		MongoManager.getArchivedDocChanges doc_id, (error, count) ->
+		MongoManager.getArchivedDocStatus doc_id, (error, result) ->
 			return callback(error) if error?
-			if count == 0
+			if result?.inS3 isnt true
 				logger.log {project_id, doc_id}, "no changes marked as in s3, not unarchiving"
 				return callback()
 			else

services/track-changes/app/coffee/MongoAWS.coffee

@@ -19,7 +19,7 @@ module.exports = MongoAWS =
 
 		query = {
 			doc_id: ObjectId(doc_id)
-			v: {$lt: update.v}
+			v: {$lte: update.v}
 			expiresAt: {$exists : false}
 		}
 

services/track-changes/app/coffee/MongoManager.coffee

@@ -157,25 +157,23 @@ module.exports = MongoManager =
 		db.docHistoryStats.findOne {doc_id: ObjectId(doc_id.toString()), inS3: {$exists:true}}, {inS3: true, lastVersion: true}, callback
 
 	getDocChangesCount: (doc_id, callback)->
-		db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, callback
+		db.docHistory.count { doc_id : ObjectId(doc_id.toString())}, callback
 
-	getArchivedDocChanges: (doc_id, callback)->
-		db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: { $exists: true }}, callback
-
-	markDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
-		db.docHistory.update { _id: update._id }, { $set : { inS3 : false } }, callback
+	markDocHistoryAsArchiveInProgress: (doc_id, lastVersion, callback) ->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: false, lastVersion: lastVersion}}, {upsert:true}, callback
 
 	clearDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
-		db.docHistory.update { _id: update._id }, { $unset : { inS3 : true } }, callback
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$unset : {inS3: true, lastVersion: true}}, callback
 
-	markDocHistoryAsArchived: (doc_id, update, callback)->
-		db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)->
+	markDocHistoryAsArchived: (doc_id, lastVersion, callback)->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: true}}, {upsert:true}, (error)->
 			return callback(error) if error?
-			db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)->
+			# clear the archived entries from the docHistory now we have finally succeeded
+			db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), v: {$lte : lastVersion}, expiresAt: {$exists : false} }, (error)->
 				return callback(error) if error?
 				callback(error)
 
 	markDocHistoryAsUnarchived: (doc_id, callback)->
 		# note this removes any inS3 field, regardless of its value (true/false/null)
-		db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)->
+		db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, { $unset : { inS3: true, lastVersion: true} }, (error)->
 			callback(error)

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -71,27 +71,27 @@ describe "Archiving updates", ->
 				throw error if error?
 				done()
 
-		it "should remain one doc change", (done) ->
+		it "should remain zero doc change", (done) ->
 			db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
 				throw error if error?
-				count.should.equal 1
+				count.should.equal 0
 				done()
 
-		it "should remained doc marked as inS3", (done) ->
-			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+		it "should have docHistoryStats marked as inS3", (done) ->
+			db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
 				throw error if error?
 				doc.inS3.should.equal true
 				done()
 
-		it "should remained doc have last version", (done) ->
-			db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
+		it "should have docHistoryStats with the last version", (done) ->
+			db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
 				throw error if error?
-				doc.v.should.equal 20
+				doc.lastVersion.should.equal 20
 				done()
 
-		it "should store nineteen doc changes in S3", (done) ->
+		it "should store twenty doc changes in S3", (done) ->
 			TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) =>
-				doc.length.should.equal 19
+				doc.length.should.equal 20
 				done()
 
 	describe "unarchiving a doc's updates", ->
@@ -107,7 +107,8 @@ describe "Archiving updates", ->
 				done()
 
 		it "should remove doc marked as inS3", (done) ->
-			db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) ->
+			db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) ->
 				throw error if error?
-				count.should.equal 0
+				doc.should.not.contain.key('inS3')
+				doc.should.not.contain.key('lastVersion')
 				done()

services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee

@@ -61,8 +61,8 @@ describe "DocArchiveManager", ->
 		beforeEach ->
 			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
 			@MongoManager.getDocChangesCount = sinon.stub().callsArg(1)
-			@MongoManager.getArchivedDocChanges = sinon.stub().callsArgWith(1, null, 0)
-			@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
+			@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, 0)
+			@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update, @update.v)
 			@MongoAWS.archiveDocHistory = sinon.stub().callsArg(3)
 			@MongoManager.markDocHistoryAsArchiveInProgress = sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2)
@@ -71,7 +71,7 @@ describe "DocArchiveManager", ->
 		it "should run markDocHistoryAsArchived with doc_id and update", ->
 			@MongoManager.markDocHistoryAsArchived
 				.calledWith(
-					@doc_id, @update
+					@doc_id, @update.v
 				)
 				.should.equal true
 		it "should call the callback", ->
@@ -107,7 +107,7 @@ describe "DocArchiveManager", ->
 
 	describe "unArchiveDocChanges", ->
 		beforeEach ->
-			@MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1)
+			@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, {inS3: true})
 			@MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1)
 			@DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback

services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee

@@ -406,21 +406,20 @@ describe "MongoManager", ->
 			@db.docHistory.count
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
-					inS3 : { $exists : false }
 				})
 				.should.equal true
 
 		it "should call the callback", ->
 			@callback.called.should.equal true
 
-	describe "getArchivedDocChanges", ->
+	describe "getArchivedDocStatus", ->
 		beforeEach ->
-			@db.docHistory =
-				count: sinon.stub().callsArg(1)
-			@MongoManager.getArchivedDocChanges @doc_id, @callback
+			@db.docHistoryStats =
+				findOne: sinon.stub().callsArg(2)
+			@MongoManager.getArchivedDocStatus @doc_id, @callback
 
 		it "should return if there is any archived doc changes", ->
-			@db.docHistory.count
+			@db.docHistoryStats.findOne
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
 					inS3: {$exists: true}
@@ -433,15 +432,16 @@ describe "MongoManager", ->
 	describe "markDocHistoryAsArchived", ->
 		beforeEach ->
 			@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
+			@db.docHistoryStats =
+				update: sinon.stub().callsArg(3)
 			@db.docHistory =
-				update: sinon.stub().callsArg(2)
 				remove: sinon.stub().callsArg(1)
-			@MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback
+			@MongoManager.markDocHistoryAsArchived @doc_id, @update.v, @callback
 
-		it "should update last doc change with inS3 flag", ->
-			@db.docHistory.update
+		it "should update doc status with inS3 flag", ->
+			@db.docHistoryStats.update
 				.calledWith({
-					_id: ObjectId(@update._id)
+					doc_id: ObjectId(@doc_id)
 				},{
 					$set : { inS3 : true }
 				})
@@ -451,8 +451,7 @@ describe "MongoManager", ->
 			@db.docHistory.remove
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
-					inS3 : { $exists : false }
-					v: { $lt : @update.v }
+					v: { $lte : @update.v }
 					expiresAt: {$exists : false}
 				})
 				.should.equal true
@@ -462,18 +461,16 @@ describe "MongoManager", ->
 
 	describe "markDocHistoryAsUnarchived", ->
 		beforeEach ->
-			@db.docHistory =
-				update: sinon.stub().callsArg(3)
+			@db.docHistoryStats =
+				update: sinon.stub().callsArg(2)
 			@MongoManager.markDocHistoryAsUnarchived @doc_id, @callback
 
 		it "should remove any doc changes inS3 flag", ->
-			@db.docHistory.update
+			@db.docHistoryStats.update
 				.calledWith({
 					doc_id: ObjectId(@doc_id)
 				},{
-					$unset : { inS3 : true }
-				},{
-					multi: true
+					$unset : { inS3 : true, lastVersion: true }
 				})
 				.should.equal true