Create processUncompressedUpdates method

services/track-changes/app/coffee/HistoryManager.coffee

@@ -1,4 +1,5 @@
 MongoManager = require "./MongoManager"
+RedisManager = require "./RedisManager"
 UpdateCompressor = require "./UpdateCompressor"
 logger = require "logger-sharelatex"
 
@@ -26,9 +27,15 @@ module.exports = HistoryManager =
 				logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates"
 				callback()
 
+	REDIS_READ_BATCH_SIZE: 100
 	processUncompressedUpdates: (doc_id, callback = (error) ->) ->
-		# Get lock - here or elsewhere?
-		# Get batch from Redis left hand side (oldest)
-		# pass batch to compressAndSaveRawUpdates
-		# Delete batch from redis
-		# release lock
+		RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
+			return callback(error) if error?
+			HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
+				return callback(error) if error?
+				RedisManager.deleteOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error) ->
+					return callback(error) if error?
+					callback()
+
+	processUncompressUpdatesWithLock: (doc_id, callback = (error) ->) ->
+

services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee

@@ -10,63 +10,33 @@ describe "HistoryManager", ->
 		@HistoryManager = SandboxedModule.require modulePath, requires:
 			"./UpdateCompressor": @UpdateCompressor = {}
 			"./MongoManager" : @MongoManager = {}
+			"./RedisManager" : @RedisManager = {}
 			"logger-sharelatex": { log: sinon.stub() }
 		@doc_id = "doc-id-123"
 		@callback = sinon.stub()
 
-	describe "when there are no raw ops", ->
-		beforeEach ->
-			@MongoManager.popLastCompressedUpdate = sinon.stub()
-			@MongoManager.insertCompressedUpdates = sinon.stub()
-			@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback
-
-		it "should not need to access the database", ->
-			@MongoManager.popLastCompressedUpdate.called.should.equal false
-			@MongoManager.insertCompressedUpdates.called.should.equal false
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
-
-	describe "when there is no compressed history to begin with", ->
-		beforeEach ->
-			@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-			@compressedUpdates = { v: 13, op: "compressed-op-12" }
-
-			@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
-			@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
-			@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
-			@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
-
-		it "should try to pop the last compressed op", ->
-			@MongoManager.popLastCompressedUpdate
-				.calledWith(@doc_id)
-				.should.equal true
-		
-		it "should compress the raw ops", ->
-			@UpdateCompressor.compressRawUpdates
-				.calledWith(null, @rawUpdates)
-				.should.equal true
-		
-		it "should save the compressed ops", ->
-			@MongoManager.insertCompressedUpdates
-				.calledWith(@doc_id, @compressedUpdates)
-				.should.equal true
-
-		it "should call the callback", ->
-			@callback.called.should.equal true
+	describe "compressAndSaveRawUpdates", ->
+		describe "when there are no raw ops", ->
+			beforeEach ->
+				@MongoManager.popLastCompressedUpdate = sinon.stub()
+				@MongoManager.insertCompressedUpdates = sinon.stub()
+				@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback
 
-	describe "when the raw ops need appending to existing history", ->
-		beforeEach ->
-			@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
-			@compressedUpdates = { v: 13, op: "compressed-op-12" }
+			it "should not need to access the database", ->
+				@MongoManager.popLastCompressedUpdate.called.should.equal false
+				@MongoManager.insertCompressedUpdates.called.should.equal false
 
-			@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
-			@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
-			@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
+			it "should call the callback", ->
+				@callback.called.should.equal true
 
-		describe "when the raw ops start where the existing history ends", ->
+		describe "when there is no compressed history to begin with", ->
 			beforeEach ->
 				@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
+				@compressedUpdates = { v: 13, op: "compressed-op-12" }
+
+				@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
+				@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
+				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
 				@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
 
 			it "should try to pop the last compressed op", ->
@@ -74,9 +44,9 @@ describe "HistoryManager", ->
 					.calledWith(@doc_id)
 					.should.equal true
 			
-			it "should compress the last compressed op and the raw ops", ->
+			it "should compress the raw ops", ->
 				@UpdateCompressor.compressRawUpdates
-					.calledWith(@lastCompressedUpdate, @rawUpdates)
+					.calledWith(null, @rawUpdates)
 					.should.equal true
 			
 			it "should save the compressed ops", ->
@@ -87,25 +57,83 @@ describe "HistoryManager", ->
 			it "should call the callback", ->
 				@callback.called.should.equal true
 
-		describe "when some raw ops are passed that have already been compressed", ->
+		describe "when the raw ops need appending to existing history", ->
 			beforeEach ->
-				@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
-
-				@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
+				@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
+				@compressedUpdates = { v: 13, op: "compressed-op-12" }
+
+				@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
+				@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
+				@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
+
+			describe "when the raw ops start where the existing history ends", ->
+				beforeEach ->
+					@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
+					@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
+
+				it "should try to pop the last compressed op", ->
+					@MongoManager.popLastCompressedUpdate
+						.calledWith(@doc_id)
+						.should.equal true
+				
+				it "should compress the last compressed op and the raw ops", ->
+					@UpdateCompressor.compressRawUpdates
+						.calledWith(@lastCompressedUpdate, @rawUpdates)
+						.should.equal true
+				
+				it "should save the compressed ops", ->
+					@MongoManager.insertCompressedUpdates
+						.calledWith(@doc_id, @compressedUpdates)
+						.should.equal true
+
+				it "should call the callback", ->
+					@callback.called.should.equal true
+
+			describe "when some raw ops are passed that have already been compressed", ->
+				beforeEach ->
+					@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
+
+					@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
+
+				it "should only compress the more recent raw ops", ->
+					@UpdateCompressor.compressRawUpdates
+						.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2))
+						.should.equal true
+
+			describe "when the raw ops do not follow from the last compressed op version", ->
+				beforeEach ->
+					@rawUpdates = [{ v: 13, op: "mock-op-13" }]
+					@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
+
+				it "should call the callback with an error", ->
+					@callback
+						.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
+						.should.equal true
+
+	describe "processUncompressedUpdates", ->
+		beforeEach ->
+			@updates = ["mock-update"]
+			@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
+			@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
+			@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
+			@HistoryManager.processUncompressedUpdates @doc_id, @callback
+
+		it "should get the oldest updates", ->
+			@RedisManager.getOldestRawUpdates
+				.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
+				.should.equal true
 
-			it "should only compress the more recent raw ops", ->
-				@UpdateCompressor.compressRawUpdates
-					.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2))
-					.should.equal true
+		it "should compress and save the updates", ->
+			@HistoryManager.compressAndSaveRawUpdates
+				.calledWith(@doc_id, @updates)
+				.should.equal true
 
-		describe "when the raw ops do not follow from the last compressed op version", ->
-			beforeEach ->
-				@rawUpdates = [{ v: 13, op: "mock-op-13" }]
-				@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
+		it "should delete the batch of uncompressed updates that was just processed", ->
+			@RedisManager.deleteOldestRawUpdates
+				.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
+				.should.equal true
 
-			it "should call the callback with an error", ->
-				@callback
-					.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
-					.should.equal true
+		it "should call the callback", ->
+			@callback.called.should.equal true