Continue to process raw updates until redis is empty

services/track-changes/app/coffee/HistoryManager.coffee

@@ -32,10 +32,17 @@ module.exports = HistoryManager =
 	processUncompressedUpdates: (doc_id, callback = (error) ->) ->
 		RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
 			return callback(error) if error?
+			length = rawUpdates.length
 			HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
 				return callback(error) if error?
 				RedisManager.deleteOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error) ->
 					return callback(error) if error?
+					if length == HistoryManager.REDIS_READ_BATCH_SIZE
+						# There might be more updates
+						setTimeout () ->
+							HistoryManager.processUncompressedUpdates doc_id, callback
+						, 0
+					else
 						callback()
 
 	processUncompressedUpdatesWithLock: (doc_id, callback = (error) ->) ->

services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee

@@ -112,6 +112,7 @@ describe "HistoryManager", ->
 						.should.equal true
 
 	describe "processUncompressedUpdates", ->
+		describe "when there is fewer than one batch to send", ->
 			beforeEach ->
 				@updates = ["mock-update"]
 				@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
@@ -137,6 +138,42 @@ describe "HistoryManager", ->
 			it "should call the callback", ->
 				@callback.called.should.equal true
 
+		describe "when there are multiple batches to send", ->
+			beforeEach (done) ->
+				@HistoryManager.REDIS_READ_BATCH_SIZE = 2
+				@updates = ["mock-update-0", "mock-update-1", "mock-update-2", "mock-update-3", "mock-update-4"]
+				@redisArray = @updates.slice()
+				@RedisManager.getOldestRawUpdates = (doc_id, batchSize, callback = (error, updates) ->) =>
+					updates = @redisArray.slice(0, batchSize)
+					@redisArray = @redisArray.slice(batchSize)
+					callback null, updates
+				sinon.spy @RedisManager, "getOldestRawUpdates"
+				@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
+				@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
+				@HistoryManager.processUncompressedUpdates @doc_id, (args...) =>
+					@callback(args...)
+					done()
+
+			it "should get the oldest updates in three batches ", ->
+				@RedisManager.getOldestRawUpdates.callCount.should.equal 3
+
+			it "should compress and save the updates in batches", ->
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(0,2))
+					.should.equal true
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(2,4))
+					.should.equal true
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(4,5))
+					.should.equal true
+
+			it "should delete the batches of uncompressed updates", ->
+				@RedisManager.deleteOldestRawUpdates.callCount.should.equal 3
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
 	describe "processCompressedUpdatesWithLock", ->
 		beforeEach ->
 			@HistoryManager.processUncompressedUpdates = sinon.stub()