From 8ae9bcd60fe52844d876c8b3c93ad70f5c32cc46 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 25 Feb 2014 12:27:42 +0000 Subject: [PATCH] Ensure that updates are compressed in continuous incrementing order --- .../app/coffee/HistoryManager.coffee | 16 +++++ .../HistoryManager/HistoryManagerTests.coffee | 70 +++++++++++++------ 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/services/track-changes/app/coffee/HistoryManager.coffee b/services/track-changes/app/coffee/HistoryManager.coffee index 7ed112f996..3024dcf7f8 100644 --- a/services/track-changes/app/coffee/HistoryManager.coffee +++ b/services/track-changes/app/coffee/HistoryManager.coffee @@ -10,9 +10,25 @@ module.exports = HistoryManager = MongoManager.popLastCompressedUpdate doc_id, (error, lastCompressedUpdate) -> return callback(error) if error? + + # Ensure that raw updates start where lastCompressedUpdate left off + if lastCompressedUpdate? + rawUpdates = rawUpdates.slice(0) + while rawUpdates[0]? and rawUpdates[0].v <= lastCompressedUpdate.v + rawUpdates.shift() + + if rawUpdates[0]? and rawUpdates[0].v != lastCompressedUpdate.v + 1 + return callback new Error("Tried to apply raw op at version #{rawUpdates[0].v} to last compressed update with version #{lastCompressedUpdate.v}") + compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates MongoManager.insertCompressedUpdates doc_id, compressedUpdates, (error) -> return callback(error) if error? logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" callback() + processUncompressedUpdates: (doc_id, callback = (error) ->) -> + # Get lock - here or elsewhere? + # Get batch from Redis left hand side (oldest) + # pass batch to compressAndSaveRawUpdates + # Delete batch from redis + # release lock diff --git a/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee b/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee index 4c1bae0564..a8bd1f9f74 100644 --- a/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee @@ -29,8 +29,8 @@ describe "HistoryManager", -> describe "when there is no compressed history to begin with", -> beforeEach -> - @rawUpdates = ["mock-raw-op-1", "mock-raw-op-2"] - @compressedUpdates = ["mock-compressed-op"] + @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] + @compressedUpdates = { v: 13, op: "compressed-op-12" } @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null) @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2) @@ -57,31 +57,55 @@ describe "HistoryManager", -> describe "when the raw ops need appending to existing history", -> beforeEach -> - @rawUpdates = ["mock-raw-op-1", "mock-raw-op-2"] - @lastCompressedUpdate = "mock-last-compressed-op-0" - @compressedUpdates = ["mock-compressed-op-1"] + @lastCompressedUpdate = { v: 11, op: "compressed-op-11" } + @compressedUpdates = { v: 13, op: "compressed-op-12" } @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate) @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2) @UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates) - @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback - it "should try to pop the last compressed op", -> - @MongoManager.popLastCompressedUpdate - .calledWith(@doc_id) - .should.equal true - - it "should compress the last compressed op and the raw ops", -> - @UpdateCompressor.compressRawUpdates - .calledWith(@lastCompressedUpdate, @rawUpdates) - .should.equal true - - it "should save the compressed ops", -> - @MongoManager.insertCompressedUpdates - .calledWith(@doc_id, @compressedUpdates) - .should.equal true + describe "when the raw ops start where the existing history ends", -> + beforeEach -> + @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] + @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback + + it "should try to pop the last compressed op", -> + @MongoManager.popLastCompressedUpdate + .calledWith(@doc_id) + .should.equal true + + it "should compress the last compressed op and the raw ops", -> + @UpdateCompressor.compressRawUpdates + .calledWith(@lastCompressedUpdate, @rawUpdates) + .should.equal true + + it "should save the compressed ops", -> + @MongoManager.insertCompressedUpdates + .calledWith(@doc_id, @compressedUpdates) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "when some raw ops are passed that have already been compressed", -> + beforeEach -> + @rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] + + @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback + + it "should only compress the more recent raw ops", -> + @UpdateCompressor.compressRawUpdates + .calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2)) + .should.equal true + + describe "when the raw ops do not follow from the last compressed op version", -> + beforeEach -> + @rawUpdates = [{ v: 13, op: "mock-op-13" }] + @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback + + it "should call the callback with an error", -> + @callback + .calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11")) + .should.equal true - it "should call the callback", -> - @callback.called.should.equal true -