github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Create processUncompressedUpdates method

Browse source
This commit is contained in:
James Allen 2014-02-25 16:48:42 +00:00
parent 34d3847fe4
commit f33a3bde3e
2 changed files with 105 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
services/track-changes/app/coffee/HistoryManager.coffee
View file

@ -1,4 +1,5 @@
MongoManager = require "./MongoManager" MongoManager = require "./MongoManager"
RedisManager = require "./RedisManager"
UpdateCompressor = require "./UpdateCompressor" UpdateCompressor = require "./UpdateCompressor"
logger = require "logger-sharelatex" logger = require "logger-sharelatex"
@ -26,9 +27,15 @@ module.exports = HistoryManager =
logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates"
callback() callback()
REDIS_READ_BATCH_SIZE: 100
processUncompressedUpdates: (doc_id, callback = (error) ->) -> processUncompressedUpdates: (doc_id, callback = (error) ->) ->
# Get lock - here or elsewhere? RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
# Get batch from Redis left hand side (oldest) return callback(error) if error?
# pass batch to compressAndSaveRawUpdates HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
# Delete batch from redis return callback(error) if error?
# release lock RedisManager.deleteOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error) ->
return callback(error) if error?
callback()
processUncompressUpdatesWithLock: (doc_id, callback = (error) ->) ->

158
services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
View file

@ -10,63 +10,33 @@ describe "HistoryManager", ->
@HistoryManager = SandboxedModule.require modulePath, requires: @HistoryManager = SandboxedModule.require modulePath, requires:
"./UpdateCompressor": @UpdateCompressor = {} "./UpdateCompressor": @UpdateCompressor = {}
"./MongoManager" : @MongoManager = {} "./MongoManager" : @MongoManager = {}
"./RedisManager" : @RedisManager = {}
"logger-sharelatex": { log: sinon.stub() } "logger-sharelatex": { log: sinon.stub() }
@doc_id = "doc-id-123" @doc_id = "doc-id-123"
@callback = sinon.stub() @callback = sinon.stub()
describe "when there are no raw ops", -> describe "compressAndSaveRawUpdates", ->
beforeEach -> describe "when there are no raw ops", ->
@MongoManager.popLastCompressedUpdate = sinon.stub() beforeEach ->
@MongoManager.insertCompressedUpdates = sinon.stub() @MongoManager.popLastCompressedUpdate = sinon.stub()
@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback @MongoManager.insertCompressedUpdates = sinon.stub()
@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback
it "should not need to access the database", -> it "should not need to access the database", ->
@MongoManager.popLastCompressedUpdate.called.should.equal false @MongoManager.popLastCompressedUpdate.called.should.equal false
@MongoManager.insertCompressedUpdates.called.should.equal false @MongoManager.insertCompressedUpdates.called.should.equal false
it "should call the callback", -> it "should call the callback", ->
@callback.called.should.equal true @callback.called.should.equal true
describe "when there is no compressed history to begin with", -> describe "when there is no compressed history to begin with", ->
beforeEach ->
@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", ->
@MongoManager.popLastCompressedUpdate
.calledWith(@doc_id)
.should.equal true
it "should compress the raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(null, @rawUpdates)
.should.equal true
it "should save the compressed ops", ->
@MongoManager.insertCompressedUpdates
.calledWith(@doc_id, @compressedUpdates)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "when the raw ops need appending to existing history", ->
beforeEach ->
@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
describe "when the raw ops start where the existing history ends", ->
beforeEach -> beforeEach ->
@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", -> it "should try to pop the last compressed op", ->
@ -74,9 +44,9 @@ describe "HistoryManager", ->
.calledWith(@doc_id) .calledWith(@doc_id)
.should.equal true .should.equal true
it "should compress the last compressed op and the raw ops", -> it "should compress the raw ops", ->
@UpdateCompressor.compressRawUpdates @UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates) .calledWith(null, @rawUpdates)
.should.equal true .should.equal true
it "should save the compressed ops", -> it "should save the compressed ops", ->
@ -87,25 +57,83 @@ describe "HistoryManager", ->
it "should call the callback", -> it "should call the callback", ->
@callback.called.should.equal true @callback.called.should.equal true
describe "when some raw ops are passed that have already been compressed", -> describe "when the raw ops need appending to existing history", ->
beforeEach -> beforeEach ->
@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] @lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
it "should only compress the more recent raw ops", -> describe "when the raw ops start where the existing history ends", ->
@UpdateCompressor.compressRawUpdates beforeEach ->
.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2)) @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
.should.equal true @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
describe "when the raw ops do not follow from the last compressed op version", -> it "should try to pop the last compressed op", ->
beforeEach -> @MongoManager.popLastCompressedUpdate
@rawUpdates = [{ v: 13, op: "mock-op-13" }] .calledWith(@doc_id)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback .should.equal true
it "should call the callback with an error", -> it "should compress the last compressed op and the raw ops", ->
@callback @UpdateCompressor.compressRawUpdates
.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11")) .calledWith(@lastCompressedUpdate, @rawUpdates)
.should.equal true .should.equal true
it "should save the compressed ops", ->
@MongoManager.insertCompressedUpdates
.calledWith(@doc_id, @compressedUpdates)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "when some raw ops are passed that have already been compressed", ->
beforeEach ->
@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should only compress the more recent raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2))
.should.equal true
describe "when the raw ops do not follow from the last compressed op version", ->
beforeEach ->
@rawUpdates = [{ v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should call the callback with an error", ->
@callback
.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
.should.equal true
describe "processUncompressedUpdates", ->
beforeEach ->
@updates = ["mock-update"]
@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
@HistoryManager.processUncompressedUpdates @doc_id, @callback
it "should get the oldest updates", ->
@RedisManager.getOldestRawUpdates
.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
.should.equal true
it "should compress and save the updates", ->
@HistoryManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates)
.should.equal true
it "should delete the batch of uncompressed updates that was just processed", ->
@RedisManager.deleteOldestRawUpdates
.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true