github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Create processUncompressedUpdates method

Browse source
This commit is contained in:
James Allen 2014-02-25 16:48:42 +00:00
parent 34d3847fe4
commit f33a3bde3e
2 changed files with 105 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
services/track-changes/app/coffee/HistoryManager.coffee
View file

@ -1,4 +1,5 @@
MongoManager = require "./MongoManager" MongoManager = require "./MongoManager"
RedisManager = require "./RedisManager"
UpdateCompressor = require "./UpdateCompressor" UpdateCompressor = require "./UpdateCompressor"
logger = require "logger-sharelatex" logger = require "logger-sharelatex"
@ -26,9 +27,15 @@ module.exports = HistoryManager =
logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" logger.log doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates"
callback() callback()
REDIS_READ_BATCH_SIZE: 100
processUncompressedUpdates: (doc_id, callback = (error) ->) -> processUncompressedUpdates: (doc_id, callback = (error) ->) ->
# Get lock - here or elsewhere? RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
# Get batch from Redis left hand side (oldest) return callback(error) if error?
# pass batch to compressAndSaveRawUpdates HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
# Delete batch from redis return callback(error) if error?
# release lock RedisManager.deleteOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error) ->
return callback(error) if error?
callback()
processUncompressUpdatesWithLock: (doc_id, callback = (error) ->) ->

158
services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
View file

@ -10,63 +10,33 @@ describe "HistoryManager", ->
@HistoryManager = SandboxedModule.require modulePath, requires: @HistoryManager = SandboxedModule.require modulePath, requires:
"./UpdateCompressor": @UpdateCompressor = {} "./UpdateCompressor": @UpdateCompressor = {}
"./MongoManager" : @MongoManager = {} "./MongoManager" : @MongoManager = {}
"./RedisManager" : @RedisManager = {}
"logger-sharelatex": { log: sinon.stub() } "logger-sharelatex": { log: sinon.stub() }
@doc_id = "doc-id-123" @doc_id = "doc-id-123"
@callback = sinon.stub() @callback = sinon.stub()
describe "when there are no raw ops", -> describe "compressAndSaveRawUpdates", ->
beforeEach -> describe "when there are no raw ops", ->
@MongoManager.popLastCompressedUpdate = sinon.stub() beforeEach ->
@MongoManager.insertCompressedUpdates = sinon.stub() @MongoManager.popLastCompressedUpdate = sinon.stub()
@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback @MongoManager.insertCompressedUpdates = sinon.stub()
@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback
it "should not need to access the database", -> it "should not need to access the database", ->
@MongoManager.popLastCompressedUpdate.called.should.equal false @MongoManager.popLastCompressedUpdate.called.should.equal false
@MongoManager.insertCompressedUpdates.called.should.equal false @MongoManager.insertCompressedUpdates.called.should.equal false
it "should call the callback", -> it "should call the callback", ->
@callback.called.should.equal true @callback.called.should.equal true
describe "when there is no compressed history to begin with", -> describe "when there is no compressed history to begin with", ->
beforeEach ->
@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", ->
@MongoManager.popLastCompressedUpdate
.calledWith(@doc_id)
.should.equal true
it "should compress the raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(null, @rawUpdates)
.should.equal true
it "should save the compressed ops", ->
@MongoManager.insertCompressedUpdates
.calledWith(@doc_id, @compressedUpdates)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "when the raw ops need appending to existing history", ->
beforeEach ->
@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
describe "when the raw ops start where the existing history ends", ->
beforeEach -> beforeEach ->
@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", -> it "should try to pop the last compressed op", ->
@ -74,9 +44,9 @@ describe "HistoryManager", ->
.calledWith(@doc_id) .calledWith(@doc_id)
.should.equal true .should.equal true
it "should compress the last compressed op and the raw ops", -> it "should compress the raw ops", ->
@UpdateCompressor.compressRawUpdates @UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates) .calledWith(null, @rawUpdates)
.should.equal true .should.equal true
it "should save the compressed ops", -> it "should save the compressed ops", ->
@ -87,25 +57,83 @@ describe "HistoryManager", ->
it "should call the callback", -> it "should call the callback", ->
@callback.called.should.equal true @callback.called.should.equal true
describe "when some raw ops are passed that have already been compressed", -> describe "when the raw ops need appending to existing history", ->
beforeEach -> beforeEach ->
@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] @lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
@compressedUpdates = { v: 13, op: "compressed-op-12" }
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
it "should only compress the more recent raw ops", -> describe "when the raw ops start where the existing history ends", ->
@UpdateCompressor.compressRawUpdates beforeEach ->
.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2)) @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
.should.equal true @HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
describe "when the raw ops do not follow from the last compressed op version", -> it "should try to pop the last compressed op", ->
beforeEach -> @MongoManager.popLastCompressedUpdate
@rawUpdates = [{ v: 13, op: "mock-op-13" }] .calledWith(@doc_id)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback .should.equal true
it "should compress the last compressed op and the raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates)
.should.equal true
it "should save the compressed ops", ->
@MongoManager.insertCompressedUpdates
.calledWith(@doc_id, @compressedUpdates)
.should.equal true
it "should call the callback with an error", -> it "should call the callback", ->
@callback @callback.called.should.equal true
.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
.should.equal true describe "when some raw ops are passed that have already been compressed", ->
beforeEach ->
@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should only compress the more recent raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2))
.should.equal true
describe "when the raw ops do not follow from the last compressed op version", ->
beforeEach ->
@rawUpdates = [{ v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should call the callback with an error", ->
@callback
.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
.should.equal true
describe "processUncompressedUpdates", ->
beforeEach ->
@updates = ["mock-update"]
@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
@HistoryManager.processUncompressedUpdates @doc_id, @callback
it "should get the oldest updates", ->
@RedisManager.getOldestRawUpdates
.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
.should.equal true
it "should compress and save the updates", ->
@HistoryManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates)
.should.equal true
it "should delete the batch of uncompressed updates that was just processed", ->
@RedisManager.deleteOldestRawUpdates
.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true