Add in /updates end point to get updates

This commit is contained in:
James Allen 2014-03-05 15:59:40 +00:00
parent a46963a349
commit 3660253fd4
11 changed files with 208 additions and 55 deletions

View file

@ -14,6 +14,8 @@ app.post "/doc/:doc_id/flush", HttpController.flushUpdatesWithLock
app.get "/project/:project_id/doc/:doc_id/diff", HttpController.getDiff
app.get "/project/:project_id/doc/:doc_id/updates", HttpController.getUpdates
app.get "/status", (req, res, next) ->
res.send "track-changes is alive"

View file

@ -1,18 +1,15 @@
HistoryManager = require "./HistoryManager"
UpdatesManager = require "./UpdatesManager"
DocumentUpdaterManager = require "./DocumentUpdaterManager"
MongoManager = require "./MongoManager"
DiffGenerator = require "./DiffGenerator"
logger = require "logger-sharelatex"
module.exports = DiffManager =
getLatestDocAndUpdates: (project_id, doc_id, fromDate, toDate, callback = (error, lines, version, updates) ->) ->
HistoryManager.processUncompressedUpdatesWithLock doc_id, (error) ->
UpdatesManager.getUpdates doc_id, from: fromDate, to: toDate, (error, updates) ->
return callback(error) if error?
DocumentUpdaterManager.getDocument project_id, doc_id, (error, lines, version) ->
return callback(error) if error?
MongoManager.getUpdatesBetweenDates doc_id, from: fromDate, to: toDate, (error, updates) ->
return callback(error) if error?
callback(null, lines, version, updates)
callback(null, lines, version, updates)
getDiff: (project_id, doc_id, fromDate, toDate, callback = (error, diff) ->) ->
logger.log project_id: project_id, doc_id: doc_id, from: fromDate, to: toDate, "getting diff"

View file

@ -1,4 +1,4 @@
HistoryManager = require "./HistoryManager"
UpdatesManager = require "./UpdatesManager"
DiffManager = require "./DiffManager"
logger = require "logger-sharelatex"
@ -6,7 +6,7 @@ module.exports = HttpController =
flushUpdatesWithLock: (req, res, next = (error) ->) ->
doc_id = req.params.doc_id
logger.log doc_id: doc_id, "compressing doc history"
HistoryManager.processUncompressedUpdatesWithLock doc_id, (error) ->
UpdatesManager.processUncompressedUpdatesWithLock doc_id, (error) ->
return next(error) if error?
res.send 204
@ -31,5 +31,18 @@ module.exports = HttpController =
getUpdates: (req, res, next = (error) ->) ->
doc_id = req.params.doc_id
project_id = req.params.project_id
if req.query.to?
to = parseInt(req.query.to, 10)
if req.query.limit?
limit = parseInt(req.query.limit, 10)
UpdatesManager.getUpdates doc_id, to: to, limit: limit, (error, updates) ->
return next(error) if error?
formattedUpdates = for update in updates
{
meta: update.meta
}
res.send JSON.stringify updates: formattedUpdates

View file

@ -39,17 +39,22 @@ module.exports = MongoManager =
v: update.v
}, callback
getUpdatesBetweenDates:(doc_id, options = {}, callback = (error, updates) ->) ->
getUpdates:(doc_id, options = {}, callback = (error, updates) ->) ->
query =
doc_id: ObjectId(doc_id.toString())
if options.from?
query["meta.end_ts"] = { $gte: options.from }
if options.to?
query["meta.start_ts"] = { $lte: options.to }
db.docHistory
cursor = db.docHistory
.find( query )
.sort( "meta.end_ts": -1 )
.toArray callback
if options.limit?
cursor.limit(options.limit)
cursor.toArray callback
ensureIndices: (callback = (error) ->) ->
db.docHistory.ensureIndex { doc_id: 1, "meta.start_ts": 1, "meta.end_ts": 1 }, callback

View file

@ -4,7 +4,7 @@ UpdateCompressor = require "./UpdateCompressor"
LockManager = require "./LockManager"
logger = require "logger-sharelatex"
module.exports = HistoryManager =
module.exports = UpdatesManager =
compressAndSaveRawUpdates: (doc_id, rawUpdates, callback = (error) ->) ->
length = rawUpdates.length
if length == 0
@ -38,20 +38,20 @@ module.exports = HistoryManager =
REDIS_READ_BATCH_SIZE: 100
processUncompressedUpdates: (doc_id, callback = (error) ->) ->
logger.log "processUncompressedUpdates"
RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
RedisManager.getOldestRawUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
return callback(error) if error?
length = rawUpdates.length
logger.log doc_id: doc_id, length: length, "got raw updates from redis"
HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
UpdatesManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
return callback(error) if error?
logger.log doc_id: doc_id, "compressed and saved doc updates"
RedisManager.deleteOldestRawUpdates doc_id, length, (error) ->
return callback(error) if error?
if length == HistoryManager.REDIS_READ_BATCH_SIZE
if length == UpdatesManager.REDIS_READ_BATCH_SIZE
# There might be more updates
logger.log doc_id: doc_id, "continuing processing updates"
setTimeout () ->
HistoryManager.processUncompressedUpdates doc_id, callback
UpdatesManager.processUncompressedUpdates doc_id, callback
, 0
else
logger.log doc_id: doc_id, "all raw updates processed"
@ -61,7 +61,12 @@ module.exports = HistoryManager =
LockManager.runWithLock(
"HistoryLock:#{doc_id}",
(releaseLock) ->
HistoryManager.processUncompressedUpdates doc_id, releaseLock
UpdatesManager.processUncompressedUpdates doc_id, releaseLock
callback
)
getUpdates: (doc_id, options = {}, callback = (error, updates) ->) ->
UpdatesManager.processUncompressedUpdatesWithLock doc_id, (error) ->
return callback(error) if error?
MongoManager.getUpdates doc_id, options, callback

View file

@ -0,0 +1,58 @@
sinon = require "sinon"
chai = require("chai")
chai.should()
expect = chai.expect
mongojs = require "../../../app/js/mongojs"
db = mongojs.db
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
TrackChangesClient = require "./helpers/TrackChangesClient"
describe "Getting updates", ->
before (done) ->
@now = Date.now()
@to = @now
@user_id = ObjectId().toString()
@doc_id = ObjectId().toString()
@project_id = ObjectId().toString()
@minutes = 60 * 1000
@updates = [{
op: [{ i: "one ", p: 0 }]
meta: { ts: @to - 4 * @minutes, user_id: @user_id }
v: 3
}, {
op: [{ i: "two ", p: 4 }]
meta: { ts: @to - 2 * @minutes, user_id: @user_id }
v: 4
}, {
op: [{ i: "three ", p: 8 }]
meta: { ts: @to, user_id: @user_id }
v: 5
}, {
op: [{ i: "four", p: 14 }]
meta: { ts: @to + 2 * @minutes, user_id: @user_id }
v: 6
}]
TrackChangesClient.pushRawUpdates @doc_id, @updates, (error) =>
throw error if error?
TrackChangesClient.getUpdates @project_id, @doc_id, { to: @to, limit: 2 }, (error, body) =>
throw error if error?
@updates = body.updates
done()
it "should return the diff", ->
expect(@updates).to.deep.equal [{
meta:
start_ts: @to
end_ts: @to
user_id: @user_id
}, {
meta:
start_ts: @to - 2 * @minutes
end_ts: @to - 2 * @minutes
user_id: @user_id
}]

View file

@ -19,6 +19,13 @@ module.exports = TrackChangesClient =
getDiff: (project_id, doc_id, from, to, callback = (error, diff) ->) ->
request.get {
url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/diff?from=#{from}&to=#{to}"
}, (error, response, body) =>
response.statusCode.should.equal 200
callback null, JSON.parse(body)
getUpdates: (project_id, doc_id, options, callback = (error, body) ->) ->
request.get {
url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/updates?to=#{options.to}&limit=#{options.limit}"
}, (error, response, body) =>
response.statusCode.should.equal 200
callback null, JSON.parse(body)

View file

@ -9,9 +9,8 @@ describe "DiffManager", ->
beforeEach ->
@DiffManager = SandboxedModule.require modulePath, requires:
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
"./HistoryManager": @HistoryManager = {}
"./UpdatesManager": @UpdatesManager = {}
"./DocumentUpdaterManager": @DocumentUpdaterManager = {}
"./MongoManager": @MongoManager = {}
"./DiffGenerator": @DiffGenerator = {}
@callback = sinon.stub()
@from = new Date()
@ -25,23 +24,17 @@ describe "DiffManager", ->
@version = 42
@updates = [ "mock-update-1", "mock-update-2" ]
@HistoryManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(1)
@DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(2, null, @lines, @version)
@MongoManager.getUpdatesBetweenDates = sinon.stub().callsArgWith(2, null, @updates)
@UpdatesManager.getUpdates = sinon.stub().callsArgWith(2, null, @updates)
@DiffManager.getLatestDocAndUpdates @project_id, @doc_id, @from, @to, @callback
it "should ensure the latest updates have been compressed", ->
@HistoryManager.processUncompressedUpdatesWithLock
.calledWith(@doc_id)
.should.equal true
it "should get the latest version of the doc", ->
@DocumentUpdaterManager.getDocument
.calledWith(@project_id, @doc_id)
.should.equal true
it "should get the requested updates from Mongo", ->
@MongoManager.getUpdatesBetweenDates
it "should get the latest updates", ->
@UpdatesManager.getUpdates
.calledWith(@doc_id, from: @from, to: @to)
.should.equal true

View file

@ -9,7 +9,7 @@ describe "HttpController", ->
beforeEach ->
@HttpController = SandboxedModule.require modulePath, requires:
"logger-sharelatex": { log: sinon.stub() }
"./HistoryManager": @HistoryManager = {}
"./UpdatesManager": @UpdatesManager = {}
"./DiffManager": @DiffManager = {}
@doc_id = "doc-id-123"
@project_id = "project-id-123"
@ -23,11 +23,11 @@ describe "HttpController", ->
doc_id: @doc_id
@res =
send: sinon.stub()
@HistoryManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(1)
@UpdatesManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(1)
@HttpController.flushUpdatesWithLock @req, @res, @next
it "should process the updates", ->
@HistoryManager.processUncompressedUpdatesWithLock
@UpdatesManager.processUncompressedUpdatesWithLock
.calledWith(@doc_id)
.should.equal true
@ -58,3 +58,37 @@ describe "HttpController", ->
it "should return the diff", ->
@res.send.calledWith(JSON.stringify(diff: @diff)).should.equal true
describe "getUpdates", ->
beforeEach ->
@to = Date.now()
@limit = 10
@req =
params:
doc_id: @doc_id
project_id: @project_id
query:
to: @to.toString()
limit: @limit.toString()
@res =
send: sinon.stub()
@rawUpdates = [{
v: @v = 42
op: @op = "mock-op"
meta: @meta = "mock-meta"
doc_id: @doc_id
}]
@UpdatesManager.getUpdates = sinon.stub().callsArgWith(2, null, @rawUpdates)
@HttpController.getUpdates @req, @res, @next
it "should get the updates", ->
@UpdatesManager.getUpdates
.calledWith(@doc_id, to: @to, limit: @limit)
.should.equal true
it "should return the updates", ->
updates = for update in @rawUpdates
{
meta: @meta
}
@res.send.calledWith(JSON.stringify(updates: updates)).should.equal true

View file

@ -132,12 +132,13 @@ describe "MongoManager", ->
it "should call the callback", ->
@callback.called.should.equal true
describe "getUpdatesBetweenDates", ->
describe "getUpdates", ->
beforeEach ->
@updates = ["mock-update"]
@db.docHistory = {}
@db.docHistory.find = sinon.stub().returns @db.docHistory
@db.docHistory.sort = sinon.stub().returns @db.docHistory
@db.docHistory.limit = sinon.stub().returns @db.docHistory
@db.docHistory.toArray = sinon.stub().callsArgWith(0, null, @updates)
@from = new Date(Date.now())
@ -145,7 +146,7 @@ describe "MongoManager", ->
describe "with a toDate", ->
beforeEach ->
@MongoManager.getUpdatesBetweenDates @doc_id, from: @from, to: @to, @callback
@MongoManager.getUpdates @doc_id, from: @from, to: @to, @callback
it "should find the all updates between the to and from date", ->
@db.docHistory.find
@ -161,12 +162,16 @@ describe "MongoManager", ->
.calledWith("meta.end_ts": -1)
.should.equal true
it "should not limit the results", ->
@db.docHistory.limit
.called.should.equal false
it "should call the call back with the updates", ->
@callback.calledWith(null, @updates).should.equal true
describe "without a todo date", ->
beforeEach ->
@MongoManager.getUpdatesBetweenDates @doc_id, from: @from, @callback
@MongoManager.getUpdates @doc_id, from: @from, @callback
it "should find the all updates after the from date", ->
@db.docHistory.find
@ -179,3 +184,13 @@ describe "MongoManager", ->
it "should call the call back with the updates", ->
@callback.calledWith(null, @updates).should.equal true
describe "with a limit", ->
beforeEach ->
@MongoManager.getUpdates @doc_id, from: @from, limit: @limit = 10, @callback
it "should limit the results", ->
@db.docHistory.limit
.calledWith(@limit)
.should.equal true

View file

@ -2,12 +2,12 @@ sinon = require('sinon')
chai = require('chai')
should = chai.should()
expect = chai.expect
modulePath = "../../../../app/js/HistoryManager.js"
modulePath = "../../../../app/js/UpdatesManager.js"
SandboxedModule = require('sandboxed-module')
describe "HistoryManager", ->
describe "UpdatesManager", ->
beforeEach ->
@HistoryManager = SandboxedModule.require modulePath, requires:
@UpdatesManager = SandboxedModule.require modulePath, requires:
"./UpdateCompressor": @UpdateCompressor = {}
"./MongoManager" : @MongoManager = {}
"./RedisManager" : @RedisManager = {}
@ -21,7 +21,7 @@ describe "HistoryManager", ->
beforeEach ->
@MongoManager.popLastCompressedUpdate = sinon.stub()
@MongoManager.insertCompressedUpdates = sinon.stub()
@HistoryManager.compressAndSaveRawUpdates @doc_id, [], @callback
@UpdatesManager.compressAndSaveRawUpdates @doc_id, [], @callback
it "should not need to access the database", ->
@MongoManager.popLastCompressedUpdate.called.should.equal false
@ -38,7 +38,7 @@ describe "HistoryManager", ->
@MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(2)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
@UpdatesManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", ->
@MongoManager.popLastCompressedUpdate
@ -70,7 +70,7 @@ describe "HistoryManager", ->
describe "when the raw ops start where the existing history ends", ->
beforeEach ->
@rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
@UpdatesManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should try to pop the last compressed op", ->
@MongoManager.popLastCompressedUpdate
@ -94,7 +94,7 @@ describe "HistoryManager", ->
beforeEach ->
@rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
@UpdatesManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should only compress the more recent raw ops", ->
@UpdateCompressor.compressRawUpdates
@ -104,7 +104,7 @@ describe "HistoryManager", ->
describe "when the raw ops do not follow from the last compressed op version", ->
beforeEach ->
@rawUpdates = [{ v: 13, op: "mock-op-13" }]
@HistoryManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
@UpdatesManager.compressAndSaveRawUpdates @doc_id, @rawUpdates, @callback
it "should call the callback with an error", ->
@callback
@ -122,17 +122,17 @@ describe "HistoryManager", ->
beforeEach ->
@updates = ["mock-update"]
@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
@HistoryManager.processUncompressedUpdates @doc_id, @callback
@UpdatesManager.processUncompressedUpdates @doc_id, @callback
it "should get the oldest updates", ->
@RedisManager.getOldestRawUpdates
.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
.calledWith(@doc_id, @UpdatesManager.REDIS_READ_BATCH_SIZE)
.should.equal true
it "should compress and save the updates", ->
@HistoryManager.compressAndSaveRawUpdates
@UpdatesManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates)
.should.equal true
@ -146,7 +146,7 @@ describe "HistoryManager", ->
describe "when there are multiple batches to send", ->
beforeEach (done) ->
@HistoryManager.REDIS_READ_BATCH_SIZE = 2
@UpdatesManager.REDIS_READ_BATCH_SIZE = 2
@updates = ["mock-update-0", "mock-update-1", "mock-update-2", "mock-update-3", "mock-update-4"]
@redisArray = @updates.slice()
@RedisManager.getOldestRawUpdates = (doc_id, batchSize, callback = (error, updates) ->) =>
@ -154,9 +154,9 @@ describe "HistoryManager", ->
@redisArray = @redisArray.slice(batchSize)
callback null, updates
sinon.spy @RedisManager, "getOldestRawUpdates"
@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
@HistoryManager.processUncompressedUpdates @doc_id, (args...) =>
@UpdatesManager.processUncompressedUpdates @doc_id, (args...) =>
@callback(args...)
done()
@ -164,13 +164,13 @@ describe "HistoryManager", ->
@RedisManager.getOldestRawUpdates.callCount.should.equal 3
it "should compress and save the updates in batches", ->
@HistoryManager.compressAndSaveRawUpdates
@UpdatesManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates.slice(0,2))
.should.equal true
@HistoryManager.compressAndSaveRawUpdates
@UpdatesManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates.slice(2,4))
.should.equal true
@HistoryManager.compressAndSaveRawUpdates
@UpdatesManager.compressAndSaveRawUpdates
.calledWith(@doc_id, @updates.slice(4,5))
.should.equal true
@ -182,9 +182,9 @@ describe "HistoryManager", ->
describe "processCompressedUpdatesWithLock", ->
beforeEach ->
@HistoryManager.processUncompressedUpdates = sinon.stub().callsArg(2)
@UpdatesManager.processUncompressedUpdates = sinon.stub().callsArg(2)
@LockManager.runWithLock = sinon.stub().callsArg(2)
@HistoryManager.processUncompressedUpdatesWithLock @doc_id, @callback
@UpdatesManager.processUncompressedUpdatesWithLock @doc_id, @callback
it "should run processUncompressedUpdates with the lock", ->
@LockManager.runWithLock
@ -195,3 +195,27 @@ describe "HistoryManager", ->
it "should call the callback", ->
@callback.called.should.equal true
describe "getUpdates", ->
beforeEach ->
@updates = ["mock-updates"]
@options = { to: "mock-to", limit: "mock-limit" }
@MongoManager.getUpdates = sinon.stub().callsArgWith(2, null, @updates)
@UpdatesManager.processUncompressedUpdatesWithLock = sinon.stub().callsArg(1)
@UpdatesManager.getUpdates @doc_id, @options, @callback
it "should process outstanding updates", ->
@UpdatesManager.processUncompressedUpdatesWithLock
.calledWith(@doc_id)
.should.equal true
it "should get the updates from the database", ->
@MongoManager.getUpdates
.calledWith(@doc_id, @options)
.should.equal true
it "should return the updates", ->
@callback
.calledWith(null, @updates)
.should.equal true