github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Don't store doc version in Mongo directly, instead use docstore

Browse source
This commit is contained in:
James Allen 2016-11-29 17:06:23 +00:00
parent 77b1d0ea21
commit 2852043a6d
19 changed files with 287 additions and 615 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
services/document-updater/app.coffee
View file

@ -8,7 +8,6 @@ DispatchManager = require('./app/js/DispatchManager')
Keys = require('./app/js/RedisKeyBuilder')
Errors = require "./app/js/Errors"
HttpController = require "./app/js/HttpController"
MongoHealthCheck = require('./app/js/MongoHealthCheck')
redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.web)
@ -59,13 +58,6 @@ app.get '/status', (req, res)->
else
res.send('document updater is alive')
app.get '/health_check/mongo', (req, res, next) ->
MongoHealthCheck.isAlive (error) ->
if error?
res.send 500, error.message
else
res.send 200
redisCheck = require("redis-sharelatex").activeHealthCheckRedis(Settings.redis.web)
app.get "/health_check/redis", (req, res, next)->
if redisCheck.isAlive()

26
services/document-updater/app/coffee/MongoHealthCheck.coffee
View file

@ -1,26 +0,0 @@
Settings = require "settings-sharelatex"
PersistenceManager = require "./PersistenceManager"
module.exports = MongoHealthCheck =
isAlive: (_callback = (error) ->) ->
# We've seen very occasionally the doc-updater losing its connection to Mongo.
# E.g. https://sharelatex.hackpad.com/29th-Aug-2015-0650-0740-fHlw8RL8zuN
# It seems that the mongo callbacks never returned.
# Mongo is only called in the persistence manager, so we do a read-only
# test call, check that it's working, and returns in a reasonable time.
callback = (args...) ->
_callback(args...)
_callback = () ->
doc_id = Settings.smokeTest?.doc_id
if !doc_id?
return callback(new Error("No test doc_id configured"))
PersistenceManager.getDocVersionInMongo doc_id, (error, version) ->
return callback(error) if error?
callback(null)
timeout = Settings.smokeTest?.timeout or 10000
setTimeout () ->
callback(new Error("Mongo did not return in #{timeout}ms"))
, timeout

45
services/document-updater/app/coffee/PersistenceManager.coffee
View file

@ -2,7 +2,6 @@ request = require "request"
Settings = require "settings-sharelatex"
Errors = require "./Errors"
Metrics = require "./Metrics"
{db, ObjectId} = require("./mongojs")
logger = require "logger-sharelatex"
# We have to be quick with HTTP calls because we're holding a lock that
@ -11,21 +10,7 @@ logger = require "logger-sharelatex"
MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds
module.exports = PersistenceManager =
getDoc: (project_id, doc_id, callback = (error, lines, version) ->) ->
PersistenceManager.getDocFromWeb project_id, doc_id, (error, lines) ->
return callback(error) if error?
PersistenceManager.getDocVersionInMongo doc_id, (error, version) ->
return callback(error) if error?
callback null, lines, version
setDoc: (project_id, doc_id, lines, version, callback = (error) ->) ->
PersistenceManager.setDocInWeb project_id, doc_id, lines, (error) ->
return callback(error) if error?
PersistenceManager.setDocVersionInMongo doc_id, version, (error) ->
return callback(error) if error?
callback()
getDocFromWeb: (project_id, doc_id, _callback = (error, lines) ->) ->
getDoc: (project_id, doc_id, _callback = (error, lines, version) ->) ->
timer = new Metrics.Timer("persistenceManager.getDoc")
callback = (args...) ->
timer.done()
@ -50,13 +35,13 @@ module.exports = PersistenceManager =
body = JSON.parse body
catch e
return callback(e)
return callback null, body.lines
return callback null, body.lines, body.version
else if res.statusCode == 404
return callback(new Errors.NotFoundError("doc not not found: #{url}"))
else
return callback(new Error("error accessing web API: #{url} #{res.statusCode}"))
setDocInWeb: (project_id, doc_id, lines, _callback = (error) ->) ->
setDoc: (project_id, doc_id, lines, version, _callback = (error) ->) ->
timer = new Metrics.Timer("persistenceManager.setDoc")
callback = (args...) ->
timer.done()
@ -68,6 +53,7 @@ module.exports = PersistenceManager =
method: "POST"
body: JSON.stringify
lines: lines
version: version
headers:
"content-type": "application/json"
auth:
@ -84,27 +70,4 @@ module.exports = PersistenceManager =
return callback(new Errors.NotFoundError("doc not not found: #{url}"))
else
return callback(new Error("error accessing web API: #{url} #{res.statusCode}"))
getDocVersionInMongo: (doc_id, callback = (error, version) ->) ->
db.docOps.find {
doc_id: ObjectId(doc_id)
}, {
version: 1
}, (error, docs) ->
return callback(error) if error?
if docs.length < 1 or !docs[0].version?
return callback null, 0
else
return callback null, docs[0].version
setDocVersionInMongo: (doc_id, version, callback = (error) ->) ->
db.docOps.update {
doc_id: ObjectId(doc_id)
}, {
$set: version: version
}, {
upsert: true
}, callback

6
services/document-updater/app/coffee/mongojs.coffee
View file

@ -1,6 +0,0 @@
Settings = require "settings-sharelatex"
mongojs = require "mongojs"
db = mongojs.connect(Settings.mongo.url, ["docOps"])
module.exports =
db: db
ObjectId: mongojs.ObjectId

1
services/document-updater/package.json
View file

@ -14,7 +14,6 @@
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.1",
"lynx": "0.0.11",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
"mongojs": "0.9.11",
"redis-sharelatex": "0.0.9",
"request": "2.25.0",
"sandboxed-module": "~0.2.0",

176
services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee
View file

@ -5,7 +5,6 @@ expect = chai.expect
async = require "async"
Settings = require('settings-sharelatex')
rclient = require("redis-sharelatex").createClient(Settings.redis.web)
{db, ObjectId} = require "../../../app/js/mongojs"
MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockWebApi = require "./helpers/MockWebApi"
@ -29,15 +28,10 @@ describe "Applying updates to a doc", ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
sinon.spy MockWebApi, "getDocument"
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
throw error if error?
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
throw error if error?
setTimeout done, 200
setTimeout done, 200
after ->
MockWebApi.getDocument.restore()
@ -66,15 +60,13 @@ describe "Applying updates to a doc", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert doc_id: ObjectId(@doc_id), version: @version, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
throw error if error?
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
throw error if error?
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) ->
throw error if error?
setTimeout done, 200
setTimeout done, 200
after ->
MockWebApi.getDocument.restore()
@ -99,24 +91,22 @@ describe "Applying updates to a doc", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
lines = ["", "", ""]
MockWebApi.insertDoc @project_id, @doc_id, lines: lines
db.docOps.insert doc_id: ObjectId(@doc_id), version: 0, (error) =>
throw error if error?
@updates = [
{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
{ doc_id: @doc_id, v: 5, op: [i: " ", p: 5 ] }
{ doc_id: @doc_id, v: 6, op: [i: "w", p: 6 ] }
{ doc_id: @doc_id, v: 7, op: [i: "o", p: 7 ] }
{ doc_id: @doc_id, v: 8, op: [i: "r", p: 8 ] }
{ doc_id: @doc_id, v: 9, op: [i: "l", p: 9 ] }
{ doc_id: @doc_id, v: 10, op: [i: "d", p: 10] }
]
@my_result = ["hello world", "", ""]
done()
MockWebApi.insertDoc @project_id, @doc_id, {lines: lines, version: 0}
@updates = [
{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
{ doc_id: @doc_id, v: 5, op: [i: " ", p: 5 ] }
{ doc_id: @doc_id, v: 6, op: [i: "w", p: 6 ] }
{ doc_id: @doc_id, v: 7, op: [i: "o", p: 7 ] }
{ doc_id: @doc_id, v: 8, op: [i: "r", p: 8 ] }
{ doc_id: @doc_id, v: 9, op: [i: "l", p: 9 ] }
{ doc_id: @doc_id, v: 10, op: [i: "d", p: 10] }
]
@my_result = ["hello world", "", ""]
done()
it "should be able to continue applying updates when the project has been deleted", (done) ->
actions = []
@ -155,21 +145,17 @@ describe "Applying updates to a doc", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
lines = ["", "", ""]
MockWebApi.insertDoc @project_id, @doc_id, lines: lines
db.docOps.insert doc_id: ObjectId(@doc_id), version: 0, (error) =>
throw error if error?
@updates = [
{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
{ doc_id: @doc_id, v: 0, op: [i: "world", p: 1 ] }
]
@my_result = ["hello", "world", ""]
done()
MockWebApi.insertDoc @project_id, @doc_id, {lines: lines, version: 0}
@updates = [
{ doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] }
{ doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] }
{ doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] }
{ doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] }
{ doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] }
{ doc_id: @doc_id, v: 0, op: [i: "world", p: 1 ] }
]
@my_result = ["hello", "world", ""]
done()
it "should be able to continue applying updates when the project has been deleted", (done) ->
actions = []
@ -190,12 +176,10 @@ describe "Applying updates to a doc", ->
describe "with a broken update", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert doc_id: ObjectId(@doc_id), version: @version, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.sendUpdate @project_id, @doc_id, @undefined, (error) ->
throw error if error?
DocUpdaterClient.sendUpdate @project_id, @doc_id, @undefined, (error) ->
throw error if error?
setTimeout done, 200
setTimeout done, 200
it "should not update the doc", (done) ->
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
@ -214,19 +198,17 @@ describe "Applying updates to a doc", ->
sinon.spy MockTrackChangesApi, "flushDoc"
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert doc_id: ObjectId(@doc_id), version: 0, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: 0}
# Send updates in chunks to causes multiple flushes
actions = []
for i in [0..9]
do (i) =>
actions.push (cb) =>
DocUpdaterClient.sendUpdates @project_id, @doc_id, updates.slice(i*10, (i+1)*10), cb
async.series actions, (error) =>
throw error if error?
# Send updates in chunks to causes multiple flushes
actions = []
for i in [0..9]
do (i) =>
actions.push (cb) =>
DocUpdaterClient.sendUpdates @project_id, @doc_id, updates.slice(i*10, (i+1)*10), cb
async.series actions, (error) =>
throw error if error?
setTimeout done, 2000
setTimeout done, 2000
after ->
MockTrackChangesApi.flushDoc.restore()
@ -257,41 +239,37 @@ describe "Applying updates to a doc", ->
describe "when the sending duplicate ops", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.subscribeToAppliedOps @messageCallback = sinon.stub()
# One user delete 'one', the next turns it into 'once'. The second becomes a NOP.
DocUpdaterClient.sendUpdate @project_id, @doc_id, {
doc: @doc_id
op: [{
i: "one and a half\n"
p: 4
}]
v: @version
meta:
source: "ikHceq3yfAdQYzBo4-xZ"
}, (error) =>
throw error if error?
# One user delete 'one', the next turns it into 'once'. The second becomes a NOP.
DocUpdaterClient.sendUpdate @project_id, @doc_id, {
doc: @doc_id
op: [{
i: "one and a half\n"
p: 4
}]
v: @version
meta:
source: "ikHceq3yfAdQYzBo4-xZ"
}, (error) =>
throw error if error?
setTimeout () =>
DocUpdaterClient.sendUpdate @project_id, @doc_id, {
doc: @doc_id
op: [{
i: "one and a half\n"
p: 4
}]
v: @version
dupIfSource: ["ikHceq3yfAdQYzBo4-xZ"]
meta:
source: "ikHceq3yfAdQYzBo4-xZ"
}, (error) =>
throw error if error?
setTimeout done, 200
, 200
DocUpdaterClient.subscribeToAppliedOps @messageCallback = sinon.stub()
setTimeout () =>
DocUpdaterClient.sendUpdate @project_id, @doc_id, {
doc: @doc_id
op: [{
i: "one and a half\n"
p: 4
}]
v: @version
dupIfSource: ["ikHceq3yfAdQYzBo4-xZ"]
meta:
source: "ikHceq3yfAdQYzBo4-xZ"
}, (error) =>
throw error if error?
setTimeout done, 200
, 200
it "should update the doc", (done) ->
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>

48
services/document-updater/test/acceptance/coffee/DeletingADocumentTests.coffee
View file

@ -1,7 +1,6 @@
sinon = require "sinon"
chai = require("chai")
chai.should()
{db, ObjectId} = require "../../../app/js/mongojs"
MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
MockWebApi = require "./helpers/MockWebApi"
@ -28,47 +27,32 @@ describe "Deleting a document", ->
describe "when the updated doc exists in the doc updater", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
sinon.spy MockWebApi, "getDocument"
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
throw error if error?
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
throw error if error?
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
throw error if error?
setTimeout () =>
DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) =>
@statusCode = res.statusCode
setTimeout done, 200
, 200
setTimeout () =>
DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) =>
@statusCode = res.statusCode
setTimeout done, 200
, 200
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
MockWebApi.getDocument.restore()
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should send the updated document to the web api", ->
MockWebApi.setDocumentLines
.calledWith(@project_id, @doc_id, @result)
it "should send the updated document and version to the web api", ->
MockWebApi.setDocument
.calledWith(@project_id, @doc_id, @result, @version + 1)
.should.equal true
it "should write the version to mongo", (done) ->
db.docOps.find {
doc_id: ObjectId(@doc_id)
}, {
version: 1
}, (error, docs) =>
doc = docs[0]
doc.version.should.equal @version + 1
done()
it "should need to reload the doc if read again", (done) ->
MockWebApi.getDocument.called.should.equal.false
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) =>
@ -86,21 +70,21 @@ describe "Deleting a document", ->
MockWebApi.insertDoc @project_id, @doc_id, {
lines: @lines
}
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) =>
@statusCode = res.statusCode
setTimeout done, 200
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
MockWebApi.getDocument.restore()
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should not need to send the updated document to the web api", ->
MockWebApi.setDocumentLines.called.should.equal false
MockWebApi.setDocument.called.should.equal false
it "should need to reload the doc if read again", (done) ->
MockWebApi.getDocument.called.should.equal.false

6
services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee
View file

@ -46,7 +46,7 @@ describe "Deleting a project", ->
describe "with documents which have been updated", ->
before (done) ->
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
async.series @docs.map((doc) =>
(callback) =>
DocUpdaterClient.preloadDoc @project_id, doc.id, (error) =>
@ -62,14 +62,14 @@ describe "Deleting a project", ->
, 200
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should send each document to the web api", ->
for doc in @docs
MockWebApi.setDocumentLines
MockWebApi.setDocument
.calledWith(@project_id, doc.id, doc.updatedLines)
.should.equal true

6
services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee
View file

@ -40,7 +40,7 @@ describe "Flushing a project", ->
describe "with documents which have been updated", ->
before (done) ->
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
async.series @docs.map((doc) =>
(callback) =>
@ -57,14 +57,14 @@ describe "Flushing a project", ->
, 200
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should send each document to the web api", ->
for doc in @docs
MockWebApi.setDocumentLines
MockWebApi.setDocument
.calledWith(@project_id, doc.id, doc.updatedLines)
.should.equal true

58
services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee
View file

@ -3,7 +3,6 @@ chai = require("chai")
chai.should()
expect = chai.expect
async = require "async"
{db, ObjectId} = require "../../../app/js/mongojs"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
@ -24,53 +23,37 @@ describe "Flushing a doc to Mongo", ->
describe "when the updated doc exists in the doc updater", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.sendUpdates @project_id, @doc_id, [@update], (error) =>
throw error if error?
DocUpdaterClient.sendUpdates @project_id, @doc_id, [@update], (error) =>
throw error if error?
setTimeout () =>
DocUpdaterClient.flushDoc @project_id, @doc_id, done
, 200
setTimeout () =>
DocUpdaterClient.flushDoc @project_id, @doc_id, done
, 200
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
it "should flush the updated doc lines to the web api", ->
MockWebApi.setDocumentLines
.calledWith(@project_id, @doc_id, @result)
it "should flush the updated doc lines and version to the web api", ->
MockWebApi.setDocument
.calledWith(@project_id, @doc_id, @result, @version + 1)
.should.equal true
it "should store the updated doc version into mongo", (done) ->
db.docOps.find {
doc_id: ObjectId(@doc_id)
}, {
version: 1
}, (error, docs) =>
doc = docs[0]
doc.version.should.equal @version + 1
done()
describe "when the doc does not exist in the doc updater", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, {
lines: @lines
}
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
DocUpdaterClient.flushDoc @project_id, @doc_id, done
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
it "should not flush the doc to the web api", ->
MockWebApi.setDocumentLines.called.should.equal false
MockWebApi.setDocument.called.should.equal false
describe "when the web api http request takes a long time", ->
before (done) ->
@ -78,19 +61,14 @@ describe "Flushing a doc to Mongo", ->
@timeout = 10000
MockWebApi.insertDoc @project_id, @doc_id, {
lines: @lines
}
sinon.stub MockWebApi, "setDocumentLines", (project_id, doc_id, lines, callback = (error) ->) ->
setTimeout callback, 30000
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
throw error if error?
DocUpdaterClient.preloadDoc @project_id, @doc_id, done
}
sinon.stub MockWebApi, "setDocument", (project_id, doc_id, lines, version, callback = (error) ->) ->
setTimeout callback, 30000
DocUpdaterClient.preloadDoc @project_id, @doc_id, done
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
it "should return quickly(ish)", (done) ->
start = Date.now()

24
services/document-updater/test/acceptance/coffee/GettingADocumentTests.coffee
View file

@ -2,7 +2,6 @@ sinon = require "sinon"
chai = require("chai")
chai.should()
expect = chai.expect
{db, ObjectId} = require "../../../app/js/mongojs"
MockWebApi = require "./helpers/MockWebApi"
DocUpdaterClient = require "./helpers/DocUpdaterClient"
@ -18,13 +17,9 @@ describe "Getting a document", ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
sinon.spy MockWebApi, "getDocument"
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
throw error if error?
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done()
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done()
after ->
MockWebApi.getDocument.restore()
@ -44,16 +39,11 @@ describe "Getting a document", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
throw error if error?
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
throw error if error?
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done()
sinon.spy MockWebApi, "getDocument"
DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done()
after ->
MockWebApi.getDocument.restore()

45
services/document-updater/test/acceptance/coffee/SettingADocumentTests.coffee
View file

@ -2,7 +2,6 @@ sinon = require "sinon"
chai = require("chai")
chai.should()
expect = require("chai").expect
{db, ObjectId} = require "../../../app/js/mongojs"
Settings = require('settings-sharelatex')
rclient = require("redis-sharelatex").createClient(Settings.redis.web)
@ -27,36 +26,31 @@ describe "Setting a document", ->
@user_id = "user-id-123"
sinon.spy MockTrackChangesApi, "flushDoc"
sinon.spy MockWebApi, "setDocumentLines"
sinon.spy MockWebApi, "setDocument"
after ->
MockWebApi.setDocumentLines.restore()
MockWebApi.setDocument.restore()
MockTrackChangesApi.flushDoc.restore()
describe "when the updated doc exists in the doc updater", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines, version: @version
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
throw error if error?
DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) =>
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
throw error if error?
DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) =>
throw error if error?
setTimeout () =>
DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, @source, @user_id, (error, res, body) =>
@statusCode = res.statusCode
done()
, 200
setTimeout () =>
DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, @source, @user_id, (error, res, body) =>
@statusCode = res.statusCode
done()
, 200
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should send the updated doc lines to the web api", ->
MockWebApi.setDocumentLines
it "should send the updated doc lines and version to the web api", ->
MockWebApi.setDocument
.calledWith(@project_id, @doc_id, @newLines)
.should.equal true
@ -79,21 +73,16 @@ describe "Setting a document", ->
describe "when the updated doc does not exist in the doc updater", ->
before (done) ->
[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
MockWebApi.insertDoc @project_id, @doc_id, lines: @lines
db.docOps.insert {
doc_id: ObjectId(@doc_id)
version: @version
}, (error) =>
throw error if error?
DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, @source, @user_id, (error, res, body) =>
@statusCode = res.statusCode
setTimeout done, 200
MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version}
DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, @source, @user_id, (error, res, body) =>
@statusCode = res.statusCode
setTimeout done, 200
it "should return a 204 status code", ->
@statusCode.should.equal 204
it "should send the updated doc lines to the web api", ->
MockWebApi.setDocumentLines
MockWebApi.setDocument
.calledWith(@project_id, @doc_id, @newLines)
.should.equal true

11
services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee
View file

@ -7,11 +7,14 @@ module.exports = MockWebApi =
clearDocs: () -> @docs = {}
insertDoc: (project_id, doc_id, doc) ->
doc.version ?= 0
doc.lines ?= []
@docs["#{project_id}:#{doc_id}"] = doc
setDocumentLines: (project_id, doc_id, lines, callback = (error) ->) ->
@docs["#{project_id}:#{doc_id}"] ||= {}
@docs["#{project_id}:#{doc_id}"].lines = lines
setDocument: (project_id, doc_id, lines, version, callback = (error) ->) ->
doc = @docs["#{project_id}:#{doc_id}"] ||= {}
doc.lines = lines
doc.version = version
callback null
getDocument: (project_id, doc_id, callback = (error, doc) ->) ->
@ -28,7 +31,7 @@ module.exports = MockWebApi =
res.send 404
app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) =>
MockWebApi.setDocumentLines req.params.project_id, req.params.doc_id, req.body.lines, (error) ->
MockWebApi.setDocument req.params.project_id, req.params.doc_id, req.body.lines, req.body.version, (error) ->
if error?
res.send 500
else

87
services/document-updater/test/unit/coffee/PersistenceManager/getDocFromWebTests.coffee
View file

@ -1,87 +0,0 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
describe "PersistenceManager.getDocFromWeb", ->
beforeEach ->
@PersistenceManager = SandboxedModule.require modulePath, requires:
"request": @request = sinon.stub()
"settings-sharelatex": @Settings = {}
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@project_id = "project-id-123"
@doc_id = "doc-id-123"
@lines = ["one", "two", "three"]
@callback = sinon.stub()
@Settings.apis =
web:
url: @url = "www.example.com"
user: @user = "sharelatex"
pass: @pass = "password"
describe "with a successful response from the web api", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines))
@PersistenceManager.getDocFromWeb(@project_id, @doc_id, @callback)
it "should call the web api", ->
@request
.calledWith({
url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
method: "GET"
headers:
"accept": "application/json"
auth:
user: @user
pass: @pass
sendImmediately: true
jar: false
timeout: 5000
})
.should.equal true
it "should call the callback with the doc lines", ->
@callback.calledWith(null, @lines).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when request returns an error", ->
beforeEach ->
@request.callsArgWith(1, @error = new Error("oops"), null, null)
@PersistenceManager.getDocFromWeb(@project_id, @doc_id, @callback)
it "should return the error", ->
@callback.calledWith(@error).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns 404", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 404}, "")
@PersistenceManager.getDocFromWeb(@project_id, @doc_id, @callback)
it "should return a NotFoundError", ->
@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns an error status code", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 500}, "")
@PersistenceManager.getDocFromWeb(@project_id, @doc_id, @callback)
it "should return an error", ->
@callback.calledWith(new Error("web api error")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true

88
services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee
View file

@ -3,7 +3,7 @@ chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
{ObjectId} = require("mongojs")
Errors = require "../../../../app/js/Errors"
describe "PersistenceManager.getDoc", ->
beforeEach ->
@ -13,34 +13,76 @@ describe "PersistenceManager.getDoc", ->
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"logger-sharelatex": @logger = {warn: sinon.stub()}
"./mongojs":
db: @db = { docOps: {} }
ObjectId: ObjectId
@project_id = ObjectId().toString()
@doc_id = ObjectId().toString()
@callback = sinon.stub()
@lines = ["mock", "doc", "lines"]
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@project_id = "project-id-123"
@doc_id = "doc-id-123"
@lines = ["one", "two", "three"]
@version = 42
@callback = sinon.stub()
@Settings.apis =
web:
url: @url = "www.example.com"
user: @user = "sharelatex"
pass: @pass = "password"
describe "successfully", ->
describe "with a successful response from the web api", ->
beforeEach ->
@PersistenceManager.getDocFromWeb = sinon.stub().callsArgWith(2, null, @lines)
@PersistenceManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @version)
@PersistenceManager.getDoc @project_id, @doc_id, @callback
@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines, version: @version))
@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
it "should look up the doc in the web api", ->
@PersistenceManager.getDocFromWeb
.calledWith(@project_id, @doc_id)
it "should call the web api", ->
@request
.calledWith({
url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
method: "GET"
headers:
"accept": "application/json"
auth:
user: @user
pass: @pass
sendImmediately: true
jar: false
timeout: 5000
})
.should.equal true
it "should look up the version in Mongo", ->
@PersistenceManager.getDocVersionInMongo
.calledWith(@doc_id)
.should.equal true
it "should call the callback with the lines and version", ->
it "should call the callback with the doc lines and version", ->
@callback.calledWith(null, @lines, @version).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when request returns an error", ->
beforeEach ->
@request.callsArgWith(1, @error = new Error("oops"), null, null)
@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
it "should return the error", ->
@callback.calledWith(@error).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns 404", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 404}, "")
@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
it "should return a NotFoundError", ->
@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns an error status code", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 500}, "")
@PersistenceManager.getDoc(@project_id, @doc_id, @callback)
it "should return an error", ->
@callback.calledWith(new Error("web api error")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true

47
services/document-updater/test/unit/coffee/PersistenceManager/getDocVersionInMongoTests.coffee
View file

@ -1,47 +0,0 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
{ObjectId} = require("mongojs")
describe "PersistenceManager.getDocVersionInMongo", ->
beforeEach ->
@PersistenceManager = SandboxedModule.require modulePath, requires:
"request": @request = sinon.stub()
"settings-sharelatex": @Settings = {}
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"./mongojs":
db: @db = { docOps: {} }
ObjectId: ObjectId
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@doc_id = ObjectId().toString()
@callback = sinon.stub()
describe "getDocVersionInMongo", ->
describe "when the doc exists", ->
beforeEach ->
@doc =
version: @version = 42
@db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc])
@PersistenceManager.getDocVersionInMongo @doc_id, @callback
it "should look for the doc in the database", ->
@db.docOps.find
.calledWith({ doc_id: ObjectId(@doc_id) }, {version: 1})
.should.equal true
it "should call the callback with the version", ->
@callback.calledWith(null, @version).should.equal true
describe "when the doc doesn't exist", ->
beforeEach ->
@db.docOps.find = sinon.stub().callsArgWith(2, null, [])
@PersistenceManager.getDocVersionInMongo @doc_id, @callback
it "should call the callback with 0", ->
@callback.calledWith(null, 0).should.equal true

88
services/document-updater/test/unit/coffee/PersistenceManager/setDocInWebTests.coffee
View file

@ -1,88 +0,0 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
describe "PersistenceManager.setDocInWeb", ->
beforeEach ->
@PersistenceManager = SandboxedModule.require modulePath, requires:
"request": @request = sinon.stub()
"settings-sharelatex": @Settings = {}
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@project_id = "project-id-123"
@doc_id = "doc-id-123"
@lines = ["one", "two", "three"]
@callback = sinon.stub()
@Settings.apis =
web:
url: @url = "www.example.com"
user: @user = "sharelatex"
pass: @pass = "password"
describe "with a successful response from the web api", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines, version: @version))
@PersistenceManager.setDocInWeb(@project_id, @doc_id, @lines, @callback)
it "should call the web api", ->
@request
.calledWith({
url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
body: JSON.stringify
lines: @lines
method: "POST"
headers:
"content-type": "application/json"
auth:
user: @user
pass: @pass
sendImmediately: true
jar: false
timeout: 5000
})
.should.equal true
it "should call the callback without error", ->
@callback.calledWith(null).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when request returns an error", ->
beforeEach ->
@request.callsArgWith(1, @error = new Error("oops"), null, null)
@PersistenceManager.setDocInWeb(@project_id, @doc_id, @lines, @callback)
it "should return the error", ->
@callback.calledWith(@error).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns 404", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 404}, "")
@PersistenceManager.setDocInWeb(@project_id, @doc_id, @lines, @callback)
it "should return a NotFoundError", ->
@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns an error status code", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 500}, "")
@PersistenceManager.setDocInWeb(@project_id, @doc_id, @lines, @callback)
it "should return an error", ->
@callback.calledWith(new Error("web api error")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true

88
services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee
View file

@ -3,6 +3,7 @@ chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
describe "PersistenceManager.setDoc", ->
beforeEach ->
@ -12,27 +13,78 @@ describe "PersistenceManager.setDoc", ->
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"logger-sharelatex": @logger = {warn: sinon.stub()}
@project_id = "mock-project-id"
@doc_id = "mock-doc-id"
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@project_id = "project-id-123"
@doc_id = "doc-id-123"
@lines = ["one", "two", "three"]
@version = 42
@callback = sinon.stub()
@lines = ["mock", "doc", "lines"]
@Settings.apis =
web:
url: @url = "www.example.com"
user: @user = "sharelatex"
pass: @pass = "password"
@PersistenceManager.setDocInWeb = sinon.stub().callsArg(3)
@PersistenceManager.setDocVersionInMongo = sinon.stub().callsArg(2)
describe "with a successful response from the web api", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines, version: @version))
@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback)
@PersistenceManager.setDoc @project_id, @doc_id, @lines, @version, @callback
it "should call the web api", ->
@request
.calledWith({
url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}"
body: JSON.stringify
lines: @lines
version: @version
method: "POST"
headers:
"content-type": "application/json"
auth:
user: @user
pass: @pass
sendImmediately: true
jar: false
timeout: 5000
})
.should.equal true
it "should set the doc in the web api", ->
@PersistenceManager.setDocInWeb
.calledWith(@project_id, @doc_id, @lines)
.should.equal true
it "should call the callback without error", ->
@callback.calledWith(null).should.equal true
it "should set the doc version in mongo", ->
@PersistenceManager.setDocVersionInMongo
.calledWith(@doc_id, @version)
.should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when request returns an error", ->
beforeEach ->
@request.callsArgWith(1, @error = new Error("oops"), null, null)
@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback)
it "should return the error", ->
@callback.calledWith(@error).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns 404", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 404}, "")
@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback)
it "should return a NotFoundError", ->
@callback.calledWith(new Errors.NotFoundError("not found")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
describe "when the request returns an error status code", ->
beforeEach ->
@request.callsArgWith(1, null, {statusCode: 500}, "")
@PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback)
it "should return an error", ->
@callback.calledWith(new Error("web api error")).should.equal true
it "should time the execution", ->
@Metrics.Timer::done.called.should.equal true
it "should call the callback", ->
@callback.called.should.equal true

44
services/document-updater/test/unit/coffee/PersistenceManager/setDocVersionInMongo.coffee
View file

@ -1,44 +0,0 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/PersistenceManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
{ObjectId} = require("mongojs")
describe "PersistenceManager.getDocVersionInMongo", ->
beforeEach ->
@PersistenceManager = SandboxedModule.require modulePath, requires:
"request": @request = sinon.stub()
"settings-sharelatex": @Settings = {}
"./Metrics": @Metrics =
Timer: class Timer
done: sinon.stub()
"./mongojs":
db: @db = { docOps: {} }
ObjectId: ObjectId
"logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()}
@doc_id = ObjectId().toString()
@callback = sinon.stub()
describe "setDocVersionInMongo", ->
beforeEach ->
@version = 42
@db.docOps.update = sinon.stub().callsArg(3)
@PersistenceManager.setDocVersionInMongo @doc_id, @version, @callback
it "should update the doc version", ->
@db.docOps.update
.calledWith({
doc_id: ObjectId(@doc_id)
}, {
$set:
version: @version
}, {
upsert: true
})
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true