github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-29 13:23:40 -05:00
Code Issues Projects Releases Packages Wiki Activity

handle non-urgent flushes in background

Browse source
This commit is contained in:
Brian Gough 2019-06-03 10:01:10 +01:00
parent 6eba954b52
commit e95059f98e
7 changed files with 20 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
services/document-updater/app/coffee/HistoryManager.coffee
View file

@ -23,14 +23,16 @@ module.exports = HistoryManager =
# flush changes in the background # flush changes in the background
flushProjectChangesAsync: (project_id) -> flushProjectChangesAsync: (project_id) ->
return if !Settings.apis?.project_history?.enabled return if !Settings.apis?.project_history?.enabled
HistoryManager.flushProjectChanges project_id, -> HistoryManager.flushProjectChanges project_id, {background:true}, ->
# flush changes and callback (for when we need to know the queue is flushed) # flush changes and callback (for when we need to know the queue is flushed)
flushProjectChanges: (project_id, callback = (error) ->) -> flushProjectChanges: (project_id, options, callback = (error) ->) ->
return callback() if !Settings.apis?.project_history?.enabled return callback() if !Settings.apis?.project_history?.enabled
url = "#{Settings.apis.project_history.url}/project/#{project_id}/flush" url = "#{Settings.apis.project_history.url}/project/#{project_id}/flush"
logger.log { project_id, url }, "flushing doc in project history api" qs = {}
request.post url, (error, res, body)-> qs.background = true if options.background # pass on the background flush option if present
logger.log { project_id, url, qs }, "flushing doc in project history api"
request.post {url: url, qs: qs}, (error, res, body)->
if error? if error?
logger.error { error, project_id}, "project history doc to track changes api" logger.error { error, project_id}, "project history doc to track changes api"
return callback(error) return callback(error)

4
services/document-updater/app/coffee/HttpController.coffee
View file

@ -131,7 +131,9 @@ module.exports = HttpController =
project_id = req.params.project_id project_id = req.params.project_id
logger.log project_id: project_id, "deleting project via http" logger.log project_id: project_id, "deleting project via http"
timer = new Metrics.Timer("http.deleteProject") timer = new Metrics.Timer("http.deleteProject")
ProjectManager.flushAndDeleteProjectWithLocks project_id, (error) -> options = {}
options.background = true if req.query?.background # allow non-urgent flushes to be queued
ProjectManager.flushAndDeleteProjectWithLocks project_id, options, (error) ->
timer.done() timer.done()
return next(error) if error? return next(error) if error?
logger.log project_id: project_id, "deleted project via http" logger.log project_id: project_id, "deleted project via http"

2
services/document-updater/app/coffee/ProjectFlusher.coffee
View file

@ -55,7 +55,7 @@ ProjectFlusher =
return callback(null, project_ids) return callback(null, project_ids)
jobs = _.map project_ids, (project_id)-> jobs = _.map project_ids, (project_id)->
return (cb)-> return (cb)->
ProjectManager.flushAndDeleteProjectWithLocks project_id, cb ProjectManager.flushAndDeleteProjectWithLocks project_id, {background:true}, cb
async.parallelLimit jobs, options.concurrency, (error)-> async.parallelLimit jobs, options.concurrency, (error)->
return callback(error, project_ids) return callback(error, project_ids)

4
services/document-updater/app/coffee/ProjectManager.coffee
View file

@ -39,7 +39,7 @@ module.exports = ProjectManager =
else else
callback(null) callback(null)
flushAndDeleteProjectWithLocks: (project_id, _callback = (error) ->) -> flushAndDeleteProjectWithLocks: (project_id, options, _callback = (error) ->) ->
timer = new Metrics.Timer("projectManager.flushAndDeleteProjectWithLocks") timer = new Metrics.Timer("projectManager.flushAndDeleteProjectWithLocks")
callback = (args...) -> callback = (args...) ->
timer.done() timer.done()
@ -64,7 +64,7 @@ module.exports = ProjectManager =
# history is completely flushed because the project may be # history is completely flushed because the project may be
# deleted in web after this call completes, and so further # deleted in web after this call completes, and so further
# attempts to flush would fail after that. # attempts to flush would fail after that.
HistoryManager.flushProjectChanges project_id, (error) -> HistoryManager.flushProjectChanges project_id, options, (error) ->
if errors.length > 0 if errors.length > 0
callback new Error("Errors deleting docs. See log for details") callback new Error("Errors deleting docs. See log for details")
else if error? else if error?

2
services/document-updater/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
View file

@ -43,7 +43,7 @@ describe "HistoryManager", ->
it "should send a request to the project history api", -> it "should send a request to the project history api", ->
@request.post @request.post
.calledWith("#{@Settings.apis.project_history.url}/project/#{@project_id}/flush") .calledWith({url: "#{@Settings.apis.project_history.url}/project/#{@project_id}/flush", qs:{background:true}})
.should.equal true .should.equal true
describe "recordAndFlushHistoryOps", -> describe "recordAndFlushHistoryOps", ->

4
services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee
View file

@ -322,7 +322,7 @@ describe "HttpController", ->
describe "successfully", -> describe "successfully", ->
beforeEach -> beforeEach ->
@ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1) @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(2)
@HttpController.deleteProject(@req, @res, @next) @HttpController.deleteProject(@req, @res, @next)
it "should delete the project", -> it "should delete the project", ->
@ -345,7 +345,7 @@ describe "HttpController", ->
describe "when an errors occurs", -> describe "when an errors occurs", ->
beforeEach -> beforeEach ->
@ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(2, new Error("oops"))
@HttpController.deleteProject(@req, @res, @next) @HttpController.deleteProject(@req, @res, @next)
it "should call next with the error", -> it "should call next with the error", ->

10
services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee
View file

@ -12,7 +12,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
"./DocumentManager": @DocumentManager = {} "./DocumentManager": @DocumentManager = {}
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
"./HistoryManager": @HistoryManager = "./HistoryManager": @HistoryManager =
flushProjectChanges: sinon.stub().callsArg(1) flushProjectChanges: sinon.stub().callsArg(2)
"./Metrics": @Metrics = "./Metrics": @Metrics =
Timer: class Timer Timer: class Timer
done: sinon.stub() done: sinon.stub()
@ -24,7 +24,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
@doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"] @doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"]
@RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids) @RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids)
@DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArg(2) @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArg(2)
@ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) => @ProjectManager.flushAndDeleteProjectWithLocks @project_id, {}, (error) =>
@callback(error) @callback(error)
done() done()
@ -41,7 +41,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
it "should flush project history", -> it "should flush project history", ->
@HistoryManager.flushProjectChanges @HistoryManager.flushProjectChanges
.calledWith(@project_id) .calledWith(@project_id, {})
.should.equal true .should.equal true
it "should call the callback without error", -> it "should call the callback without error", ->
@ -59,7 +59,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
callback(@error = new Error("oops, something went wrong")) callback(@error = new Error("oops, something went wrong"))
else else
callback() callback()
@ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) => @ProjectManager.flushAndDeleteProjectWithLocks @project_id, {}, (error) =>
@callback(error) @callback(error)
done() done()
@ -71,7 +71,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
it "should still flush project history", -> it "should still flush project history", ->
@HistoryManager.flushProjectChanges @HistoryManager.flushProjectChanges
.calledWith(@project_id) .calledWith(@project_id, {})
.should.equal true .should.equal true
it "should record the error", -> it "should record the error", ->