github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-09 17:49:03 +00:00
Code Issues Projects Releases Packages Wiki Activity

simplify incremental request to docupdater

Browse source 
if project state hasn't changed, get the docs from the docupdater -- we
check/set the hash and return the docs in a single request.  Otherwise
do a full request from mongo.
This commit is contained in:
Brian Gough 2017-08-07 14:45:04 +01:00
parent 1321009fe1
commit 849e905efb
4 changed files with 41 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
services/web/app/coffee/Features/Compile/ClsiManager.coffee
View file

@ -123,35 +123,40 @@ module.exports = ClsiManager =
if project.compiler not in ClsiManager.VALID_COMPILERS
project.compiler = "pdflatex"
ClsiStateManager.checkProjectStateMatch project_id, project, (error, projectStateUnchanged, projectStateHash) ->
ClsiManager.getContentFromDocUpdaterIfMatch project_id, project, (error, projectStateHash, docUpdaterDocs) ->
return callback(error) if error?
logger.log project_id: project_id, projectStateUnchanged: projectStateUnchanged, "checked project state"
logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state"
# see if we can send an incremental update to the CLSI
if projectStateUnchanged and options.syncType isnt "full"
ClsiManager._getContentFromDocUpdater project_id, (error, docUpdaterDocs) ->
return callback(error) if error?
ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
return callback(error) if error?
docs = {}
for doc in docUpdaterDocs or []
path = docPath[doc._id]
docs[path] = doc
options.syncType = "incremental"
options.syncState = projectStateHash
# send new docs but not files as those are already on the clsi
ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
if docUpdaterDocs? and options.syncType isnt "full"
ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
else
ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
return callback(error) if error?
# FIXME want to store state after project has been sent to
# clsi, but need to do it here.
ClsiStateManager.setProjectState project_id, project, (error, projectStateHash) ->
if error?
logger.err err:error, project_id:project_id, "error storing state in redis"
#return callback(error)
options.syncType = "full"
options.syncState = projectStateHash
ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback
getContentFromDocUpdaterIfMatch: (project_id, project, callback = (error, projectStateHash, docs) ->) ->
ClsiStateManager.computeHash project, (error, projectStateHash) ->
return callback(error) if error?
DocumentUpdaterHandler.getProjectDocsIfMatch project_id, projectStateHash, (error, docs) ->
return callback(error) if error?
callback(null, projectStateHash, docs)
_buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) ->
ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
return callback(error) if error?
docs = {}
for doc in docUpdaterDocs or []
path = docPath[doc._id]
docs[path] = doc
# send new docs but not files as those are already on the clsi
options.syncType = "incremental"
options.syncState = projectStateHash
ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
_buildRequestFromMongo: (project_id, options, project, projectStateHash, callback = (error, request) ->) ->
ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
return callback(error) if error?
options.syncType = "full"
options.syncState = projectStateHash
ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
_getContentFromDocUpdater: (project_id, callback = (error, docs) ->) ->
DocumentUpdaterHandler.getProjectDocs project_id, (error, docs) ->

32
services/web/app/coffee/Features/Compile/ClsiStateManager.coffee
View file

@ -1,12 +1,7 @@
Settings = require "settings-sharelatex"
RedisWrapper = require("../../infrastructure/RedisWrapper")
rclient = RedisWrapper.client("clsi_state")
logger = require "logger-sharelatex"
crypto = require "crypto"
buildKey = (project_id)->
return "clsistate:#{project_id}" # FIXME: should we cluster these on project??
# The "state" of a project is a hash of the relevant attributes in the
# project object in this case we only need the rootFolder.
#
@ -18,32 +13,15 @@ buildKey = (project_id)->
# be updated. If it doesn't change then we can overwrite changed docs
# in place on the clsi, getting them from the docupdater.
#
# The docupdater is also responsible for unsetting the key in redis if
# it removes any documents from the doc updater.
# The docupdater is responsible for setting the key in redis, and
# unsetting it if it removes any documents from the doc updater.
buildState = (project) ->
json = JSON.stringify(project.rootFolder)
return crypto.createHash('sha1').update(json, 'utf8').digest('hex')
clsiStateEnabled = Settings.clsiState
OneHour = 3600 * 1000
module.exports = ClsiStateManager =
checkProjectStateMatch: (project_id, project, callback = (err, ok) ->) ->
newState = buildState(project)
rclient.get buildKey(project_id), (err, oldState) ->
return callback(err) if err?
logger.log project_id: project_id, new_state: newState, old_state: oldState, "got project state from redis"
if newState is oldState
callback(null,true,oldState)
else
callback(null,false)
setProjectState: (project_id, project, callback = (err)->)->
projectState = buildState(project)
logger.log project_id: project_id, projectState: projectState, "setting project state in redis"
rclient.set buildKey(project_id), projectState, "PX", OneHour, (err) ->
return callback(err) if err?
callback(null,projectState)
computeHash: (project, callback = (err, hash) ->) ->
hash = buildState(project)
callback(null, hash)

9
services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee
View file

@ -123,16 +123,18 @@ module.exports = DocumentUpdaterHandler =
logger.error project_id:project_id, doc_id:doc_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}"
callback new Error("doc updater returned a non-success status code: #{res.statusCode}")
getProjectDocs: (project_id, callback = (error, docs) ->) ->
getProjectDocsIfMatch: (project_id, projectStateHash, callback = (error, docs) ->) ->
timer = new metrics.Timer("get-project-docs")
url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc"
url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc?state=#{projectStateHash}"
logger.log project_id:project_id, "getting project docs from document updater"
request.get url, (error, res, body)->
timer.done()
if error?
logger.error err:error, url:url, project_id:project_id, "error getting project docs from doc updater"
return callback(error)
if res.statusCode >= 200 and res.statusCode < 300
if res.statusCode is 409
return callback()
else if res.statusCode >= 200 and res.statusCode < 300
logger.log project_id:project_id, "got project docs from document document updater"
try
docs = JSON.parse(body)
@ -140,7 +142,6 @@ module.exports = DocumentUpdaterHandler =
doc.lines = JSON.parse(doc.lines)
catch error
return callback(error)
logger.log project_id: project_id, docs: docs, "RESULT"
callback null, docs
else
logger.error project_id:project_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}"

1
services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee
View file

@ -84,7 +84,6 @@ module.exports = ProjectEntityHandler =
docPath = {}
for folderPath, folder of folders
for doc in (folder.docs or [])
console.log "PATH", path.join(folderPath, doc.name), doc._id, doc.name
docPath[doc._id] = path.join(folderPath, doc.name)
logger.log count:_.keys(docPath).length, project_id:project._id, "returning docPaths for project"
callback null, docPath