diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index d14160b46d..7ec267ede2 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -123,35 +123,40 @@ module.exports = ClsiManager = if project.compiler not in ClsiManager.VALID_COMPILERS project.compiler = "pdflatex" - ClsiStateManager.checkProjectStateMatch project_id, project, (error, projectStateUnchanged, projectStateHash) -> + ClsiManager.getContentFromDocUpdaterIfMatch project_id, project, (error, projectStateHash, docUpdaterDocs) -> return callback(error) if error? - logger.log project_id: project_id, projectStateUnchanged: projectStateUnchanged, "checked project state" + logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state" # see if we can send an incremental update to the CLSI - if projectStateUnchanged and options.syncType isnt "full" - ClsiManager._getContentFromDocUpdater project_id, (error, docUpdaterDocs) -> - return callback(error) if error? - ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) -> - return callback(error) if error? - docs = {} - for doc in docUpdaterDocs or [] - path = docPath[doc._id] - docs[path] = doc - options.syncType = "incremental" - options.syncState = projectStateHash - # send new docs but not files as those are already on the clsi - ClsiManager._finaliseRequest project_id, options, project, docs, [], callback + if docUpdaterDocs? and options.syncType isnt "full" + ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback else - ClsiManager._getContentFromMongo project_id, (error, docs, files) -> - return callback(error) if error? - # FIXME want to store state after project has been sent to - # clsi, but need to do it here. - ClsiStateManager.setProjectState project_id, project, (error, projectStateHash) -> - if error? - logger.err err:error, project_id:project_id, "error storing state in redis" - #return callback(error) - options.syncType = "full" - options.syncState = projectStateHash - ClsiManager._finaliseRequest project_id, options, project, docs, files, callback + ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback + + getContentFromDocUpdaterIfMatch: (project_id, project, callback = (error, projectStateHash, docs) ->) -> + ClsiStateManager.computeHash project, (error, projectStateHash) -> + return callback(error) if error? + DocumentUpdaterHandler.getProjectDocsIfMatch project_id, projectStateHash, (error, docs) -> + return callback(error) if error? + callback(null, projectStateHash, docs) + + _buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) -> + ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) -> + return callback(error) if error? + docs = {} + for doc in docUpdaterDocs or [] + path = docPath[doc._id] + docs[path] = doc + # send new docs but not files as those are already on the clsi + options.syncType = "incremental" + options.syncState = projectStateHash + ClsiManager._finaliseRequest project_id, options, project, docs, [], callback + + _buildRequestFromMongo: (project_id, options, project, projectStateHash, callback = (error, request) ->) -> + ClsiManager._getContentFromMongo project_id, (error, docs, files) -> + return callback(error) if error? + options.syncType = "full" + options.syncState = projectStateHash + ClsiManager._finaliseRequest project_id, options, project, docs, files, callback _getContentFromDocUpdater: (project_id, callback = (error, docs) ->) -> DocumentUpdaterHandler.getProjectDocs project_id, (error, docs) -> diff --git a/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee b/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee index 6fc7b1dbf0..715592e4dc 100644 --- a/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee @@ -1,12 +1,7 @@ Settings = require "settings-sharelatex" -RedisWrapper = require("../../infrastructure/RedisWrapper") -rclient = RedisWrapper.client("clsi_state") logger = require "logger-sharelatex" crypto = require "crypto" -buildKey = (project_id)-> - return "clsistate:#{project_id}" # FIXME: should we cluster these on project?? - # The "state" of a project is a hash of the relevant attributes in the # project object in this case we only need the rootFolder. # @@ -18,32 +13,15 @@ buildKey = (project_id)-> # be updated. If it doesn't change then we can overwrite changed docs # in place on the clsi, getting them from the docupdater. # -# The docupdater is also responsible for unsetting the key in redis if -# it removes any documents from the doc updater. +# The docupdater is responsible for setting the key in redis, and +# unsetting it if it removes any documents from the doc updater. buildState = (project) -> json = JSON.stringify(project.rootFolder) return crypto.createHash('sha1').update(json, 'utf8').digest('hex') -clsiStateEnabled = Settings.clsiState - -OneHour = 3600 * 1000 - module.exports = ClsiStateManager = - checkProjectStateMatch: (project_id, project, callback = (err, ok) ->) -> - newState = buildState(project) - rclient.get buildKey(project_id), (err, oldState) -> - return callback(err) if err? - logger.log project_id: project_id, new_state: newState, old_state: oldState, "got project state from redis" - if newState is oldState - callback(null,true,oldState) - else - callback(null,false) - - setProjectState: (project_id, project, callback = (err)->)-> - projectState = buildState(project) - logger.log project_id: project_id, projectState: projectState, "setting project state in redis" - rclient.set buildKey(project_id), projectState, "PX", OneHour, (err) -> - return callback(err) if err? - callback(null,projectState) + computeHash: (project, callback = (err, hash) ->) -> + hash = buildState(project) + callback(null, hash) diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index 50c263f938..6064a41363 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -123,16 +123,18 @@ module.exports = DocumentUpdaterHandler = logger.error project_id:project_id, doc_id:doc_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}" callback new Error("doc updater returned a non-success status code: #{res.statusCode}") - getProjectDocs: (project_id, callback = (error, docs) ->) -> + getProjectDocsIfMatch: (project_id, projectStateHash, callback = (error, docs) ->) -> timer = new metrics.Timer("get-project-docs") - url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc" + url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc?state=#{projectStateHash}" logger.log project_id:project_id, "getting project docs from document updater" request.get url, (error, res, body)-> timer.done() if error? logger.error err:error, url:url, project_id:project_id, "error getting project docs from doc updater" return callback(error) - if res.statusCode >= 200 and res.statusCode < 300 + if res.statusCode is 409 + return callback() + else if res.statusCode >= 200 and res.statusCode < 300 logger.log project_id:project_id, "got project docs from document document updater" try docs = JSON.parse(body) @@ -140,7 +142,6 @@ module.exports = DocumentUpdaterHandler = doc.lines = JSON.parse(doc.lines) catch error return callback(error) - logger.log project_id: project_id, docs: docs, "RESULT" callback null, docs else logger.error project_id:project_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}" diff --git a/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee index 88a8642358..ca828ebf8d 100644 --- a/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee +++ b/services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee @@ -84,7 +84,6 @@ module.exports = ProjectEntityHandler = docPath = {} for folderPath, folder of folders for doc in (folder.docs or []) - console.log "PATH", path.join(folderPath, doc.name), doc._id, doc.name docPath[doc._id] = path.join(folderPath, doc.name) logger.log count:_.keys(docPath).length, project_id:project._id, "returning docPaths for project" callback null, docPath