From cf780fd8bb927731e6e8b955e4ae22f2371709e7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 1 Aug 2017 14:39:57 +0100 Subject: [PATCH] start making requests incremental --- .../Features/Compile/ClsiManager.coffee | 67 +++++++++++++++++-- .../Features/Compile/ClsiStateManager.coffee | 41 ++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 services/web/app/coffee/Features/Compile/ClsiStateManager.coffee diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index 7caa9cc174..6f9b765554 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -7,6 +7,7 @@ ProjectEntityHandler = require("../Project/ProjectEntityHandler") logger = require "logger-sharelatex" Url = require("url") ClsiCookieManager = require("./ClsiCookieManager") +ClsiStateManager = require("./ClsiStateManager") _ = require("underscore") async = require("async") ClsiFormatChecker = require("./ClsiFormatChecker") @@ -14,10 +15,20 @@ DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler" module.exports = ClsiManager = - sendRequest: (project_id, user_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) -> - ClsiManager._buildRequest project_id, options, (error, req) -> + sendRequest: (project_id, user_id, options = {}, callback) -> + ClsiManager.sendRequestOnce project_id, user_id, _.clone(options), (error, status, result...) -> + return callback(error) if error? + if status is 'conflict' + options.state = "conflict" # will force full compile + ClsiManager.sendRequestOnce project_id, user_id, options, callback # try again + else + callback(error, status, result...) + + sendRequestOnce: (project_id, user_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) -> + ClsiManager._buildRequest project_id, user_id, options, (error, req) -> return callback(error) if error? logger.log project_id: project_id, "sending compile to CLSI" + console.log "REQUEST", JSON.stringify(req, null, 2) ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)-> if err? logger.err err, project_id, "could not check resources for potential problems before sending to clsi" @@ -29,6 +40,9 @@ module.exports = ClsiManager = if error? logger.err err:error, project_id:project_id, "error sending request to clsi" return callback(error) + if response?.compile?.status is "conflict" + # FIXME try again without incremental option + console.log "CONFLICT TRY AGAIN" logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI" ClsiCookieManager._getServerId project_id, (err, clsiServerId)-> if err? @@ -86,6 +100,8 @@ module.exports = ClsiManager = callback null, body else if response.statusCode == 413 callback null, compile:status:"project-too-large" + else if response.statusCode == 409 + callback null, compile:status:"conflict" else error = new Error("CLSI returned non-success code: #{response.statusCode}") logger.error err: error, project_id: project_id, "CLSI returned failure code" @@ -102,17 +118,50 @@ module.exports = ClsiManager = return outputFiles VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"] - _buildRequest: (project_id, options={}, callback = (error, request) ->) -> - Project.findById project_id, {compiler: 1, rootDoc_id: 1, imageName: 1}, (error, project) -> + + _buildRequest: (project_id, user_id, options={}, callback = (error, request) ->) -> + Project.findById project_id, {}, (error, project) -> return callback(error) if error? return callback(new Errors.NotFoundError("project does not exist: #{project_id}")) if !project? - + console.log "PROJECT", project, JSON.stringify(project.rootFolder,null,2) if project.compiler not in ClsiManager.VALID_COMPILERS project.compiler = "pdflatex" - ClsiManager._getContentFromMongo project_id, (error, docs, files) -> + ClsiStateManager.checkState project_id, user_id, project, (error, stateOk, state) -> return callback(error) if error? - ClsiManager._finaliseRequest project_id, options, project, docs, files, callback + logger.log project_id: project_id, checkState: stateOk, "checked project state" + console.log "OPTIONS ARE", options + if stateOk and not options.state? # incremental + ClsiManager._getContentFromDocUpdater project_id, (error, docUpdaterDocs) -> + return callback(error) if error? + # make this incremental + ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) -> + return callback(error) if error? + console.log "PATHS", docPath + console.log "DOCS", docUpdaterDocs + docs = {} + for doc in docUpdaterDocs or [] + path = docPath[doc._id] + docs[path] = doc + console.log "MAPPED DOCS", docs + options.incremental = state + ClsiManager._finaliseRequest project_id, options, project, docs, [], callback + else + ClsiManager._getContentFromMongo project_id, (error, docs, files) -> + return callback(error) if error? + console.log "DOCS", docs + # FIXME want to store state after project has been sent to clsi + ClsiStateManager.setState project_id, user_id, project, (error, state) -> + if error? + logger.err err:error, project_id:project_id, "error storing state in redis" + #return callback(error) + options.state = state + ClsiManager._finaliseRequest project_id, options, project, docs, files, callback + + _getContentFromDocUpdater: (project_id, callback = (error, docs) ->) -> + DocumentUpdaterHandler.getProjectDocs project_id, (error, docs) -> + return callback(error) if error? + callback(null, docs) _getContentFromMongo: (project_id, callback = (error, docs, files) ->) -> DocumentUpdaterHandler.flushProjectToMongo project_id, (error) -> @@ -120,6 +169,7 @@ module.exports = ClsiManager = ProjectEntityHandler.getAllDocs project_id, (error, docs = {}) -> return callback(error) if error? ProjectEntityHandler.getAllFiles project_id, (error, files = {}) -> + return callback(error) if error? callback(null, docs, files) _finaliseRequest: (project_id, options, project, docs, files, callback = (error, params) -> ) -> @@ -132,6 +182,7 @@ module.exports = ClsiManager = resources.push path: path content: doc.lines.join("\n") + rev: doc.rev if project.rootDoc_id? and doc._id.toString() == project.rootDoc_id.toString() rootResourcePath = path if options.rootDoc_id? and doc._id.toString() == options.rootDoc_id.toString() @@ -157,6 +208,8 @@ module.exports = ClsiManager = imageName: project.imageName draft: !!options.draft check: options.check + incremental: options.incremental + state: options.state rootResourcePath: rootResourcePath resources: resources } diff --git a/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee b/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee new file mode 100644 index 0000000000..f090ec6fc2 --- /dev/null +++ b/services/web/app/coffee/Features/Compile/ClsiStateManager.coffee @@ -0,0 +1,41 @@ +Settings = require "settings-sharelatex" +RedisWrapper = require("../../infrastructure/RedisWrapper") +rclient = RedisWrapper.client("clsi_state") +logger = require "logger-sharelatex" +crypto = require "crypto" + +buildKey = (project_id, user_id)-> + return "clsistate:#{project_id}-#{user_id}" # FIXME: should we cluster these on project?? + +buildState = (project) -> + JSON.stringify project + +clsiStateEnabled = Settings.clsiState + +OneHour = 3600 * 1000 + +module.exports = ClsiStateManager = + + checkState: (project_id, user_id, project, callback = (err, ok) ->) -> + newState = buildState(project) + @getState project_id, user_id, (err, oldState) -> + return callback(err) if err? + if newState is oldState + hash = crypto.createHash('sha1').update(newState, 'utf8').digest('hex') + callback(null,true,hash) + else + callback(null,false) + + getState: (project_id, user_id, callback = (err, state)->)-> + rclient.get buildKey(project_id, user_id), (err, state)-> + return callback(err) if err? + logger.log project_id: project_id, user_id: user_id, state: state, "got project state from redis" + return callback(null, state) + + setState: (project_id, user_id, project, callback = (err)->)-> + projectState = buildState project + logger.log project_id: project_id, user_id: user_id, projectState: projectState, "setting project state in redis" + rclient.set buildKey(project_id, user_id), projectState, "PX", OneHour, (err) -> + return callback(err) if err? + hash = crypto.createHash('sha1').update(projectState, 'utf8').digest('hex') + callback(null,hash)