mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
342 lines
15 KiB
CoffeeScript
Executable file
342 lines
15 KiB
CoffeeScript
Executable file
Path = require "path"
|
|
async = require "async"
|
|
Settings = require "settings-sharelatex"
|
|
request = require('request')
|
|
Project = require("../../models/Project").Project
|
|
ProjectGetter = require("../Project/ProjectGetter")
|
|
ProjectEntityHandler = require("../Project/ProjectEntityHandler")
|
|
logger = require "logger-sharelatex"
|
|
Url = require("url")
|
|
ClsiCookieManager = require("./ClsiCookieManager")()
|
|
NewBackendCloudClsiCookieManager = require("./ClsiCookieManager")("newBackendcloud")
|
|
ClsiStateManager = require("./ClsiStateManager")
|
|
_ = require("underscore")
|
|
async = require("async")
|
|
ClsiFormatChecker = require("./ClsiFormatChecker")
|
|
DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler"
|
|
Metrics = require('metrics-sharelatex')
|
|
Errors = require ('../Errors/Errors')
|
|
|
|
module.exports = ClsiManager =
|
|
|
|
sendRequest: (project_id, user_id, options = {}, callback) ->
|
|
ClsiManager.sendRequestOnce project_id, user_id, options, (error, status, result...) ->
|
|
return callback(error) if error?
|
|
if status is 'conflict'
|
|
options = _.clone(options)
|
|
options.syncType = "full" # force full compile
|
|
ClsiManager.sendRequestOnce project_id, user_id, options, callback # try again
|
|
else
|
|
callback(error, status, result...)
|
|
|
|
sendRequestOnce: (project_id, user_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
|
|
ClsiManager._buildRequest project_id, options, (error, req) ->
|
|
if error?
|
|
if error.message is "no main file specified"
|
|
return callback(null, "validation-problems", null, null, {mainFile:error.message})
|
|
else
|
|
return callback(error)
|
|
logger.log project_id: project_id, "sending compile to CLSI"
|
|
ClsiManager._sendBuiltRequest project_id, user_id, req, options, (error, status, result...) ->
|
|
return callback(error) if error?
|
|
callback(error, status, result...)
|
|
|
|
# for public API requests where there is no project id
|
|
sendExternalRequest: (submission_id, clsi_request, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
|
|
logger.log submission_id: submission_id, "sending external compile to CLSI", clsi_request
|
|
ClsiManager._sendBuiltRequest submission_id, null, clsi_request, options, (error, status, result...) ->
|
|
return callback(error) if error?
|
|
callback(error, status, result...)
|
|
|
|
stopCompile: (project_id, user_id, options, callback = (error) ->) ->
|
|
compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id, "compile/stop")
|
|
opts =
|
|
url:compilerUrl
|
|
method:"POST"
|
|
ClsiManager._makeRequest project_id, opts, callback
|
|
|
|
deleteAuxFiles: (project_id, user_id, options, callback = (error) ->) ->
|
|
compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id)
|
|
opts =
|
|
url:compilerUrl
|
|
method:"DELETE"
|
|
ClsiManager._makeRequest project_id, opts, (clsiError) ->
|
|
# always clear the project state from the docupdater, even if there
|
|
# was a problem with the request to the clsi
|
|
DocumentUpdaterHandler.clearProjectState project_id, (docUpdaterError) ->
|
|
error = clsiError or docUpdaterError
|
|
return callback(error) if error?
|
|
callback()
|
|
|
|
_sendBuiltRequest: (project_id, user_id, req, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
|
|
ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
|
|
if err?
|
|
logger.err err, project_id, "could not check resources for potential problems before sending to clsi"
|
|
return callback(err)
|
|
if validationProblems?
|
|
logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
|
|
return callback(null, "validation-problems", null, null, validationProblems)
|
|
ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) ->
|
|
if error?
|
|
logger.err err:error, project_id:project_id, "error sending request to clsi"
|
|
return callback(error)
|
|
logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI"
|
|
ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
|
|
if err?
|
|
logger.err err:err, project_id:project_id, "error getting server id"
|
|
return callback(err)
|
|
outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
|
|
callback(null, response?.compile?.status, outputFiles, clsiServerId)
|
|
|
|
_makeRequest: (project_id, opts, callback)->
|
|
async.series {
|
|
currentBackend: (cb)->
|
|
startTime = new Date()
|
|
ClsiCookieManager.getCookieJar project_id, (err, jar)->
|
|
if err?
|
|
logger.err err:err, "error getting cookie jar for clsi request"
|
|
return callback(err)
|
|
opts.jar = jar
|
|
timer = new Metrics.Timer("compile.currentBackend")
|
|
request opts, (err, response, body)->
|
|
timer.done()
|
|
if err?
|
|
logger.err err:err, project_id:project_id, url:opts?.url, "error making request to clsi"
|
|
return callback(err)
|
|
ClsiCookieManager.setServerId project_id, response, (err)->
|
|
if err?
|
|
logger.warn err:err, project_id:project_id, "error setting server id"
|
|
callback err, response, body #return as soon as the standard compile has returned
|
|
cb(err, {response:response, body:body, finishTime:new Date() - startTime })
|
|
newBackend: (cb)->
|
|
startTime = new Date()
|
|
ClsiManager._makeNewBackendRequest project_id, opts, (err, response, body)->
|
|
cb(err, {response:response, body:body, finishTime:new Date() - startTime})
|
|
}, (err, results)->
|
|
timeDifference = results.newBackend?.finishTime - results.currentBackend?.finishTime
|
|
statusCodeSame = results.newBackend?.response?.statusCode == results.currentBackend?.response?.statusCode
|
|
currentCompileTime = results.currentBackend?.finishTime
|
|
newBackendCompileTime = results.newBackend?.finishTime
|
|
logger.log {statusCodeSame, timeDifference, currentCompileTime, newBackendCompileTime}, "both clsi requests returned"
|
|
|
|
|
|
|
|
_makeNewBackendRequest: (project_id, baseOpts, callback)->
|
|
if !Settings.apis.clsi_new?.url?
|
|
return callback()
|
|
opts = _.clone(baseOpts)
|
|
opts.url = opts.url.replace(Settings.apis.clsi.url, Settings.apis.clsi_new?.url)
|
|
NewBackendCloudClsiCookieManager.getCookieJar project_id, (err, jar)->
|
|
if err?
|
|
logger.err err:err, "error getting cookie jar for clsi request"
|
|
return callback(err)
|
|
opts.jar = jar
|
|
timer = new Metrics.Timer("compile.newBackend")
|
|
request opts, (err, response, body)->
|
|
timer.done()
|
|
if err?
|
|
logger.warn err:err, project_id:project_id, url:opts?.url, "error making request to new clsi"
|
|
return callback(err)
|
|
NewBackendCloudClsiCookieManager.setServerId project_id, response, (err)->
|
|
if err?
|
|
logger.warn err:err, project_id:project_id, "error setting server id new backend"
|
|
return callback err, response, body
|
|
|
|
|
|
_getCompilerUrl: (compileGroup, project_id, user_id, action) ->
|
|
host = Settings.apis.clsi.url
|
|
path = "/project/#{project_id}"
|
|
path += "/user/#{user_id}" if user_id?
|
|
path += "/#{action}" if action?
|
|
return "#{host}#{path}"
|
|
|
|
_postToClsi: (project_id, user_id, req, compileGroup, callback = (error, response) ->) ->
|
|
compileUrl = @_getCompilerUrl(compileGroup, project_id, user_id, "compile")
|
|
opts =
|
|
url: compileUrl
|
|
json: req
|
|
method: "POST"
|
|
ClsiManager._makeRequest project_id, opts, (error, response, body) ->
|
|
return callback(error) if error?
|
|
if 200 <= response.statusCode < 300
|
|
callback null, body
|
|
else if response.statusCode == 413
|
|
callback null, compile:status:"project-too-large"
|
|
else if response.statusCode == 409
|
|
callback null, compile:status:"conflict"
|
|
else if response.statusCode == 423
|
|
callback null, compile:status:"compile-in-progress"
|
|
else
|
|
error = new Error("CLSI returned non-success code: #{response.statusCode}")
|
|
logger.error err: error, project_id: project_id, "CLSI returned failure code"
|
|
callback error, body
|
|
|
|
_parseOutputFiles: (project_id, rawOutputFiles = []) ->
|
|
outputFiles = []
|
|
for file in rawOutputFiles
|
|
outputFiles.push
|
|
path: file.path # the clsi is now sending this to web
|
|
url: Url.parse(file.url).path # the location of the file on the clsi, excluding the host part
|
|
type: file.type
|
|
build: file.build
|
|
return outputFiles
|
|
|
|
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
|
|
|
|
_buildRequest: (project_id, options={}, callback = (error, request) ->) ->
|
|
ProjectGetter.getProject project_id, {compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder:1}, (error, project) ->
|
|
return callback(error) if error?
|
|
return callback(new Errors.NotFoundError("project does not exist: #{project_id}")) if !project?
|
|
if project.compiler not in ClsiManager.VALID_COMPILERS
|
|
project.compiler = "pdflatex"
|
|
|
|
if options.incrementalCompilesEnabled or options.syncType? # new way, either incremental or full
|
|
timer = new Metrics.Timer("editor.compile-getdocs-redis")
|
|
ClsiManager.getContentFromDocUpdaterIfMatch project_id, project, options, (error, projectStateHash, docUpdaterDocs) ->
|
|
timer.done()
|
|
if error?
|
|
logger.error err: error, project_id: project_id, "error checking project state"
|
|
# note: we don't bail out when there's an error getting
|
|
# incremental files from the docupdater, we just fall back
|
|
# to a normal compile below
|
|
else
|
|
logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state"
|
|
# see if we can send an incremental update to the CLSI
|
|
if docUpdaterDocs? and (options.syncType isnt "full") and not error?
|
|
Metrics.inc "compile-from-redis"
|
|
ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
|
|
else
|
|
Metrics.inc "compile-from-mongo"
|
|
ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback
|
|
else # old way, always from mongo
|
|
timer = new Metrics.Timer("editor.compile-getdocs-mongo")
|
|
ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
|
|
timer.done()
|
|
return callback(error) if error?
|
|
ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
|
|
|
|
getContentFromDocUpdaterIfMatch: (project_id, project, options, callback = (error, projectStateHash, docs) ->) ->
|
|
ClsiStateManager.computeHash project, options, (error, projectStateHash) ->
|
|
return callback(error) if error?
|
|
DocumentUpdaterHandler.getProjectDocsIfMatch project_id, projectStateHash, (error, docs) ->
|
|
return callback(error) if error?
|
|
callback(null, projectStateHash, docs)
|
|
|
|
getOutputFileStream: (project_id, user_id, build_id, output_file_path, callback=(err, readStream)->) ->
|
|
url = "#{Settings.apis.clsi.url}/project/#{project_id}/user/#{user_id}/build/#{build_id}/output/#{output_file_path}"
|
|
ClsiCookieManager.getCookieJar project_id, (err, jar)->
|
|
return callback(err) if err?
|
|
options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar }
|
|
readStream = request(options)
|
|
callback(null, readStream)
|
|
|
|
_buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) ->
|
|
ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
|
|
return callback(error) if error?
|
|
docs = {}
|
|
for doc in docUpdaterDocs or []
|
|
path = docPath[doc._id]
|
|
docs[path] = doc
|
|
# send new docs but not files as those are already on the clsi
|
|
options = _.clone(options)
|
|
options.syncType = "incremental"
|
|
options.syncState = projectStateHash
|
|
# create stub doc entries for any possible root docs, if not
|
|
# present in the docupdater. This allows finaliseRequest to
|
|
# identify the root doc.
|
|
possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id]
|
|
for rootDoc_id in possibleRootDocIds when rootDoc_id? and rootDoc_id of docPath
|
|
path = docPath[rootDoc_id]
|
|
docs[path] ?= {_id: rootDoc_id, path: path}
|
|
ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
|
|
|
|
_buildRequestFromMongo: (project_id, options, project, projectStateHash, callback = (error, request) ->) ->
|
|
ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
|
|
return callback(error) if error?
|
|
options = _.clone(options)
|
|
options.syncType = "full"
|
|
options.syncState = projectStateHash
|
|
ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
|
|
|
|
_getContentFromMongo: (project_id, callback = (error, docs, files) ->) ->
|
|
DocumentUpdaterHandler.flushProjectToMongo project_id, (error) ->
|
|
return callback(error) if error?
|
|
ProjectEntityHandler.getAllDocs project_id, (error, docs = {}) ->
|
|
return callback(error) if error?
|
|
ProjectEntityHandler.getAllFiles project_id, (error, files = {}) ->
|
|
return callback(error) if error?
|
|
callback(null, docs, files)
|
|
|
|
_finaliseRequest: (project_id, options, project, docs, files, callback = (error, params) -> ) ->
|
|
resources = []
|
|
rootResourcePath = null
|
|
rootResourcePathOverride = null
|
|
hasMainFile = false
|
|
numberOfDocsInProject = 0
|
|
|
|
for path, doc of docs
|
|
path = path.replace(/^\//, "") # Remove leading /
|
|
numberOfDocsInProject++
|
|
if doc.lines? # add doc to resources unless it is just a stub entry
|
|
resources.push
|
|
path: path
|
|
content: doc.lines.join("\n")
|
|
if project.rootDoc_id? and doc._id.toString() == project.rootDoc_id.toString()
|
|
rootResourcePath = path
|
|
if options.rootDoc_id? and doc._id.toString() == options.rootDoc_id.toString()
|
|
rootResourcePathOverride = path
|
|
if path is "main.tex"
|
|
hasMainFile = true
|
|
|
|
rootResourcePath = rootResourcePathOverride if rootResourcePathOverride?
|
|
if !rootResourcePath?
|
|
if hasMainFile
|
|
logger.warn {project_id}, "no root document found, setting to main.tex"
|
|
rootResourcePath = "main.tex"
|
|
else if numberOfDocsInProject is 1 # only one file, must be the main document
|
|
for path, doc of docs
|
|
rootResourcePath = path.replace(/^\//, "") # Remove leading /
|
|
logger.warn {project_id, rootResourcePath: rootResourcePath}, "no root document found, single document in project"
|
|
else
|
|
return callback new Error("no main file specified")
|
|
|
|
for path, file of files
|
|
path = path.replace(/^\//, "") # Remove leading /
|
|
resources.push
|
|
path: path
|
|
url: "#{Settings.apis.filestore.url}/project/#{project._id}/file/#{file._id}"
|
|
modified: file.created?.getTime()
|
|
|
|
callback null, {
|
|
compile:
|
|
options:
|
|
compiler: project.compiler
|
|
timeout: options.timeout
|
|
imageName: project.imageName
|
|
draft: !!options.draft
|
|
check: options.check
|
|
syncType: options.syncType
|
|
syncState: options.syncState
|
|
rootResourcePath: rootResourcePath
|
|
resources: resources
|
|
}
|
|
|
|
wordCount: (project_id, user_id, file, options, callback = (error, response) ->) ->
|
|
ClsiManager._buildRequest project_id, options, (error, req) ->
|
|
filename = file || req?.compile?.rootResourcePath
|
|
wordcount_url = ClsiManager._getCompilerUrl(options?.compileGroup, project_id, user_id, "wordcount")
|
|
opts =
|
|
url: wordcount_url
|
|
qs:
|
|
file: filename
|
|
image: req.compile.options.imageName
|
|
method: "GET"
|
|
ClsiManager._makeRequest project_id, opts, (error, response, body) ->
|
|
return callback(error) if error?
|
|
if 200 <= response.statusCode < 300
|
|
callback null, body
|
|
else
|
|
error = new Error("CLSI returned non-success code: #{response.statusCode}")
|
|
logger.error err: error, project_id: project_id, "CLSI returned failure code"
|
|
callback error, body
|
|
|