mirror of
https://github.com/overleaf/overleaf.git
synced 2025-04-08 21:00:45 +00:00
decaffeinate: Convert CommandRunner.coffee and 25 other files to JS
This commit is contained in:
decaffeinate
mserranom
parent
2d07bab23d
commit
4eab36e3d5
26 changed files with 2801 additions and 1964 deletions
|
|
@ -1,11 +1,18 @@
|
|||
Settings = require "settings-sharelatex"
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let commandRunnerPath;
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
if Settings.clsi?.dockerRunner == true
|
||||
commandRunnerPath = "./DockerRunner"
|
||||
else
|
||||
commandRunnerPath = "./LocalCommandRunner"
|
||||
logger.info commandRunnerPath:commandRunnerPath, "selecting command runner for clsi"
|
||||
CommandRunner = require(commandRunnerPath)
|
||||
if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
|
||||
commandRunnerPath = "./DockerRunner";
|
||||
} else {
|
||||
commandRunnerPath = "./LocalCommandRunner";
|
||||
}
|
||||
logger.info({commandRunnerPath}, "selecting command runner for clsi");
|
||||
const CommandRunner = require(commandRunnerPath);
|
||||
|
||||
module.exports = CommandRunner
|
||||
module.exports = CommandRunner;
|
||||
|
|
|
|||
|
|
@ -1,119 +1,163 @@
|
|||
RequestParser = require "./RequestParser"
|
||||
CompileManager = require "./CompileManager"
|
||||
Settings = require "settings-sharelatex"
|
||||
Metrics = require "./Metrics"
|
||||
ProjectPersistenceManager = require "./ProjectPersistenceManager"
|
||||
logger = require "logger-sharelatex"
|
||||
Errors = require "./Errors"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileController;
|
||||
const RequestParser = require("./RequestParser");
|
||||
const CompileManager = require("./CompileManager");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const ProjectPersistenceManager = require("./ProjectPersistenceManager");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Errors = require("./Errors");
|
||||
|
||||
module.exports = CompileController =
|
||||
compile: (req, res, next = (error) ->) ->
|
||||
timer = new Metrics.Timer("compile-request")
|
||||
RequestParser.parse req.body, (error, request) ->
|
||||
return next(error) if error?
|
||||
request.project_id = req.params.project_id
|
||||
request.user_id = req.params.user_id if req.params.user_id?
|
||||
ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) ->
|
||||
return next(error) if error?
|
||||
CompileManager.doCompileWithLock request, (error, outputFiles = []) ->
|
||||
if error instanceof Errors.AlreadyCompilingError
|
||||
code = 423 # Http 423 Locked
|
||||
status = "compile-in-progress"
|
||||
else if error instanceof Errors.FilesOutOfSyncError
|
||||
code = 409 # Http 409 Conflict
|
||||
status = "retry"
|
||||
else if error?.terminated
|
||||
status = "terminated"
|
||||
else if error?.validate
|
||||
status = "validation-#{error.validate}"
|
||||
else if error?.timedout
|
||||
status = "timedout"
|
||||
logger.log err: error, project_id: request.project_id, "timeout running compile"
|
||||
else if error?
|
||||
status = "error"
|
||||
code = 500
|
||||
logger.warn err: error, project_id: request.project_id, "error running compile"
|
||||
else
|
||||
status = "failure"
|
||||
for file in outputFiles
|
||||
if file.path?.match(/output\.pdf$/)
|
||||
status = "success"
|
||||
module.exports = (CompileController = {
|
||||
compile(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const timer = new Metrics.Timer("compile-request");
|
||||
return RequestParser.parse(req.body, function(error, request) {
|
||||
if (error != null) { return next(error); }
|
||||
request.project_id = req.params.project_id;
|
||||
if (req.params.user_id != null) { request.user_id = req.params.user_id; }
|
||||
return ProjectPersistenceManager.markProjectAsJustAccessed(request.project_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return CompileManager.doCompileWithLock(request, function(error, outputFiles) {
|
||||
let code, status;
|
||||
if (outputFiles == null) { outputFiles = []; }
|
||||
if (error instanceof Errors.AlreadyCompilingError) {
|
||||
code = 423; // Http 423 Locked
|
||||
status = "compile-in-progress";
|
||||
} else if (error instanceof Errors.FilesOutOfSyncError) {
|
||||
code = 409; // Http 409 Conflict
|
||||
status = "retry";
|
||||
} else if (error != null ? error.terminated : undefined) {
|
||||
status = "terminated";
|
||||
} else if (error != null ? error.validate : undefined) {
|
||||
status = `validation-${error.validate}`;
|
||||
} else if (error != null ? error.timedout : undefined) {
|
||||
status = "timedout";
|
||||
logger.log({err: error, project_id: request.project_id}, "timeout running compile");
|
||||
} else if (error != null) {
|
||||
status = "error";
|
||||
code = 500;
|
||||
logger.warn({err: error, project_id: request.project_id}, "error running compile");
|
||||
} else {
|
||||
let file;
|
||||
status = "failure";
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (file.path != null ? file.path.match(/output\.pdf$/) : undefined) {
|
||||
status = "success";
|
||||
}
|
||||
}
|
||||
|
||||
if status == "failure"
|
||||
logger.warn project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
|
||||
if (status === "failure") {
|
||||
logger.warn({project_id: request.project_id, outputFiles}, "project failed to compile successfully, no output.pdf generated");
|
||||
}
|
||||
|
||||
# log an error if any core files are found
|
||||
for file in outputFiles
|
||||
if file.path is "core"
|
||||
logger.error project_id:request.project_id, req:req, outputFiles:outputFiles, "core file found in output"
|
||||
|
||||
if error?
|
||||
outputFiles = error.outputFiles || []
|
||||
|
||||
timer.done()
|
||||
res.status(code or 200).send {
|
||||
compile:
|
||||
status: status
|
||||
error: error?.message or error
|
||||
outputFiles: outputFiles.map (file) ->
|
||||
url:
|
||||
"#{Settings.apis.clsi.url}/project/#{request.project_id}" +
|
||||
(if request.user_id? then "/user/#{request.user_id}" else "") +
|
||||
(if file.build? then "/build/#{file.build}" else "") +
|
||||
"/output/#{file.path}"
|
||||
path: file.path
|
||||
type: file.type
|
||||
build: file.build
|
||||
// log an error if any core files are found
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (file.path === "core") {
|
||||
logger.error({project_id:request.project_id, req, outputFiles}, "core file found in output");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stopCompile: (req, res, next) ->
|
||||
{project_id, user_id} = req.params
|
||||
CompileManager.stopCompile project_id, user_id, (error) ->
|
||||
return next(error) if error?
|
||||
res.sendStatus(204)
|
||||
if (error != null) {
|
||||
outputFiles = error.outputFiles || [];
|
||||
}
|
||||
|
||||
clearCache: (req, res, next = (error) ->) ->
|
||||
ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) ->
|
||||
return next(error) if error?
|
||||
res.sendStatus(204) # No content
|
||||
timer.done();
|
||||
return res.status(code || 200).send({
|
||||
compile: {
|
||||
status,
|
||||
error: (error != null ? error.message : undefined) || error,
|
||||
outputFiles: outputFiles.map(file =>
|
||||
({
|
||||
url:
|
||||
`${Settings.apis.clsi.url}/project/${request.project_id}` +
|
||||
((request.user_id != null) ? `/user/${request.user_id}` : "") +
|
||||
((file.build != null) ? `/build/${file.build}` : "") +
|
||||
`/output/${file.path}`,
|
||||
path: file.path,
|
||||
type: file.type,
|
||||
build: file.build
|
||||
})
|
||||
)
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
syncFromCode: (req, res, next = (error) ->) ->
|
||||
file = req.query.file
|
||||
line = parseInt(req.query.line, 10)
|
||||
column = parseInt(req.query.column, 10)
|
||||
project_id = req.params.project_id
|
||||
user_id = req.params.user_id
|
||||
CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) ->
|
||||
return next(error) if error?
|
||||
res.json {
|
||||
stopCompile(req, res, next) {
|
||||
const {project_id, user_id} = req.params;
|
||||
return CompileManager.stopCompile(project_id, user_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.sendStatus(204);
|
||||
});
|
||||
},
|
||||
|
||||
clearCache(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
return ProjectPersistenceManager.clearProject(req.params.project_id, req.params.user_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.sendStatus(204);
|
||||
});
|
||||
}, // No content
|
||||
|
||||
syncFromCode(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const { file } = req.query;
|
||||
const line = parseInt(req.query.line, 10);
|
||||
const column = parseInt(req.query.column, 10);
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
return CompileManager.syncFromCode(project_id, user_id, file, line, column, function(error, pdfPositions) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
pdf: pdfPositions
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
syncFromPdf: (req, res, next = (error) ->) ->
|
||||
page = parseInt(req.query.page, 10)
|
||||
h = parseFloat(req.query.h)
|
||||
v = parseFloat(req.query.v)
|
||||
project_id = req.params.project_id
|
||||
user_id = req.params.user_id
|
||||
CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) ->
|
||||
return next(error) if error?
|
||||
res.json {
|
||||
syncFromPdf(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const page = parseInt(req.query.page, 10);
|
||||
const h = parseFloat(req.query.h);
|
||||
const v = parseFloat(req.query.v);
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(error, codePositions) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
code: codePositions
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
wordcount: (req, res, next = (error) ->) ->
|
||||
file = req.query.file || "main.tex"
|
||||
project_id = req.params.project_id
|
||||
user_id = req.params.user_id
|
||||
image = req.query.image
|
||||
logger.log {image, file, project_id}, "word count request"
|
||||
wordcount(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const file = req.query.file || "main.tex";
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
const { image } = req.query;
|
||||
logger.log({image, file, project_id}, "word count request");
|
||||
|
||||
CompileManager.wordcount project_id, user_id, file, image, (error, result) ->
|
||||
return next(error) if error?
|
||||
res.json {
|
||||
return CompileManager.wordcount(project_id, user_id, file, image, function(error, result) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
texcount: result
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
status: (req, res, next = (error)-> )->
|
||||
res.send("OK")
|
||||
status(req, res, next ){
|
||||
if (next == null) { next = function(error){}; }
|
||||
return res.send("OK");
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,345 +1,454 @@
|
|||
ResourceWriter = require "./ResourceWriter"
|
||||
LatexRunner = require "./LatexRunner"
|
||||
OutputFileFinder = require "./OutputFileFinder"
|
||||
OutputCacheManager = require "./OutputCacheManager"
|
||||
Settings = require("settings-sharelatex")
|
||||
Path = require "path"
|
||||
logger = require "logger-sharelatex"
|
||||
Metrics = require "./Metrics"
|
||||
child_process = require "child_process"
|
||||
DraftModeManager = require "./DraftModeManager"
|
||||
TikzManager = require "./TikzManager"
|
||||
LockManager = require "./LockManager"
|
||||
fs = require("fs")
|
||||
fse = require "fs-extra"
|
||||
os = require("os")
|
||||
async = require "async"
|
||||
Errors = require './Errors'
|
||||
CommandRunner = require "./CommandRunner"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileManager;
|
||||
const ResourceWriter = require("./ResourceWriter");
|
||||
const LatexRunner = require("./LatexRunner");
|
||||
const OutputFileFinder = require("./OutputFileFinder");
|
||||
const OutputCacheManager = require("./OutputCacheManager");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const Path = require("path");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const child_process = require("child_process");
|
||||
const DraftModeManager = require("./DraftModeManager");
|
||||
const TikzManager = require("./TikzManager");
|
||||
const LockManager = require("./LockManager");
|
||||
const fs = require("fs");
|
||||
const fse = require("fs-extra");
|
||||
const os = require("os");
|
||||
const async = require("async");
|
||||
const Errors = require('./Errors');
|
||||
const CommandRunner = require("./CommandRunner");
|
||||
|
||||
getCompileName = (project_id, user_id) ->
|
||||
if user_id? then "#{project_id}-#{user_id}" else project_id
|
||||
const getCompileName = function(project_id, user_id) {
|
||||
if (user_id != null) { return `${project_id}-${user_id}`; } else { return project_id; }
|
||||
};
|
||||
|
||||
getCompileDir = (project_id, user_id) ->
|
||||
Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
|
||||
const getCompileDir = (project_id, user_id) => Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id));
|
||||
|
||||
module.exports = CompileManager =
|
||||
module.exports = (CompileManager = {
|
||||
|
||||
doCompileWithLock: (request, callback = (error, outputFiles) ->) ->
|
||||
compileDir = getCompileDir(request.project_id, request.user_id)
|
||||
lockFile = Path.join(compileDir, ".project-lock")
|
||||
# use a .project-lock file in the compile directory to prevent
|
||||
# simultaneous compiles
|
||||
fse.ensureDir compileDir, (error) ->
|
||||
return callback(error) if error?
|
||||
LockManager.runWithLock lockFile, (releaseLock) ->
|
||||
CompileManager.doCompile(request, releaseLock)
|
||||
, callback
|
||||
doCompileWithLock(request, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles) {}; }
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id);
|
||||
const lockFile = Path.join(compileDir, ".project-lock");
|
||||
// use a .project-lock file in the compile directory to prevent
|
||||
// simultaneous compiles
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return LockManager.runWithLock(lockFile, releaseLock => CompileManager.doCompile(request, releaseLock)
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
doCompile: (request, callback = (error, outputFiles) ->) ->
|
||||
compileDir = getCompileDir(request.project_id, request.user_id)
|
||||
timer = new Metrics.Timer("write-to-disk")
|
||||
logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
|
||||
ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) ->
|
||||
# NOTE: resourceList is insecure, it should only be used to exclude files from the output list
|
||||
if error? and error instanceof Errors.FilesOutOfSyncError
|
||||
logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry"
|
||||
return callback(error)
|
||||
else if error?
|
||||
logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk"
|
||||
return callback(error)
|
||||
logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk"
|
||||
timer.done()
|
||||
doCompile(request, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles) {}; }
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id);
|
||||
let timer = new Metrics.Timer("write-to-disk");
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "syncing resources to disk");
|
||||
return ResourceWriter.syncResourcesToDisk(request, compileDir, function(error, resourceList) {
|
||||
// NOTE: resourceList is insecure, it should only be used to exclude files from the output list
|
||||
if ((error != null) && error instanceof Errors.FilesOutOfSyncError) {
|
||||
logger.warn({project_id: request.project_id, user_id: request.user_id}, "files out of sync, please retry");
|
||||
return callback(error);
|
||||
} else if (error != null) {
|
||||
logger.err({err:error, project_id: request.project_id, user_id: request.user_id}, "error writing resources to disk");
|
||||
return callback(error);
|
||||
}
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start}, "written files to disk");
|
||||
timer.done();
|
||||
|
||||
injectDraftModeIfRequired = (callback) ->
|
||||
if request.draft
|
||||
DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback
|
||||
else
|
||||
callback()
|
||||
const injectDraftModeIfRequired = function(callback) {
|
||||
if (request.draft) {
|
||||
return DraftModeManager.injectDraftMode(Path.join(compileDir, request.rootResourcePath), callback);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
};
|
||||
|
||||
createTikzFileIfRequired = (callback) ->
|
||||
TikzManager.checkMainFile compileDir, request.rootResourcePath, resourceList, (error, needsMainFile) ->
|
||||
return callback(error) if error?
|
||||
if needsMainFile
|
||||
TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback
|
||||
else
|
||||
callback()
|
||||
const createTikzFileIfRequired = callback =>
|
||||
TikzManager.checkMainFile(compileDir, request.rootResourcePath, resourceList, function(error, needsMainFile) {
|
||||
if (error != null) { return callback(error); }
|
||||
if (needsMainFile) {
|
||||
return TikzManager.injectOutputFile(compileDir, request.rootResourcePath, callback);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
})
|
||||
;
|
||||
|
||||
# set up environment variables for chktex
|
||||
env = {}
|
||||
# only run chktex on LaTeX files (not knitr .Rtex files or any others)
|
||||
isLaTeXFile = request.rootResourcePath?.match(/\.tex$/i)
|
||||
if request.check? and isLaTeXFile
|
||||
env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16'
|
||||
env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000'
|
||||
if request.check is 'error'
|
||||
env['CHKTEX_EXIT_ON_ERROR'] = 1
|
||||
if request.check is 'validate'
|
||||
env['CHKTEX_VALIDATE'] = 1
|
||||
// set up environment variables for chktex
|
||||
const env = {};
|
||||
// only run chktex on LaTeX files (not knitr .Rtex files or any others)
|
||||
const isLaTeXFile = request.rootResourcePath != null ? request.rootResourcePath.match(/\.tex$/i) : undefined;
|
||||
if ((request.check != null) && isLaTeXFile) {
|
||||
env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16';
|
||||
env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000';
|
||||
if (request.check === 'error') {
|
||||
env['CHKTEX_EXIT_ON_ERROR'] = 1;
|
||||
}
|
||||
if (request.check === 'validate') {
|
||||
env['CHKTEX_VALIDATE'] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
# apply a series of file modifications/creations for draft mode and tikz
|
||||
async.series [injectDraftModeIfRequired, createTikzFileIfRequired], (error) ->
|
||||
return callback(error) if error?
|
||||
timer = new Metrics.Timer("run-compile")
|
||||
# find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
|
||||
tag = request.imageName?.match(/:(.*)/)?[1]?.replace(/\./g,'-') or "default"
|
||||
tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test
|
||||
Metrics.inc("compiles")
|
||||
Metrics.inc("compiles-with-image.#{tag}")
|
||||
compileName = getCompileName(request.project_id, request.user_id)
|
||||
LatexRunner.runLatex compileName, {
|
||||
directory: compileDir
|
||||
mainFile: request.rootResourcePath
|
||||
compiler: request.compiler
|
||||
timeout: request.timeout
|
||||
image: request.imageName
|
||||
flags: request.flags
|
||||
// apply a series of file modifications/creations for draft mode and tikz
|
||||
return async.series([injectDraftModeIfRequired, createTikzFileIfRequired], function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
timer = new Metrics.Timer("run-compile");
|
||||
// find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
|
||||
let tag = __guard__(__guard__(request.imageName != null ? request.imageName.match(/:(.*)/) : undefined, x1 => x1[1]), x => x.replace(/\./g,'-')) || "default";
|
||||
if (!request.project_id.match(/^[0-9a-f]{24}$/)) { tag = "other"; } // exclude smoke test
|
||||
Metrics.inc("compiles");
|
||||
Metrics.inc(`compiles-with-image.${tag}`);
|
||||
const compileName = getCompileName(request.project_id, request.user_id);
|
||||
return LatexRunner.runLatex(compileName, {
|
||||
directory: compileDir,
|
||||
mainFile: request.rootResourcePath,
|
||||
compiler: request.compiler,
|
||||
timeout: request.timeout,
|
||||
image: request.imageName,
|
||||
flags: request.flags,
|
||||
environment: env
|
||||
}, (error, output, stats, timings) ->
|
||||
# request was for validation only
|
||||
if request.check is "validate"
|
||||
result = if error?.code then "fail" else "pass"
|
||||
error = new Error("validation")
|
||||
error.validate = result
|
||||
# request was for compile, and failed on validation
|
||||
if request.check is "error" and error?.message is 'exited'
|
||||
error = new Error("compilation")
|
||||
error.validate = "fail"
|
||||
# compile was killed by user, was a validation, or a compile which failed validation
|
||||
if error?.terminated or error?.validate or error?.timedout
|
||||
OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) ->
|
||||
return callback(err) if err?
|
||||
error.outputFiles = outputFiles # return output files so user can check logs
|
||||
callback(error)
|
||||
return
|
||||
# compile completed normally
|
||||
return callback(error) if error?
|
||||
Metrics.inc("compiles-succeeded")
|
||||
for metric_key, metric_value of stats or {}
|
||||
Metrics.count(metric_key, metric_value)
|
||||
for metric_key, metric_value of timings or {}
|
||||
Metrics.timing(metric_key, metric_value)
|
||||
loadavg = os.loadavg?()
|
||||
Metrics.gauge("load-avg", loadavg[0]) if loadavg?
|
||||
ts = timer.done()
|
||||
logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile"
|
||||
if stats?["latex-runs"] > 0
|
||||
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"])
|
||||
if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0
|
||||
Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"])
|
||||
}, function(error, output, stats, timings) {
|
||||
// request was for validation only
|
||||
let metric_key, metric_value;
|
||||
if (request.check === "validate") {
|
||||
const result = (error != null ? error.code : undefined) ? "fail" : "pass";
|
||||
error = new Error("validation");
|
||||
error.validate = result;
|
||||
}
|
||||
// request was for compile, and failed on validation
|
||||
if ((request.check === "error") && ((error != null ? error.message : undefined) === 'exited')) {
|
||||
error = new Error("compilation");
|
||||
error.validate = "fail";
|
||||
}
|
||||
// compile was killed by user, was a validation, or a compile which failed validation
|
||||
if ((error != null ? error.terminated : undefined) || (error != null ? error.validate : undefined) || (error != null ? error.timedout : undefined)) {
|
||||
OutputFileFinder.findOutputFiles(resourceList, compileDir, function(err, outputFiles) {
|
||||
if (err != null) { return callback(err); }
|
||||
error.outputFiles = outputFiles; // return output files so user can check logs
|
||||
return callback(error);
|
||||
});
|
||||
return;
|
||||
}
|
||||
// compile completed normally
|
||||
if (error != null) { return callback(error); }
|
||||
Metrics.inc("compiles-succeeded");
|
||||
const object = stats || {};
|
||||
for (metric_key in object) {
|
||||
metric_value = object[metric_key];
|
||||
Metrics.count(metric_key, metric_value);
|
||||
}
|
||||
const object1 = timings || {};
|
||||
for (metric_key in object1) {
|
||||
metric_value = object1[metric_key];
|
||||
Metrics.timing(metric_key, metric_value);
|
||||
}
|
||||
const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined;
|
||||
if (loadavg != null) { Metrics.gauge("load-avg", loadavg[0]); }
|
||||
const ts = timer.done();
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats, timings, loadavg}, "done compile");
|
||||
if ((stats != null ? stats["latex-runs"] : undefined) > 0) {
|
||||
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]);
|
||||
}
|
||||
if (((stats != null ? stats["latex-runs"] : undefined) > 0) && ((timings != null ? timings["cpu-time"] : undefined) > 0)) {
|
||||
Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]);
|
||||
}
|
||||
|
||||
OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) ->
|
||||
return callback(error) if error?
|
||||
OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) ->
|
||||
callback null, newOutputFiles
|
||||
return OutputFileFinder.findOutputFiles(resourceList, compileDir, function(error, outputFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
return OutputCacheManager.saveOutputFiles(outputFiles, compileDir, (error, newOutputFiles) => callback(null, newOutputFiles));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
stopCompile: (project_id, user_id, callback = (error) ->) ->
|
||||
compileName = getCompileName(project_id, user_id)
|
||||
LatexRunner.killLatex compileName, callback
|
||||
stopCompile(project_id, user_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return LatexRunner.killLatex(compileName, callback);
|
||||
},
|
||||
|
||||
clearProject: (project_id, user_id, _callback = (error) ->) ->
|
||||
callback = (error) ->
|
||||
_callback(error)
|
||||
_callback = () ->
|
||||
clearProject(project_id, user_id, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callback = function(error) {
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
|
||||
CompileManager._checkDirectory compileDir, (err, exists) ->
|
||||
return callback(err) if err?
|
||||
return callback() if not exists # skip removal if no directory present
|
||||
return CompileManager._checkDirectory(compileDir, function(err, exists) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (!exists) { return callback(); } // skip removal if no directory present
|
||||
|
||||
proc = child_process.spawn "rm", ["-r", compileDir]
|
||||
const proc = child_process.spawn("rm", ["-r", compileDir]);
|
||||
|
||||
proc.on "error", callback
|
||||
proc.on("error", callback);
|
||||
|
||||
stderr = ""
|
||||
proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
|
||||
let stderr = "";
|
||||
proc.stderr.on("data", chunk => stderr += chunk.toString());
|
||||
|
||||
proc.on "close", (code) ->
|
||||
if code == 0
|
||||
return callback(null)
|
||||
else
|
||||
return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
|
||||
return proc.on("close", function(code) {
|
||||
if (code === 0) {
|
||||
return callback(null);
|
||||
} else {
|
||||
return callback(new Error(`rm -r ${compileDir} failed: ${stderr}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_findAllDirs: (callback = (error, allDirs) ->) ->
|
||||
root = Settings.path.compilesDir
|
||||
fs.readdir root, (err, files) ->
|
||||
return callback(err) if err?
|
||||
allDirs = (Path.join(root, file) for file in files)
|
||||
callback(null, allDirs)
|
||||
_findAllDirs(callback) {
|
||||
if (callback == null) { callback = function(error, allDirs) {}; }
|
||||
const root = Settings.path.compilesDir;
|
||||
return fs.readdir(root, function(err, files) {
|
||||
if (err != null) { return callback(err); }
|
||||
const allDirs = (Array.from(files).map((file) => Path.join(root, file)));
|
||||
return callback(null, allDirs);
|
||||
});
|
||||
},
|
||||
|
||||
clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) ->
|
||||
now = Date.now()
|
||||
# action for each directory
|
||||
expireIfNeeded = (checkDir, cb) ->
|
||||
fs.stat checkDir, (err, stats) ->
|
||||
return cb() if err? # ignore errors checking directory
|
||||
age = now - stats.mtime
|
||||
hasExpired = (age > max_cache_age_ms)
|
||||
if hasExpired then fse.remove(checkDir, cb) else cb()
|
||||
# iterate over all project directories
|
||||
CompileManager._findAllDirs (error, allDirs) ->
|
||||
return callback() if error?
|
||||
async.eachSeries allDirs, expireIfNeeded, callback
|
||||
clearExpiredProjects(max_cache_age_ms, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const now = Date.now();
|
||||
// action for each directory
|
||||
const expireIfNeeded = (checkDir, cb) =>
|
||||
fs.stat(checkDir, function(err, stats) {
|
||||
if (err != null) { return cb(); } // ignore errors checking directory
|
||||
const age = now - stats.mtime;
|
||||
const hasExpired = (age > max_cache_age_ms);
|
||||
if (hasExpired) { return fse.remove(checkDir, cb); } else { return cb(); }
|
||||
})
|
||||
;
|
||||
// iterate over all project directories
|
||||
return CompileManager._findAllDirs(function(error, allDirs) {
|
||||
if (error != null) { return callback(); }
|
||||
return async.eachSeries(allDirs, expireIfNeeded, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_checkDirectory: (compileDir, callback = (error, exists) ->) ->
|
||||
fs.lstat compileDir, (err, stats) ->
|
||||
if err?.code is 'ENOENT'
|
||||
return callback(null, false) # directory does not exist
|
||||
else if err?
|
||||
logger.err {dir: compileDir, err:err}, "error on stat of project directory for removal"
|
||||
return callback(err)
|
||||
else if not stats?.isDirectory()
|
||||
logger.err {dir: compileDir, stats:stats}, "bad project directory for removal"
|
||||
return callback new Error("project directory is not directory")
|
||||
else
|
||||
callback(null, true) # directory exists
|
||||
_checkDirectory(compileDir, callback) {
|
||||
if (callback == null) { callback = function(error, exists) {}; }
|
||||
return fs.lstat(compileDir, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
return callback(null, false); // directory does not exist
|
||||
} else if (err != null) {
|
||||
logger.err({dir: compileDir, err}, "error on stat of project directory for removal");
|
||||
return callback(err);
|
||||
} else if (!(stats != null ? stats.isDirectory() : undefined)) {
|
||||
logger.err({dir: compileDir, stats}, "bad project directory for removal");
|
||||
return callback(new Error("project directory is not directory"));
|
||||
} else {
|
||||
return callback(null, true);
|
||||
}
|
||||
});
|
||||
}, // directory exists
|
||||
|
||||
syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
|
||||
# If LaTeX was run in a virtual environment, the file path that synctex expects
|
||||
# might not match the file path on the host. The .synctex.gz file however, will be accessed
|
||||
# wherever it is on the host.
|
||||
compileName = getCompileName(project_id, user_id)
|
||||
base_dir = Settings.path.synctexBaseDir(compileName)
|
||||
file_path = base_dir + "/" + file_name
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
synctex_path = "#{base_dir}/output.pdf"
|
||||
command = ["code", synctex_path, file_path, line, column]
|
||||
fse.ensureDir compileDir, (error) ->
|
||||
if error?
|
||||
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
|
||||
return callback(error)
|
||||
CompileManager._runSynctex project_id, user_id, command, (error, stdout) ->
|
||||
return callback(error) if error?
|
||||
logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
|
||||
callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
|
||||
syncFromCode(project_id, user_id, file_name, line, column, callback) {
|
||||
// If LaTeX was run in a virtual environment, the file path that synctex expects
|
||||
// might not match the file path on the host. The .synctex.gz file however, will be accessed
|
||||
// wherever it is on the host.
|
||||
if (callback == null) { callback = function(error, pdfPositions) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName);
|
||||
const file_path = base_dir + "/" + file_name;
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const synctex_path = `${base_dir}/output.pdf`;
|
||||
const command = ["code", synctex_path, file_path, line, column];
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
|
||||
return callback(error);
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_id, user_id, file_name, line, column, command, stdout}, "synctex code output");
|
||||
return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout));
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
|
||||
compileName = getCompileName(project_id, user_id)
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
base_dir = Settings.path.synctexBaseDir(compileName)
|
||||
synctex_path = "#{base_dir}/output.pdf"
|
||||
command = ["pdf", synctex_path, page, h, v]
|
||||
fse.ensureDir compileDir, (error) ->
|
||||
if error?
|
||||
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync to code"
|
||||
return callback(error)
|
||||
CompileManager._runSynctex project_id, user_id, command, (error, stdout) ->
|
||||
return callback(error) if error?
|
||||
logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
|
||||
callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
|
||||
syncFromPdf(project_id, user_id, page, h, v, callback) {
|
||||
if (callback == null) { callback = function(error, filePositions) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName);
|
||||
const synctex_path = `${base_dir}/output.pdf`;
|
||||
const command = ["pdf", synctex_path, page, h, v];
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync to code");
|
||||
return callback(error);
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_id, user_id, page, h, v, stdout}, "synctex pdf output");
|
||||
return callback(null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir));
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_checkFileExists: (path, callback = (error) ->) ->
|
||||
synctexDir = Path.dirname(path)
|
||||
synctexFile = Path.join(synctexDir, "output.synctex.gz")
|
||||
fs.stat synctexDir, (error, stats) ->
|
||||
if error?.code is 'ENOENT'
|
||||
return callback(new Errors.NotFoundError("called synctex with no output directory"))
|
||||
return callback(error) if error?
|
||||
fs.stat synctexFile, (error, stats) ->
|
||||
if error?.code is 'ENOENT'
|
||||
return callback(new Errors.NotFoundError("called synctex with no output file"))
|
||||
return callback(error) if error?
|
||||
return callback(new Error("not a file")) if not stats?.isFile()
|
||||
callback()
|
||||
_checkFileExists(path, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const synctexDir = Path.dirname(path);
|
||||
const synctexFile = Path.join(synctexDir, "output.synctex.gz");
|
||||
return fs.stat(synctexDir, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(new Errors.NotFoundError("called synctex with no output directory"));
|
||||
}
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.stat(synctexFile, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(new Errors.NotFoundError("called synctex with no output file"));
|
||||
}
|
||||
if (error != null) { return callback(error); }
|
||||
if (!(stats != null ? stats.isFile() : undefined)) { return callback(new Error("not a file")); }
|
||||
return callback();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_runSynctex: (project_id, user_id, command, callback = (error, stdout) ->) ->
|
||||
seconds = 1000
|
||||
_runSynctex(project_id, user_id, command, callback) {
|
||||
if (callback == null) { callback = function(error, stdout) {}; }
|
||||
const seconds = 1000;
|
||||
|
||||
command.unshift("/opt/synctex")
|
||||
command.unshift("/opt/synctex");
|
||||
|
||||
directory = getCompileDir(project_id, user_id)
|
||||
timeout = 60 * 1000 # increased to allow for large projects
|
||||
compileName = getCompileName(project_id, user_id)
|
||||
CommandRunner.run compileName, command, directory, Settings.clsi?.docker.image, timeout, {}, (error, output) ->
|
||||
if error?
|
||||
logger.err err:error, command:command, project_id:project_id, user_id:user_id, "error running synctex"
|
||||
return callback(error)
|
||||
callback(null, output.stdout)
|
||||
const directory = getCompileDir(project_id, user_id);
|
||||
const timeout = 60 * 1000; // increased to allow for large projects
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return CommandRunner.run(compileName, command, directory, Settings.clsi != null ? Settings.clsi.docker.image : undefined, timeout, {}, function(error, output) {
|
||||
if (error != null) {
|
||||
logger.err({err:error, command, project_id, user_id}, "error running synctex");
|
||||
return callback(error);
|
||||
}
|
||||
return callback(null, output.stdout);
|
||||
});
|
||||
},
|
||||
|
||||
_parseSynctexFromCodeOutput: (output) ->
|
||||
results = []
|
||||
for line in output.split("\n")
|
||||
[node, page, h, v, width, height] = line.split("\t")
|
||||
if node == "NODE"
|
||||
results.push {
|
||||
page: parseInt(page, 10)
|
||||
h: parseFloat(h)
|
||||
v: parseFloat(v)
|
||||
height: parseFloat(height)
|
||||
_parseSynctexFromCodeOutput(output) {
|
||||
const results = [];
|
||||
for (let line of Array.from(output.split("\n"))) {
|
||||
const [node, page, h, v, width, height] = Array.from(line.split("\t"));
|
||||
if (node === "NODE") {
|
||||
results.push({
|
||||
page: parseInt(page, 10),
|
||||
h: parseFloat(h),
|
||||
v: parseFloat(v),
|
||||
height: parseFloat(height),
|
||||
width: parseFloat(width)
|
||||
}
|
||||
return results
|
||||
|
||||
_parseSynctexFromPdfOutput: (output, base_dir) ->
|
||||
results = []
|
||||
for line in output.split("\n")
|
||||
[node, file_path, line, column] = line.split("\t")
|
||||
if node == "NODE"
|
||||
file = file_path.slice(base_dir.length + 1)
|
||||
results.push {
|
||||
file: file
|
||||
line: parseInt(line, 10)
|
||||
column: parseInt(column, 10)
|
||||
}
|
||||
return results
|
||||
|
||||
|
||||
wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) ->
|
||||
logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount"
|
||||
file_path = "$COMPILE_DIR/" + file_name
|
||||
command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"]
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
timeout = 60 * 1000
|
||||
compileName = getCompileName(project_id, user_id)
|
||||
fse.ensureDir compileDir, (error) ->
|
||||
if error?
|
||||
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
|
||||
return callback(error)
|
||||
CommandRunner.run compileName, command, compileDir, image, timeout, {}, (error) ->
|
||||
return callback(error) if error?
|
||||
fs.readFile compileDir + "/" + file_name + ".wc", "utf-8", (err, stdout) ->
|
||||
if err?
|
||||
#call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
|
||||
logger.err node_err:err, command:command, compileDir:compileDir, project_id:project_id, user_id:user_id, "error reading word count output"
|
||||
return callback(err)
|
||||
results = CompileManager._parseWordcountFromOutput(stdout)
|
||||
logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results"
|
||||
callback null, results
|
||||
|
||||
_parseWordcountFromOutput: (output) ->
|
||||
results = {
|
||||
encode: ""
|
||||
textWords: 0
|
||||
headWords: 0
|
||||
outside: 0
|
||||
headers: 0
|
||||
elements: 0
|
||||
mathInline: 0
|
||||
mathDisplay: 0
|
||||
errors: 0
|
||||
messages: ""
|
||||
});
|
||||
}
|
||||
}
|
||||
for line in output.split("\n")
|
||||
[data, info] = line.split(":")
|
||||
if data.indexOf("Encoding") > -1
|
||||
results['encode'] = info.trim()
|
||||
if data.indexOf("in text") > -1
|
||||
results['textWords'] = parseInt(info, 10)
|
||||
if data.indexOf("in head") > -1
|
||||
results['headWords'] = parseInt(info, 10)
|
||||
if data.indexOf("outside") > -1
|
||||
results['outside'] = parseInt(info, 10)
|
||||
if data.indexOf("of head") > -1
|
||||
results['headers'] = parseInt(info, 10)
|
||||
if data.indexOf("Number of floats/tables/figures") > -1
|
||||
results['elements'] = parseInt(info, 10)
|
||||
if data.indexOf("Number of math inlines") > -1
|
||||
results['mathInline'] = parseInt(info, 10)
|
||||
if data.indexOf("Number of math displayed") > -1
|
||||
results['mathDisplay'] = parseInt(info, 10)
|
||||
if data is "(errors" # errors reported as (errors:123)
|
||||
results['errors'] = parseInt(info, 10)
|
||||
if line.indexOf("!!! ") > -1 # errors logged as !!! message !!!
|
||||
results['messages'] += line + "\n"
|
||||
return results
|
||||
return results;
|
||||
},
|
||||
|
||||
_parseSynctexFromPdfOutput(output, base_dir) {
|
||||
const results = [];
|
||||
for (let line of Array.from(output.split("\n"))) {
|
||||
let column, file_path, node;
|
||||
[node, file_path, line, column] = Array.from(line.split("\t"));
|
||||
if (node === "NODE") {
|
||||
const file = file_path.slice(base_dir.length + 1);
|
||||
results.push({
|
||||
file,
|
||||
line: parseInt(line, 10),
|
||||
column: parseInt(column, 10)
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
},
|
||||
|
||||
|
||||
wordcount(project_id, user_id, file_name, image, callback) {
|
||||
if (callback == null) { callback = function(error, pdfPositions) {}; }
|
||||
logger.log({project_id, user_id, file_name, image}, "running wordcount");
|
||||
const file_path = `$COMPILE_DIR/${file_name}`;
|
||||
const command = [ "texcount", '-nocol', '-inc', file_path, `-out=${file_path}.wc`];
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const timeout = 60 * 1000;
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
|
||||
return callback(error);
|
||||
}
|
||||
return CommandRunner.run(compileName, command, compileDir, image, timeout, {}, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.readFile(compileDir + "/" + file_name + ".wc", "utf-8", function(err, stdout) {
|
||||
if (err != null) {
|
||||
//call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
|
||||
logger.err({node_err:err, command, compileDir, project_id, user_id}, "error reading word count output");
|
||||
return callback(err);
|
||||
}
|
||||
const results = CompileManager._parseWordcountFromOutput(stdout);
|
||||
logger.log({project_id, user_id, wordcount: results}, "word count results");
|
||||
return callback(null, results);
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_parseWordcountFromOutput(output) {
|
||||
const results = {
|
||||
encode: "",
|
||||
textWords: 0,
|
||||
headWords: 0,
|
||||
outside: 0,
|
||||
headers: 0,
|
||||
elements: 0,
|
||||
mathInline: 0,
|
||||
mathDisplay: 0,
|
||||
errors: 0,
|
||||
messages: ""
|
||||
};
|
||||
for (let line of Array.from(output.split("\n"))) {
|
||||
const [data, info] = Array.from(line.split(":"));
|
||||
if (data.indexOf("Encoding") > -1) {
|
||||
results['encode'] = info.trim();
|
||||
}
|
||||
if (data.indexOf("in text") > -1) {
|
||||
results['textWords'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("in head") > -1) {
|
||||
results['headWords'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("outside") > -1) {
|
||||
results['outside'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("of head") > -1) {
|
||||
results['headers'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of floats/tables/figures") > -1) {
|
||||
results['elements'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of math inlines") > -1) {
|
||||
results['mathInline'] = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of math displayed") > -1) {
|
||||
results['mathDisplay'] = parseInt(info, 10);
|
||||
}
|
||||
if (data === "(errors") { // errors reported as (errors:123)
|
||||
results['errors'] = parseInt(info, 10);
|
||||
}
|
||||
if (line.indexOf("!!! ") > -1) { // errors logged as !!! message !!!
|
||||
results['messages'] += line + "\n";
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,24 +1,28 @@
|
|||
Path = require 'path'
|
||||
let ContentTypeMapper;
|
||||
const Path = require('path');
|
||||
|
||||
# here we coerce html, css and js to text/plain,
|
||||
# otherwise choose correct mime type based on file extension,
|
||||
# falling back to octet-stream
|
||||
module.exports = ContentTypeMapper =
|
||||
map: (path) ->
|
||||
switch Path.extname(path)
|
||||
when '.txt', '.html', '.js', '.css', '.svg'
|
||||
return 'text/plain'
|
||||
when '.csv'
|
||||
return 'text/csv'
|
||||
when '.pdf'
|
||||
return 'application/pdf'
|
||||
when '.png'
|
||||
return 'image/png'
|
||||
when '.jpg', '.jpeg'
|
||||
return 'image/jpeg'
|
||||
when '.tiff'
|
||||
return 'image/tiff'
|
||||
when '.gif'
|
||||
return 'image/gif'
|
||||
else
|
||||
return 'application/octet-stream'
|
||||
// here we coerce html, css and js to text/plain,
|
||||
// otherwise choose correct mime type based on file extension,
|
||||
// falling back to octet-stream
|
||||
module.exports = (ContentTypeMapper = {
|
||||
map(path) {
|
||||
switch (Path.extname(path)) {
|
||||
case '.txt': case '.html': case '.js': case '.css': case '.svg':
|
||||
return 'text/plain';
|
||||
case '.csv':
|
||||
return 'text/csv';
|
||||
case '.pdf':
|
||||
return 'application/pdf';
|
||||
case '.png':
|
||||
return 'image/png';
|
||||
case '.jpg': case '.jpeg':
|
||||
return 'image/jpeg';
|
||||
case '.tiff':
|
||||
return 'image/tiff';
|
||||
case '.gif':
|
||||
return 'image/gif';
|
||||
default:
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
async = require "async"
|
||||
Settings = require "settings-sharelatex"
|
||||
logger = require("logger-sharelatex")
|
||||
queue = async.queue((task, cb)->
|
||||
task(cb)
|
||||
, Settings.parallelSqlQueryLimit)
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const async = require("async");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const queue = async.queue((task, cb)=> task(cb)
|
||||
, Settings.parallelSqlQueryLimit);
|
||||
|
||||
queue.drain = ()->
|
||||
logger.debug('all items have been processed')
|
||||
queue.drain = ()=> logger.debug('all items have been processed');
|
||||
|
||||
module.exports =
|
||||
queue: queue
|
||||
{queue};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,56 +1,84 @@
|
|||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager;
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
LockState = {} # locks for docker container operations, by container name
|
||||
const LockState = {}; // locks for docker container operations, by container name
|
||||
|
||||
module.exports = LockManager =
|
||||
module.exports = (LockManager = {
|
||||
|
||||
MAX_LOCK_HOLD_TIME: 15000 # how long we can keep a lock
|
||||
MAX_LOCK_WAIT_TIME: 10000 # how long we wait for a lock
|
||||
LOCK_TEST_INTERVAL: 1000 # retry time
|
||||
MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
|
||||
MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
|
||||
LOCK_TEST_INTERVAL: 1000, // retry time
|
||||
|
||||
tryLock: (key, callback = (err, gotLock) ->) ->
|
||||
existingLock = LockState[key]
|
||||
if existingLock? # the lock is already taken, check how old it is
|
||||
lockAge = Date.now() - existingLock.created
|
||||
if lockAge < LockManager.MAX_LOCK_HOLD_TIME
|
||||
return callback(null, false) # we didn't get the lock, bail out
|
||||
else
|
||||
logger.error {key: key, lock: existingLock, age:lockAge}, "taking old lock by force"
|
||||
# take the lock
|
||||
LockState[key] = lockValue = {created: Date.now()}
|
||||
callback(null, true, lockValue)
|
||||
tryLock(key, callback) {
|
||||
let lockValue;
|
||||
if (callback == null) { callback = function(err, gotLock) {}; }
|
||||
const existingLock = LockState[key];
|
||||
if (existingLock != null) { // the lock is already taken, check how old it is
|
||||
const lockAge = Date.now() - existingLock.created;
|
||||
if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
|
||||
return callback(null, false); // we didn't get the lock, bail out
|
||||
} else {
|
||||
logger.error({key, lock: existingLock, age:lockAge}, "taking old lock by force");
|
||||
}
|
||||
}
|
||||
// take the lock
|
||||
LockState[key] = (lockValue = {created: Date.now()});
|
||||
return callback(null, true, lockValue);
|
||||
},
|
||||
|
||||
getLock: (key, callback = (error, lockValue) ->) ->
|
||||
startTime = Date.now()
|
||||
do attempt = () ->
|
||||
LockManager.tryLock key, (error, gotLock, lockValue) ->
|
||||
return callback(error) if error?
|
||||
if gotLock
|
||||
callback(null, lockValue)
|
||||
else if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
|
||||
e = new Error("Lock timeout")
|
||||
e.key = key
|
||||
return callback(e)
|
||||
else
|
||||
setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
|
||||
getLock(key, callback) {
|
||||
let attempt;
|
||||
if (callback == null) { callback = function(error, lockValue) {}; }
|
||||
const startTime = Date.now();
|
||||
return (attempt = () =>
|
||||
LockManager.tryLock(key, function(error, gotLock, lockValue) {
|
||||
if (error != null) { return callback(error); }
|
||||
if (gotLock) {
|
||||
return callback(null, lockValue);
|
||||
} else if ((Date.now() - startTime) > LockManager.MAX_LOCK_WAIT_TIME) {
|
||||
const e = new Error("Lock timeout");
|
||||
e.key = key;
|
||||
return callback(e);
|
||||
} else {
|
||||
return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL);
|
||||
}
|
||||
})
|
||||
)();
|
||||
},
|
||||
|
||||
releaseLock: (key, lockValue, callback = (error) ->) ->
|
||||
existingLock = LockState[key]
|
||||
if existingLock is lockValue # lockValue is an object, so we can test by reference
|
||||
delete LockState[key] # our lock, so we can free it
|
||||
callback()
|
||||
else if existingLock? # lock exists but doesn't match ours
|
||||
logger.error {key:key, lock: existingLock}, "tried to release lock taken by force"
|
||||
callback()
|
||||
else
|
||||
logger.error {key:key, lock: existingLock}, "tried to release lock that has gone"
|
||||
callback()
|
||||
releaseLock(key, lockValue, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const existingLock = LockState[key];
|
||||
if (existingLock === lockValue) { // lockValue is an object, so we can test by reference
|
||||
delete LockState[key]; // our lock, so we can free it
|
||||
return callback();
|
||||
} else if (existingLock != null) { // lock exists but doesn't match ours
|
||||
logger.error({key, lock: existingLock}, "tried to release lock taken by force");
|
||||
return callback();
|
||||
} else {
|
||||
logger.error({key, lock: existingLock}, "tried to release lock that has gone");
|
||||
return callback();
|
||||
}
|
||||
},
|
||||
|
||||
runWithLock: (key, runner, callback = ( (error) -> )) ->
|
||||
LockManager.getLock key, (error, lockValue) ->
|
||||
return callback(error) if error?
|
||||
runner (error1, args...) ->
|
||||
LockManager.releaseLock key, lockValue, (error2) ->
|
||||
error = error1 or error2
|
||||
return callback(error) if error?
|
||||
callback(null, args...)
|
||||
runWithLock(key, runner, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return LockManager.getLock(key, function(error, lockValue) {
|
||||
if (error != null) { return callback(error); }
|
||||
return runner((error1, ...args) =>
|
||||
LockManager.releaseLock(key, lockValue, function(error2) {
|
||||
error = error1 || error2;
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, ...Array.from(args));
|
||||
})
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,358 +1,475 @@
|
|||
Settings = require "settings-sharelatex"
|
||||
logger = require "logger-sharelatex"
|
||||
Docker = require("dockerode")
|
||||
dockerode = new Docker()
|
||||
crypto = require "crypto"
|
||||
async = require "async"
|
||||
LockManager = require "./DockerLockManager"
|
||||
fs = require "fs"
|
||||
Path = require 'path'
|
||||
_ = require "underscore"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DockerRunner, oneHour;
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Docker = require("dockerode");
|
||||
const dockerode = new Docker();
|
||||
const crypto = require("crypto");
|
||||
const async = require("async");
|
||||
const LockManager = require("./DockerLockManager");
|
||||
const fs = require("fs");
|
||||
const Path = require('path');
|
||||
const _ = require("underscore");
|
||||
|
||||
logger.info "using docker runner"
|
||||
logger.info("using docker runner");
|
||||
|
||||
usingSiblingContainers = () ->
|
||||
Settings?.path?.sandboxedCompilesHostDir?
|
||||
const usingSiblingContainers = () => __guard__(Settings != null ? Settings.path : undefined, x => x.sandboxedCompilesHostDir) != null;
|
||||
|
||||
module.exports = DockerRunner =
|
||||
ERR_NOT_DIRECTORY: new Error("not a directory")
|
||||
ERR_TERMINATED: new Error("terminated")
|
||||
ERR_EXITED: new Error("exited")
|
||||
ERR_TIMED_OUT: new Error("container timed out")
|
||||
module.exports = (DockerRunner = {
|
||||
ERR_NOT_DIRECTORY: new Error("not a directory"),
|
||||
ERR_TERMINATED: new Error("terminated"),
|
||||
ERR_EXITED: new Error("exited"),
|
||||
ERR_TIMED_OUT: new Error("container timed out"),
|
||||
|
||||
run: (project_id, command, directory, image, timeout, environment, callback = (error, output) ->) ->
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
|
||||
if usingSiblingContainers()
|
||||
_newPath = Settings.path.sandboxedCompilesHostDir
|
||||
logger.log {path: _newPath}, "altering bind path for sibling containers"
|
||||
# Server Pro, example:
|
||||
# '/var/lib/sharelatex/data/compiles/<project-id>'
|
||||
# ... becomes ...
|
||||
# '/opt/sharelatex_data/data/compiles/<project-id>'
|
||||
directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory))
|
||||
let name;
|
||||
if (callback == null) { callback = function(error, output) {}; }
|
||||
if (usingSiblingContainers()) {
|
||||
const _newPath = Settings.path.sandboxedCompilesHostDir;
|
||||
logger.log({path: _newPath}, "altering bind path for sibling containers");
|
||||
// Server Pro, example:
|
||||
// '/var/lib/sharelatex/data/compiles/<project-id>'
|
||||
// ... becomes ...
|
||||
// '/opt/sharelatex_data/data/compiles/<project-id>'
|
||||
directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory));
|
||||
}
|
||||
|
||||
volumes = {}
|
||||
volumes[directory] = "/compile"
|
||||
const volumes = {};
|
||||
volumes[directory] = "/compile";
|
||||
|
||||
command = (arg.toString().replace?('$COMPILE_DIR', "/compile") for arg in command)
|
||||
if !image?
|
||||
image = Settings.clsi.docker.image
|
||||
command = (Array.from(command).map((arg) => __guardMethod__(arg.toString(), 'replace', o => o.replace('$COMPILE_DIR', "/compile"))));
|
||||
if ((image == null)) {
|
||||
({ image } = Settings.clsi.docker);
|
||||
}
|
||||
|
||||
if Settings.texliveImageNameOveride?
|
||||
img = image.split("/")
|
||||
image = "#{Settings.texliveImageNameOveride}/#{img[2]}"
|
||||
if (Settings.texliveImageNameOveride != null) {
|
||||
const img = image.split("/");
|
||||
image = `${Settings.texliveImageNameOveride}/${img[2]}`;
|
||||
}
|
||||
|
||||
options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment)
|
||||
fingerprint = DockerRunner._fingerprintContainer(options)
|
||||
options.name = name = "project-#{project_id}-#{fingerprint}"
|
||||
const options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment);
|
||||
const fingerprint = DockerRunner._fingerprintContainer(options);
|
||||
options.name = (name = `project-${project_id}-${fingerprint}`);
|
||||
|
||||
# logOptions = _.clone(options)
|
||||
# logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
|
||||
logger.log project_id: project_id, "running docker container"
|
||||
DockerRunner._runAndWaitForContainer options, volumes, timeout, (error, output) ->
|
||||
if error?.message?.match("HTTP code is 500")
|
||||
logger.log err: error, project_id: project_id, "error running container so destroying and retrying"
|
||||
DockerRunner.destroyContainer name, null, true, (error) ->
|
||||
return callback(error) if error?
|
||||
DockerRunner._runAndWaitForContainer options, volumes, timeout, callback
|
||||
else
|
||||
callback(error, output)
|
||||
// logOptions = _.clone(options)
|
||||
// logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
|
||||
logger.log({project_id}, "running docker container");
|
||||
DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(error, output) {
|
||||
if (__guard__(error != null ? error.message : undefined, x => x.match("HTTP code is 500"))) {
|
||||
logger.log({err: error, project_id}, "error running container so destroying and retrying");
|
||||
return DockerRunner.destroyContainer(name, null, true, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return DockerRunner._runAndWaitForContainer(options, volumes, timeout, callback);
|
||||
});
|
||||
} else {
|
||||
return callback(error, output);
|
||||
}
|
||||
});
|
||||
|
||||
return name # pass back the container name to allow it to be killed
|
||||
return name;
|
||||
}, // pass back the container name to allow it to be killed
|
||||
|
||||
kill: (container_id, callback = (error) ->) ->
|
||||
logger.log container_id: container_id, "sending kill signal to container"
|
||||
container = dockerode.getContainer(container_id)
|
||||
container.kill (error) ->
|
||||
if error? and error?.message?.match?(/Cannot kill container .* is not running/)
|
||||
logger.warn err: error, container_id: container_id, "container not running, continuing"
|
||||
error = null
|
||||
if error?
|
||||
logger.error err: error, container_id: container_id, "error killing container"
|
||||
return callback(error)
|
||||
else
|
||||
callback()
|
||||
kill(container_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({container_id}, "sending kill signal to container");
|
||||
const container = dockerode.getContainer(container_id);
|
||||
return container.kill(function(error) {
|
||||
if ((error != null) && __guardMethod__(error != null ? error.message : undefined, 'match', o => o.match(/Cannot kill container .* is not running/))) {
|
||||
logger.warn({err: error, container_id}, "container not running, continuing");
|
||||
error = null;
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id}, "error killing container");
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_runAndWaitForContainer: (options, volumes, timeout, _callback = (error, output) ->) ->
|
||||
callback = (args...) ->
|
||||
_callback(args...)
|
||||
# Only call the callback once
|
||||
_callback = () ->
|
||||
_runAndWaitForContainer(options, volumes, timeout, _callback) {
|
||||
if (_callback == null) { _callback = function(error, output) {}; }
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []));
|
||||
// Only call the callback once
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
name = options.name
|
||||
const { name } = options;
|
||||
|
||||
streamEnded = false
|
||||
containerReturned = false
|
||||
output = {}
|
||||
let streamEnded = false;
|
||||
let containerReturned = false;
|
||||
let output = {};
|
||||
|
||||
callbackIfFinished = () ->
|
||||
if streamEnded and containerReturned
|
||||
callback(null, output)
|
||||
const callbackIfFinished = function() {
|
||||
if (streamEnded && containerReturned) {
|
||||
return callback(null, output);
|
||||
}
|
||||
};
|
||||
|
||||
attachStreamHandler = (error, _output) ->
|
||||
return callback(error) if error?
|
||||
output = _output
|
||||
streamEnded = true
|
||||
callbackIfFinished()
|
||||
const attachStreamHandler = function(error, _output) {
|
||||
if (error != null) { return callback(error); }
|
||||
output = _output;
|
||||
streamEnded = true;
|
||||
return callbackIfFinished();
|
||||
};
|
||||
|
||||
DockerRunner.startContainer options, volumes, attachStreamHandler, (error, containerId) ->
|
||||
return callback(error) if error?
|
||||
return DockerRunner.startContainer(options, volumes, attachStreamHandler, function(error, containerId) {
|
||||
if (error != null) { return callback(error); }
|
||||
|
||||
DockerRunner.waitForContainer name, timeout, (error, exitCode) ->
|
||||
return callback(error) if error?
|
||||
if exitCode is 137 # exit status from kill -9
|
||||
err = DockerRunner.ERR_TERMINATED
|
||||
err.terminated = true
|
||||
return callback(err)
|
||||
if exitCode is 1 # exit status from chktex
|
||||
err = DockerRunner.ERR_EXITED
|
||||
err.code = exitCode
|
||||
return callback(err)
|
||||
containerReturned = true
|
||||
options?.HostConfig?.SecurityOpt = null #small log line
|
||||
logger.log err:err, exitCode:exitCode, options:options, "docker container has exited"
|
||||
callbackIfFinished()
|
||||
|
||||
_getContainerOptions: (command, image, volumes, timeout, environment) ->
|
||||
timeoutInSeconds = timeout / 1000
|
||||
|
||||
dockerVolumes = {}
|
||||
for hostVol, dockerVol of volumes
|
||||
dockerVolumes[dockerVol] = {}
|
||||
|
||||
if volumes[hostVol].slice(-3).indexOf(":r") == -1
|
||||
volumes[hostVol] = "#{dockerVol}:rw"
|
||||
|
||||
# merge settings and environment parameter
|
||||
env = {}
|
||||
for src in [Settings.clsi.docker.env, environment or {}]
|
||||
env[key] = value for key, value of src
|
||||
# set the path based on the image year
|
||||
if m = image.match /:([0-9]+)\.[0-9]+/
|
||||
year = m[1]
|
||||
else
|
||||
year = "2014"
|
||||
env['PATH'] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/#{year}/bin/x86_64-linux/"
|
||||
options =
|
||||
"Cmd" : command,
|
||||
"Image" : image
|
||||
"Volumes" : dockerVolumes
|
||||
"WorkingDir" : "/compile"
|
||||
"NetworkDisabled" : true
|
||||
"Memory" : 1024 * 1024 * 1024 * 1024 # 1 Gb
|
||||
"User" : Settings.clsi.docker.user
|
||||
"Env" : ("#{key}=#{value}" for key, value of env) # convert the environment hash to an array
|
||||
"HostConfig" :
|
||||
"Binds": ("#{hostVol}:#{dockerVol}" for hostVol, dockerVol of volumes)
|
||||
"LogConfig": {"Type": "none", "Config": {}}
|
||||
"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}]
|
||||
"CapDrop": "ALL"
|
||||
"SecurityOpt": ["no-new-privileges"]
|
||||
|
||||
|
||||
if Settings.path?.synctexBinHostPath?
|
||||
options["HostConfig"]["Binds"].push("#{Settings.path.synctexBinHostPath}:/opt/synctex:ro")
|
||||
|
||||
if Settings.clsi.docker.seccomp_profile?
|
||||
options.HostConfig.SecurityOpt.push "seccomp=#{Settings.clsi.docker.seccomp_profile}"
|
||||
|
||||
return options
|
||||
|
||||
_fingerprintContainer: (containerOptions) ->
|
||||
# Yay, Hashing!
|
||||
json = JSON.stringify(containerOptions)
|
||||
return crypto.createHash("md5").update(json).digest("hex")
|
||||
|
||||
startContainer: (options, volumes, attachStreamHandler, callback) ->
|
||||
LockManager.runWithLock options.name, (releaseLock) ->
|
||||
# Check that volumes exist before starting the container.
|
||||
# When a container is started with volume pointing to a
|
||||
# non-existent directory then docker creates the directory but
|
||||
# with root ownership.
|
||||
DockerRunner._checkVolumes options, volumes, (err) ->
|
||||
return releaseLock(err) if err?
|
||||
DockerRunner._startContainer options, volumes, attachStreamHandler, releaseLock
|
||||
, callback
|
||||
|
||||
# Check that volumes exist and are directories
|
||||
_checkVolumes: (options, volumes, callback = (error, containerName) ->) ->
|
||||
if usingSiblingContainers()
|
||||
# Server Pro, with sibling-containers active, skip checks
|
||||
return callback(null)
|
||||
|
||||
checkVolume = (path, cb) ->
|
||||
fs.stat path, (err, stats) ->
|
||||
return cb(err) if err?
|
||||
return cb(DockerRunner.ERR_NOT_DIRECTORY) if not stats?.isDirectory()
|
||||
cb()
|
||||
jobs = []
|
||||
for vol of volumes
|
||||
do (vol) ->
|
||||
jobs.push (cb) -> checkVolume(vol, cb)
|
||||
async.series jobs, callback
|
||||
|
||||
_startContainer: (options, volumes, attachStreamHandler, callback = ((error, output) ->)) ->
|
||||
callback = _.once(callback)
|
||||
name = options.name
|
||||
|
||||
logger.log {container_name: name}, "starting container"
|
||||
container = dockerode.getContainer(name)
|
||||
|
||||
createAndStartContainer = ->
|
||||
dockerode.createContainer options, (error, container) ->
|
||||
return callback(error) if error?
|
||||
startExistingContainer()
|
||||
|
||||
startExistingContainer = ->
|
||||
DockerRunner.attachToContainer options.name, attachStreamHandler, (error)->
|
||||
return callback(error) if error?
|
||||
container.start (error) ->
|
||||
if error? and error?.statusCode != 304 #already running
|
||||
return callback(error)
|
||||
else
|
||||
callback()
|
||||
|
||||
container.inspect (error, stats)->
|
||||
if error?.statusCode == 404
|
||||
createAndStartContainer()
|
||||
else if error?
|
||||
logger.err {container_name: name, error:error}, "unable to inspect container to start"
|
||||
return callback(error)
|
||||
else
|
||||
startExistingContainer()
|
||||
|
||||
|
||||
attachToContainer: (containerId, attachStreamHandler, attachStartCallback) ->
|
||||
container = dockerode.getContainer(containerId)
|
||||
container.attach {stdout: 1, stderr: 1, stream: 1}, (error, stream) ->
|
||||
if error?
|
||||
logger.error err: error, container_id: containerId, "error attaching to container"
|
||||
return attachStartCallback(error)
|
||||
else
|
||||
attachStartCallback()
|
||||
|
||||
|
||||
logger.log container_id: containerId, "attached to container"
|
||||
|
||||
MAX_OUTPUT = 1024 * 1024 # limit output to 1MB
|
||||
createStringOutputStream = (name) ->
|
||||
return {
|
||||
data: ""
|
||||
overflowed: false
|
||||
write: (data) ->
|
||||
return if @overflowed
|
||||
if @data.length < MAX_OUTPUT
|
||||
@data += data
|
||||
else
|
||||
logger.error container_id: containerId, length: @data.length, maxLen: MAX_OUTPUT, "#{name} exceeds max size"
|
||||
@data += "(...truncated at #{MAX_OUTPUT} chars...)"
|
||||
@overflowed = true
|
||||
# kill container if too much output
|
||||
# docker.containers.kill(containerId, () ->)
|
||||
return DockerRunner.waitForContainer(name, timeout, function(error, exitCode) {
|
||||
let err;
|
||||
if (error != null) { return callback(error); }
|
||||
if (exitCode === 137) { // exit status from kill -9
|
||||
err = DockerRunner.ERR_TERMINATED;
|
||||
err.terminated = true;
|
||||
return callback(err);
|
||||
}
|
||||
if (exitCode === 1) { // exit status from chktex
|
||||
err = DockerRunner.ERR_EXITED;
|
||||
err.code = exitCode;
|
||||
return callback(err);
|
||||
}
|
||||
containerReturned = true;
|
||||
__guard__(options != null ? options.HostConfig : undefined, x => x.SecurityOpt = null); //small log line
|
||||
logger.log({err, exitCode, options}, "docker container has exited");
|
||||
return callbackIfFinished();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
stdout = createStringOutputStream "stdout"
|
||||
stderr = createStringOutputStream "stderr"
|
||||
_getContainerOptions(command, image, volumes, timeout, environment) {
|
||||
let m, year;
|
||||
let key, value, hostVol, dockerVol;
|
||||
const timeoutInSeconds = timeout / 1000;
|
||||
|
||||
container.modem.demuxStream(stream, stdout, stderr)
|
||||
const dockerVolumes = {};
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol];
|
||||
dockerVolumes[dockerVol] = {};
|
||||
|
||||
stream.on "error", (err) ->
|
||||
logger.error err: err, container_id: containerId, "error reading from container stream"
|
||||
if (volumes[hostVol].slice(-3).indexOf(":r") === -1) {
|
||||
volumes[hostVol] = `${dockerVol}:rw`;
|
||||
}
|
||||
}
|
||||
|
||||
stream.on "end", () ->
|
||||
attachStreamHandler null, {stdout: stdout.data, stderr: stderr.data}
|
||||
// merge settings and environment parameter
|
||||
const env = {};
|
||||
for (let src of [Settings.clsi.docker.env, environment || {}]) {
|
||||
for (key in src) { value = src[key]; env[key] = value; }
|
||||
}
|
||||
// set the path based on the image year
|
||||
if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
|
||||
year = m[1];
|
||||
} else {
|
||||
year = "2014";
|
||||
}
|
||||
env['PATH'] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`;
|
||||
const options = {
|
||||
"Cmd" : command,
|
||||
"Image" : image,
|
||||
"Volumes" : dockerVolumes,
|
||||
"WorkingDir" : "/compile",
|
||||
"NetworkDisabled" : true,
|
||||
"Memory" : 1024 * 1024 * 1024 * 1024, // 1 Gb
|
||||
"User" : Settings.clsi.docker.user,
|
||||
"Env" : (((() => {
|
||||
const result = [];
|
||||
for (key in env) {
|
||||
value = env[key];
|
||||
result.push(`${key}=${value}`);
|
||||
}
|
||||
return result;
|
||||
})())), // convert the environment hash to an array
|
||||
"HostConfig" : {
|
||||
"Binds": (((() => {
|
||||
const result1 = [];
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol];
|
||||
result1.push(`${hostVol}:${dockerVol}`);
|
||||
}
|
||||
return result1;
|
||||
})())),
|
||||
"LogConfig": {"Type": "none", "Config": {}},
|
||||
"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}],
|
||||
"CapDrop": "ALL",
|
||||
"SecurityOpt": ["no-new-privileges"]
|
||||
}
|
||||
};
|
||||
|
||||
waitForContainer: (containerId, timeout, _callback = (error, exitCode) ->) ->
|
||||
callback = (args...) ->
|
||||
_callback(args...)
|
||||
# Only call the callback once
|
||||
_callback = () ->
|
||||
|
||||
container = dockerode.getContainer(containerId)
|
||||
if ((Settings.path != null ? Settings.path.synctexBinHostPath : undefined) != null) {
|
||||
options["HostConfig"]["Binds"].push(`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`);
|
||||
}
|
||||
|
||||
timedOut = false
|
||||
timeoutId = setTimeout () ->
|
||||
timedOut = true
|
||||
logger.log container_id: containerId, "timeout reached, killing container"
|
||||
container.kill(() ->)
|
||||
, timeout
|
||||
if (Settings.clsi.docker.seccomp_profile != null) {
|
||||
options.HostConfig.SecurityOpt.push(`seccomp=${Settings.clsi.docker.seccomp_profile}`);
|
||||
}
|
||||
|
||||
logger.log container_id: containerId, "waiting for docker container"
|
||||
container.wait (error, res) ->
|
||||
if error?
|
||||
clearTimeout timeoutId
|
||||
logger.error err: error, container_id: containerId, "error waiting for container"
|
||||
return callback(error)
|
||||
if timedOut
|
||||
logger.log containerId: containerId, "docker container timed out"
|
||||
error = DockerRunner.ERR_TIMED_OUT
|
||||
error.timedout = true
|
||||
callback error
|
||||
else
|
||||
clearTimeout timeoutId
|
||||
logger.log container_id: containerId, exitCode: res.StatusCode, "docker container returned"
|
||||
callback null, res.StatusCode
|
||||
return options;
|
||||
},
|
||||
|
||||
destroyContainer: (containerName, containerId, shouldForce, callback = (error) ->) ->
|
||||
# We want the containerName for the lock and, ideally, the
|
||||
# containerId to delete. There is a bug in the docker.io module
|
||||
# where if you delete by name and there is an error, it throws an
|
||||
# async exception, but if you delete by id it just does a normal
|
||||
# error callback. We fall back to deleting by name if no id is
|
||||
# supplied.
|
||||
LockManager.runWithLock containerName, (releaseLock) ->
|
||||
DockerRunner._destroyContainer containerId or containerName, shouldForce, releaseLock
|
||||
, callback
|
||||
_fingerprintContainer(containerOptions) {
|
||||
// Yay, Hashing!
|
||||
const json = JSON.stringify(containerOptions);
|
||||
return crypto.createHash("md5").update(json).digest("hex");
|
||||
},
|
||||
|
||||
_destroyContainer: (containerId, shouldForce, callback = (error) ->) ->
|
||||
logger.log container_id: containerId, "destroying docker container"
|
||||
container = dockerode.getContainer(containerId)
|
||||
container.remove {force: shouldForce == true}, (error) ->
|
||||
if error? and error?.statusCode == 404
|
||||
logger.warn err: error, container_id: containerId, "container not found, continuing"
|
||||
error = null
|
||||
if error?
|
||||
logger.error err: error, container_id: containerId, "error destroying container"
|
||||
else
|
||||
logger.log container_id: containerId, "destroyed container"
|
||||
callback(error)
|
||||
startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
return LockManager.runWithLock(options.name, releaseLock =>
|
||||
// Check that volumes exist before starting the container.
|
||||
// When a container is started with volume pointing to a
|
||||
// non-existent directory then docker creates the directory but
|
||||
// with root ownership.
|
||||
DockerRunner._checkVolumes(options, volumes, function(err) {
|
||||
if (err != null) { return releaseLock(err); }
|
||||
return DockerRunner._startContainer(options, volumes, attachStreamHandler, releaseLock);
|
||||
})
|
||||
|
||||
, callback);
|
||||
},
|
||||
|
||||
# handle expiry of docker containers
|
||||
// Check that volumes exist and are directories
|
||||
_checkVolumes(options, volumes, callback) {
|
||||
if (callback == null) { callback = function(error, containerName) {}; }
|
||||
if (usingSiblingContainers()) {
|
||||
// Server Pro, with sibling-containers active, skip checks
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge or oneHour = 60 * 60 * 1000
|
||||
const checkVolume = (path, cb) =>
|
||||
fs.stat(path, function(err, stats) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!(stats != null ? stats.isDirectory() : undefined)) { return cb(DockerRunner.ERR_NOT_DIRECTORY); }
|
||||
return cb();
|
||||
})
|
||||
;
|
||||
const jobs = [];
|
||||
for (let vol in volumes) {
|
||||
(vol => jobs.push(cb => checkVolume(vol, cb)))(vol);
|
||||
}
|
||||
return async.series(jobs, callback);
|
||||
},
|
||||
|
||||
examineOldContainer: (container, callback = (error, name, id, ttl)->) ->
|
||||
name = container.Name or container.Names?[0]
|
||||
created = container.Created * 1000 # creation time is returned in seconds
|
||||
now = Date.now()
|
||||
age = now - created
|
||||
maxAge = DockerRunner.MAX_CONTAINER_AGE
|
||||
ttl = maxAge - age
|
||||
logger.log {containerName: name, created: created, now: now, age: age, maxAge: maxAge, ttl: ttl}, "checking whether to destroy container"
|
||||
callback(null, name, container.Id, ttl)
|
||||
_startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
if (callback == null) { callback = function(error, output) {}; }
|
||||
callback = _.once(callback);
|
||||
const { name } = options;
|
||||
|
||||
destroyOldContainers: (callback = (error) ->) ->
|
||||
dockerode.listContainers all: true, (error, containers) ->
|
||||
return callback(error) if error?
|
||||
jobs = []
|
||||
for container in containers or []
|
||||
do (container) ->
|
||||
DockerRunner.examineOldContainer container, (err, name, id, ttl) ->
|
||||
if name.slice(0, 9) == '/project-' && ttl <= 0
|
||||
jobs.push (cb) ->
|
||||
DockerRunner.destroyContainer name, id, false, () -> cb()
|
||||
# Ignore errors because some containers get stuck but
|
||||
# will be destroyed next time
|
||||
async.series jobs, callback
|
||||
logger.log({container_name: name}, "starting container");
|
||||
const container = dockerode.getContainer(name);
|
||||
|
||||
startContainerMonitor: () ->
|
||||
logger.log {maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry"
|
||||
# randomise the start time
|
||||
randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
|
||||
setTimeout () ->
|
||||
setInterval () ->
|
||||
DockerRunner.destroyOldContainers()
|
||||
, oneHour = 60 * 60 * 1000
|
||||
, randomDelay
|
||||
const createAndStartContainer = () =>
|
||||
dockerode.createContainer(options, function(error, container) {
|
||||
if (error != null) { return callback(error); }
|
||||
return startExistingContainer();
|
||||
})
|
||||
;
|
||||
|
||||
DockerRunner.startContainerMonitor()
|
||||
var startExistingContainer = () =>
|
||||
DockerRunner.attachToContainer(options.name, attachStreamHandler, function(error){
|
||||
if (error != null) { return callback(error); }
|
||||
return container.start(function(error) {
|
||||
if ((error != null) && ((error != null ? error.statusCode : undefined) !== 304)) { //already running
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
})
|
||||
;
|
||||
|
||||
return container.inspect(function(error, stats){
|
||||
if ((error != null ? error.statusCode : undefined) === 404) {
|
||||
return createAndStartContainer();
|
||||
} else if (error != null) {
|
||||
logger.err({container_name: name, error}, "unable to inspect container to start");
|
||||
return callback(error);
|
||||
} else {
|
||||
return startExistingContainer();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
|
||||
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
|
||||
const container = dockerode.getContainer(containerId);
|
||||
return container.attach({stdout: 1, stderr: 1, stream: 1}, function(error, stream) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id: containerId}, "error attaching to container");
|
||||
return attachStartCallback(error);
|
||||
} else {
|
||||
attachStartCallback();
|
||||
}
|
||||
|
||||
|
||||
logger.log({container_id: containerId}, "attached to container");
|
||||
|
||||
const MAX_OUTPUT = 1024 * 1024; // limit output to 1MB
|
||||
const createStringOutputStream = function(name) {
|
||||
return {
|
||||
data: "",
|
||||
overflowed: false,
|
||||
write(data) {
|
||||
if (this.overflowed) { return; }
|
||||
if (this.data.length < MAX_OUTPUT) {
|
||||
return this.data += data;
|
||||
} else {
|
||||
logger.error({container_id: containerId, length: this.data.length, maxLen: MAX_OUTPUT}, `${name} exceeds max size`);
|
||||
this.data += `(...truncated at ${MAX_OUTPUT} chars...)`;
|
||||
return this.overflowed = true;
|
||||
}
|
||||
}
|
||||
// kill container if too much output
|
||||
// docker.containers.kill(containerId, () ->)
|
||||
};
|
||||
};
|
||||
|
||||
const stdout = createStringOutputStream("stdout");
|
||||
const stderr = createStringOutputStream("stderr");
|
||||
|
||||
container.modem.demuxStream(stream, stdout, stderr);
|
||||
|
||||
stream.on("error", err => logger.error({err, container_id: containerId}, "error reading from container stream"));
|
||||
|
||||
return stream.on("end", () => attachStreamHandler(null, {stdout: stdout.data, stderr: stderr.data}));
|
||||
});
|
||||
},
|
||||
|
||||
waitForContainer(containerId, timeout, _callback) {
|
||||
if (_callback == null) { _callback = function(error, exitCode) {}; }
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []));
|
||||
// Only call the callback once
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
const container = dockerode.getContainer(containerId);
|
||||
|
||||
let timedOut = false;
|
||||
const timeoutId = setTimeout(function() {
|
||||
timedOut = true;
|
||||
logger.log({container_id: containerId}, "timeout reached, killing container");
|
||||
return container.kill(function() {});
|
||||
}
|
||||
, timeout);
|
||||
|
||||
logger.log({container_id: containerId}, "waiting for docker container");
|
||||
return container.wait(function(error, res) {
|
||||
if (error != null) {
|
||||
clearTimeout(timeoutId);
|
||||
logger.error({err: error, container_id: containerId}, "error waiting for container");
|
||||
return callback(error);
|
||||
}
|
||||
if (timedOut) {
|
||||
logger.log({containerId}, "docker container timed out");
|
||||
error = DockerRunner.ERR_TIMED_OUT;
|
||||
error.timedout = true;
|
||||
return callback(error);
|
||||
} else {
|
||||
clearTimeout(timeoutId);
|
||||
logger.log({container_id: containerId, exitCode: res.StatusCode}, "docker container returned");
|
||||
return callback(null, res.StatusCode);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
destroyContainer(containerName, containerId, shouldForce, callback) {
|
||||
// We want the containerName for the lock and, ideally, the
|
||||
// containerId to delete. There is a bug in the docker.io module
|
||||
// where if you delete by name and there is an error, it throws an
|
||||
// async exception, but if you delete by id it just does a normal
|
||||
// error callback. We fall back to deleting by name if no id is
|
||||
// supplied.
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return LockManager.runWithLock(containerName, releaseLock => DockerRunner._destroyContainer(containerId || containerName, shouldForce, releaseLock)
|
||||
, callback);
|
||||
},
|
||||
|
||||
_destroyContainer(containerId, shouldForce, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({container_id: containerId}, "destroying docker container");
|
||||
const container = dockerode.getContainer(containerId);
|
||||
return container.remove({force: shouldForce === true}, function(error) {
|
||||
if ((error != null) && ((error != null ? error.statusCode : undefined) === 404)) {
|
||||
logger.warn({err: error, container_id: containerId}, "container not found, continuing");
|
||||
error = null;
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id: containerId}, "error destroying container");
|
||||
} else {
|
||||
logger.log({container_id: containerId}, "destroyed container");
|
||||
}
|
||||
return callback(error);
|
||||
});
|
||||
},
|
||||
|
||||
// handle expiry of docker containers
|
||||
|
||||
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
|
||||
|
||||
examineOldContainer(container, callback) {
|
||||
if (callback == null) { callback = function(error, name, id, ttl){}; }
|
||||
const name = container.Name || (container.Names != null ? container.Names[0] : undefined);
|
||||
const created = container.Created * 1000; // creation time is returned in seconds
|
||||
const now = Date.now();
|
||||
const age = now - created;
|
||||
const maxAge = DockerRunner.MAX_CONTAINER_AGE;
|
||||
const ttl = maxAge - age;
|
||||
logger.log({containerName: name, created, now, age, maxAge, ttl}, "checking whether to destroy container");
|
||||
return callback(null, name, container.Id, ttl);
|
||||
},
|
||||
|
||||
destroyOldContainers(callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return dockerode.listContainers({all: true}, function(error, containers) {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = [];
|
||||
for (let container of Array.from(containers || [])) {
|
||||
(container =>
|
||||
DockerRunner.examineOldContainer(container, function(err, name, id, ttl) {
|
||||
if ((name.slice(0, 9) === '/project-') && (ttl <= 0)) {
|
||||
return jobs.push(cb => DockerRunner.destroyContainer(name, id, false, () => cb()));
|
||||
}
|
||||
})
|
||||
)(container);
|
||||
}
|
||||
// Ignore errors because some containers get stuck but
|
||||
// will be destroyed next time
|
||||
return async.series(jobs, callback);
|
||||
});
|
||||
},
|
||||
|
||||
startContainerMonitor() {
|
||||
logger.log({maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry");
|
||||
// randomise the start time
|
||||
const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000);
|
||||
return setTimeout(() =>
|
||||
setInterval(() => DockerRunner.destroyOldContainers()
|
||||
, (oneHour = 60 * 60 * 1000))
|
||||
|
||||
, randomDelay);
|
||||
}
|
||||
});
|
||||
|
||||
DockerRunner.startContainerMonitor();
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
function __guardMethod__(obj, methodName, transform) {
|
||||
if (typeof obj !== 'undefined' && obj !== null && typeof obj[methodName] === 'function') {
|
||||
return transform(obj, methodName);
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +1,37 @@
|
|||
fs = require "fs"
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DraftModeManager;
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = DraftModeManager =
|
||||
injectDraftMode: (filename, callback = (error) ->) ->
|
||||
fs.readFile filename, "utf8", (error, content) ->
|
||||
return callback(error) if error?
|
||||
# avoid adding draft mode more than once
|
||||
if content?.indexOf("\\documentclass\[draft") >= 0
|
||||
return callback()
|
||||
modified_content = DraftModeManager._injectDraftOption content
|
||||
logger.log {
|
||||
content: content.slice(0,1024), # \documentclass is normally v near the top
|
||||
module.exports = (DraftModeManager = {
|
||||
injectDraftMode(filename, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.readFile(filename, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
// avoid adding draft mode more than once
|
||||
if ((content != null ? content.indexOf("\\documentclass\[draft") : undefined) >= 0) {
|
||||
return callback();
|
||||
}
|
||||
const modified_content = DraftModeManager._injectDraftOption(content);
|
||||
logger.log({
|
||||
content: content.slice(0,1024), // \documentclass is normally v near the top
|
||||
modified_content: modified_content.slice(0,1024),
|
||||
filename
|
||||
}, "injected draft class"
|
||||
fs.writeFile filename, modified_content, callback
|
||||
}, "injected draft class");
|
||||
return fs.writeFile(filename, modified_content, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_injectDraftOption: (content) ->
|
||||
content
|
||||
# With existing options (must be first, otherwise both are applied)
|
||||
_injectDraftOption(content) {
|
||||
return content
|
||||
// With existing options (must be first, otherwise both are applied)
|
||||
.replace(/\\documentclass\[/g, "\\documentclass[draft,")
|
||||
# Without existing options
|
||||
.replace(/\\documentclass\{/g, "\\documentclass[draft]{")
|
||||
// Without existing options
|
||||
.replace(/\\documentclass\{/g, "\\documentclass[draft]{");
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,25 +1,30 @@
|
|||
NotFoundError = (message) ->
|
||||
error = new Error(message)
|
||||
error.name = "NotFoundError"
|
||||
error.__proto__ = NotFoundError.prototype
|
||||
return error
|
||||
NotFoundError.prototype.__proto__ = Error.prototype
|
||||
let Errors;
|
||||
var NotFoundError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "NotFoundError";
|
||||
error.__proto__ = NotFoundError.prototype;
|
||||
return error;
|
||||
};
|
||||
NotFoundError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
FilesOutOfSyncError = (message) ->
|
||||
error = new Error(message)
|
||||
error.name = "FilesOutOfSyncError"
|
||||
error.__proto__ = FilesOutOfSyncError.prototype
|
||||
return error
|
||||
FilesOutOfSyncError.prototype.__proto__ = Error.prototype
|
||||
var FilesOutOfSyncError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "FilesOutOfSyncError";
|
||||
error.__proto__ = FilesOutOfSyncError.prototype;
|
||||
return error;
|
||||
};
|
||||
FilesOutOfSyncError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
AlreadyCompilingError = (message) ->
|
||||
error = new Error(message)
|
||||
error.name = "AlreadyCompilingError"
|
||||
error.__proto__ = AlreadyCompilingError.prototype
|
||||
return error
|
||||
AlreadyCompilingError.prototype.__proto__ = Error.prototype
|
||||
var AlreadyCompilingError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "AlreadyCompilingError";
|
||||
error.__proto__ = AlreadyCompilingError.prototype;
|
||||
return error;
|
||||
};
|
||||
AlreadyCompilingError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
module.exports = Errors =
|
||||
NotFoundError: NotFoundError
|
||||
FilesOutOfSyncError: FilesOutOfSyncError
|
||||
AlreadyCompilingError: AlreadyCompilingError
|
||||
module.exports = (Errors = {
|
||||
NotFoundError,
|
||||
FilesOutOfSyncError,
|
||||
AlreadyCompilingError
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,95 +1,123 @@
|
|||
Path = require "path"
|
||||
Settings = require "settings-sharelatex"
|
||||
logger = require "logger-sharelatex"
|
||||
Metrics = require "./Metrics"
|
||||
CommandRunner = require "./CommandRunner"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LatexRunner;
|
||||
const Path = require("path");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const CommandRunner = require("./CommandRunner");
|
||||
|
||||
ProcessTable = {} # table of currently running jobs (pids or docker container names)
|
||||
const ProcessTable = {}; // table of currently running jobs (pids or docker container names)
|
||||
|
||||
module.exports = LatexRunner =
|
||||
runLatex: (project_id, options, callback = (error) ->) ->
|
||||
{directory, mainFile, compiler, timeout, image, environment, flags} = options
|
||||
compiler ||= "pdflatex"
|
||||
timeout ||= 60000 # milliseconds
|
||||
module.exports = (LatexRunner = {
|
||||
runLatex(project_id, options, callback) {
|
||||
let command;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
let {directory, mainFile, compiler, timeout, image, environment, flags} = options;
|
||||
if (!compiler) { compiler = "pdflatex"; }
|
||||
if (!timeout) { timeout = 60000; } // milliseconds
|
||||
|
||||
logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, environment: environment, flags:flags, "starting compile"
|
||||
logger.log({directory, compiler, timeout, mainFile, environment, flags}, "starting compile");
|
||||
|
||||
# We want to run latexmk on the tex file which we will automatically
|
||||
# generate from the Rtex/Rmd/md file.
|
||||
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex")
|
||||
// We want to run latexmk on the tex file which we will automatically
|
||||
// generate from the Rtex/Rmd/md file.
|
||||
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex");
|
||||
|
||||
if compiler == "pdflatex"
|
||||
command = LatexRunner._pdflatexCommand mainFile, flags
|
||||
else if compiler == "latex"
|
||||
command = LatexRunner._latexCommand mainFile, flags
|
||||
else if compiler == "xelatex"
|
||||
command = LatexRunner._xelatexCommand mainFile, flags
|
||||
else if compiler == "lualatex"
|
||||
command = LatexRunner._lualatexCommand mainFile, flags
|
||||
else
|
||||
return callback new Error("unknown compiler: #{compiler}")
|
||||
if (compiler === "pdflatex") {
|
||||
command = LatexRunner._pdflatexCommand(mainFile, flags);
|
||||
} else if (compiler === "latex") {
|
||||
command = LatexRunner._latexCommand(mainFile, flags);
|
||||
} else if (compiler === "xelatex") {
|
||||
command = LatexRunner._xelatexCommand(mainFile, flags);
|
||||
} else if (compiler === "lualatex") {
|
||||
command = LatexRunner._lualatexCommand(mainFile, flags);
|
||||
} else {
|
||||
return callback(new Error(`unknown compiler: ${compiler}`));
|
||||
}
|
||||
|
||||
if Settings.clsi?.strace
|
||||
command = ["strace", "-o", "strace", "-ff"].concat(command)
|
||||
if (Settings.clsi != null ? Settings.clsi.strace : undefined) {
|
||||
command = ["strace", "-o", "strace", "-ff"].concat(command);
|
||||
}
|
||||
|
||||
id = "#{project_id}" # record running project under this id
|
||||
const id = `${project_id}`; // record running project under this id
|
||||
|
||||
ProcessTable[id] = CommandRunner.run project_id, command, directory, image, timeout, environment, (error, output) ->
|
||||
delete ProcessTable[id]
|
||||
return callback(error) if error?
|
||||
runs = output?.stderr?.match(/^Run number \d+ of .*latex/mg)?.length or 0
|
||||
failed = if output?.stdout?.match(/^Latexmk: Errors/m)? then 1 else 0
|
||||
# counters from latexmk output
|
||||
stats = {}
|
||||
stats["latexmk-errors"] = failed
|
||||
stats["latex-runs"] = runs
|
||||
stats["latex-runs-with-errors"] = if failed then runs else 0
|
||||
stats["latex-runs-#{runs}"] = 1
|
||||
stats["latex-runs-with-errors-#{runs}"] = if failed then 1 else 0
|
||||
# timing information from /usr/bin/time
|
||||
timings = {}
|
||||
stderr = output?.stderr
|
||||
timings["cpu-percent"] = stderr?.match(/Percent of CPU this job got: (\d+)/m)?[1] or 0
|
||||
timings["cpu-time"] = stderr?.match(/User time.*: (\d+.\d+)/m)?[1] or 0
|
||||
timings["sys-time"] = stderr?.match(/System time.*: (\d+.\d+)/m)?[1] or 0
|
||||
callback error, output, stats, timings
|
||||
return ProcessTable[id] = CommandRunner.run(project_id, command, directory, image, timeout, environment, function(error, output) {
|
||||
delete ProcessTable[id];
|
||||
if (error != null) { return callback(error); }
|
||||
const runs = __guard__(__guard__(output != null ? output.stderr : undefined, x1 => x1.match(/^Run number \d+ of .*latex/mg)), x => x.length) || 0;
|
||||
const failed = (__guard__(output != null ? output.stdout : undefined, x2 => x2.match(/^Latexmk: Errors/m)) != null) ? 1 : 0;
|
||||
// counters from latexmk output
|
||||
const stats = {};
|
||||
stats["latexmk-errors"] = failed;
|
||||
stats["latex-runs"] = runs;
|
||||
stats["latex-runs-with-errors"] = failed ? runs : 0;
|
||||
stats[`latex-runs-${runs}`] = 1;
|
||||
stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0;
|
||||
// timing information from /usr/bin/time
|
||||
const timings = {};
|
||||
const stderr = output != null ? output.stderr : undefined;
|
||||
timings["cpu-percent"] = __guard__(stderr != null ? stderr.match(/Percent of CPU this job got: (\d+)/m) : undefined, x3 => x3[1]) || 0;
|
||||
timings["cpu-time"] = __guard__(stderr != null ? stderr.match(/User time.*: (\d+.\d+)/m) : undefined, x4 => x4[1]) || 0;
|
||||
timings["sys-time"] = __guard__(stderr != null ? stderr.match(/System time.*: (\d+.\d+)/m) : undefined, x5 => x5[1]) || 0;
|
||||
return callback(error, output, stats, timings);
|
||||
});
|
||||
},
|
||||
|
||||
killLatex: (project_id, callback = (error) ->) ->
|
||||
id = "#{project_id}"
|
||||
logger.log {id:id}, "killing running compile"
|
||||
if not ProcessTable[id]?
|
||||
logger.warn {id}, "no such project to kill"
|
||||
return callback(null)
|
||||
else
|
||||
CommandRunner.kill ProcessTable[id], callback
|
||||
killLatex(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const id = `${project_id}`;
|
||||
logger.log({id}, "killing running compile");
|
||||
if ((ProcessTable[id] == null)) {
|
||||
logger.warn({id}, "no such project to kill");
|
||||
return callback(null);
|
||||
} else {
|
||||
return CommandRunner.kill(ProcessTable[id], callback);
|
||||
}
|
||||
},
|
||||
|
||||
_latexmkBaseCommand: (flags) ->
|
||||
args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"]
|
||||
if flags
|
||||
args = args.concat(flags)
|
||||
(Settings?.clsi?.latexmkCommandPrefix || []).concat(args)
|
||||
_latexmkBaseCommand(flags) {
|
||||
let args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"];
|
||||
if (flags) {
|
||||
args = args.concat(flags);
|
||||
}
|
||||
return (__guard__(Settings != null ? Settings.clsi : undefined, x => x.latexmkCommandPrefix) || []).concat(args);
|
||||
},
|
||||
|
||||
_pdflatexCommand: (mainFile, flags) ->
|
||||
LatexRunner._latexmkBaseCommand(flags).concat [
|
||||
_pdflatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-pdf",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]
|
||||
]);
|
||||
},
|
||||
|
||||
_latexCommand: (mainFile, flags) ->
|
||||
LatexRunner._latexmkBaseCommand(flags).concat [
|
||||
_latexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-pdfdvi",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]
|
||||
]);
|
||||
},
|
||||
|
||||
_xelatexCommand: (mainFile, flags) ->
|
||||
LatexRunner._latexmkBaseCommand(flags).concat [
|
||||
_xelatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-xelatex",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]
|
||||
]);
|
||||
},
|
||||
|
||||
_lualatexCommand: (mainFile, flags) ->
|
||||
LatexRunner._latexmkBaseCommand(flags).concat [
|
||||
_lualatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-lualatex",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,48 +1,66 @@
|
|||
spawn = require("child_process").spawn
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CommandRunner;
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
logger.info "using standard command runner"
|
||||
logger.info("using standard command runner");
|
||||
|
||||
module.exports = CommandRunner =
|
||||
run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
|
||||
command = (arg.toString().replace('$COMPILE_DIR', directory) for arg in command)
|
||||
logger.log project_id: project_id, command: command, directory: directory, "running command"
|
||||
logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
|
||||
module.exports = (CommandRunner = {
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
let key, value;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
command = (Array.from(command).map((arg) => arg.toString().replace('$COMPILE_DIR', directory)));
|
||||
logger.log({project_id, command, directory}, "running command");
|
||||
logger.warn("timeouts and sandboxing are not enabled with CommandRunner");
|
||||
|
||||
# merge environment settings
|
||||
env = {}
|
||||
env[key] = value for key, value of process.env
|
||||
env[key] = value for key, value of environment
|
||||
// merge environment settings
|
||||
const env = {};
|
||||
for (key in process.env) { value = process.env[key]; env[key] = value; }
|
||||
for (key in environment) { value = environment[key]; env[key] = value; }
|
||||
|
||||
# run command as detached process so it has its own process group (which can be killed if needed)
|
||||
proc = spawn command[0], command.slice(1), cwd: directory, env: env
|
||||
// run command as detached process so it has its own process group (which can be killed if needed)
|
||||
const proc = spawn(command[0], command.slice(1), {cwd: directory, env});
|
||||
|
||||
stdout = ""
|
||||
proc.stdout.on "data", (data)->
|
||||
stdout += data
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", data=> stdout += data);
|
||||
|
||||
proc.on "error", (err)->
|
||||
logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command"
|
||||
callback(err)
|
||||
proc.on("error", function(err){
|
||||
logger.err({err, project_id, command, directory}, "error running command");
|
||||
return callback(err);
|
||||
});
|
||||
|
||||
proc.on "close", (code, signal) ->
|
||||
logger.info code:code, signal:signal, project_id:project_id, "command exited"
|
||||
if signal is 'SIGTERM' # signal from kill method below
|
||||
err = new Error("terminated")
|
||||
err.terminated = true
|
||||
return callback(err)
|
||||
else if code is 1 # exit status from chktex
|
||||
err = new Error("exited")
|
||||
err.code = code
|
||||
return callback(err)
|
||||
else
|
||||
callback(null, {"stdout": stdout})
|
||||
proc.on("close", function(code, signal) {
|
||||
let err;
|
||||
logger.info({code, signal, project_id}, "command exited");
|
||||
if (signal === 'SIGTERM') { // signal from kill method below
|
||||
err = new Error("terminated");
|
||||
err.terminated = true;
|
||||
return callback(err);
|
||||
} else if (code === 1) { // exit status from chktex
|
||||
err = new Error("exited");
|
||||
err.code = code;
|
||||
return callback(err);
|
||||
} else {
|
||||
return callback(null, {"stdout": stdout});
|
||||
}
|
||||
});
|
||||
|
||||
return proc.pid # return process id to allow job to be killed if necessary
|
||||
return proc.pid;
|
||||
}, // return process id to allow job to be killed if necessary
|
||||
|
||||
kill: (pid, callback = (error) ->) ->
|
||||
try
|
||||
process.kill -pid # kill all processes in group
|
||||
catch err
|
||||
return callback(err)
|
||||
callback()
|
||||
kill(pid, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
try {
|
||||
process.kill(-pid); // kill all processes in group
|
||||
} catch (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,31 +1,50 @@
|
|||
Settings = require('settings-sharelatex')
|
||||
logger = require "logger-sharelatex"
|
||||
Lockfile = require('lockfile') # from https://github.com/npm/lockfile
|
||||
Errors = require "./Errors"
|
||||
fs = require("fs")
|
||||
Path = require("path")
|
||||
module.exports = LockManager =
|
||||
LOCK_TEST_INTERVAL: 1000 # 50ms between each test of the lock
|
||||
MAX_LOCK_WAIT_TIME: 15000 # 10s maximum time to spend trying to get the lock
|
||||
LOCK_STALE: 5*60*1000 # 5 mins time until lock auto expires
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager;
|
||||
const Settings = require('settings-sharelatex');
|
||||
const logger = require("logger-sharelatex");
|
||||
const Lockfile = require('lockfile'); // from https://github.com/npm/lockfile
|
||||
const Errors = require("./Errors");
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
module.exports = (LockManager = {
|
||||
LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock
|
||||
MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock
|
||||
LOCK_STALE: 5*60*1000, // 5 mins time until lock auto expires
|
||||
|
||||
runWithLock: (path, runner, callback = ((error) ->)) ->
|
||||
lockOpts =
|
||||
wait: @MAX_LOCK_WAIT_TIME
|
||||
pollPeriod: @LOCK_TEST_INTERVAL
|
||||
stale: @LOCK_STALE
|
||||
Lockfile.lock path, lockOpts, (error) ->
|
||||
if error?.code is 'EEXIST'
|
||||
return callback new Errors.AlreadyCompilingError("compile in progress")
|
||||
else if error?
|
||||
fs.lstat path, (statLockErr, statLock)->
|
||||
fs.lstat Path.dirname(path), (statDirErr, statDir)->
|
||||
fs.readdir Path.dirname(path), (readdirErr, readdirDir)->
|
||||
logger.err error:error, path:path, statLock:statLock, statLockErr:statLockErr, statDir:statDir, statDirErr: statDirErr, readdirErr:readdirErr, readdirDir:readdirDir, "unable to get lock"
|
||||
return callback(error)
|
||||
else
|
||||
runner (error1, args...) ->
|
||||
Lockfile.unlock path, (error2) ->
|
||||
error = error1 or error2
|
||||
return callback(error) if error?
|
||||
callback(null, args...)
|
||||
runWithLock(path, runner, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const lockOpts = {
|
||||
wait: this.MAX_LOCK_WAIT_TIME,
|
||||
pollPeriod: this.LOCK_TEST_INTERVAL,
|
||||
stale: this.LOCK_STALE
|
||||
};
|
||||
return Lockfile.lock(path, lockOpts, function(error) {
|
||||
if ((error != null ? error.code : undefined) === 'EEXIST') {
|
||||
return callback(new Errors.AlreadyCompilingError("compile in progress"));
|
||||
} else if (error != null) {
|
||||
return fs.lstat(path, (statLockErr, statLock)=>
|
||||
fs.lstat(Path.dirname(path), (statDirErr, statDir)=>
|
||||
fs.readdir(Path.dirname(path), function(readdirErr, readdirDir){
|
||||
logger.err({error, path, statLock, statLockErr, statDir, statDirErr, readdirErr, readdirDir}, "unable to get lock");
|
||||
return callback(error);
|
||||
})
|
||||
)
|
||||
);
|
||||
} else {
|
||||
return runner((error1, ...args) =>
|
||||
Lockfile.unlock(path, function(error2) {
|
||||
error = error1 || error2;
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, ...Array.from(args));
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
module.exports = require "metrics-sharelatex"
|
||||
module.exports = require("metrics-sharelatex");
|
||||
|
||||
|
|
|
|||
|
|
@ -1,199 +1,270 @@
|
|||
async = require "async"
|
||||
fs = require "fs"
|
||||
fse = require "fs-extra"
|
||||
Path = require "path"
|
||||
logger = require "logger-sharelatex"
|
||||
_ = require "underscore"
|
||||
Settings = require "settings-sharelatex"
|
||||
crypto = require "crypto"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS104: Avoid inline assignments
|
||||
* DS204: Change includes calls to have a more natural evaluation order
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputCacheManager;
|
||||
const async = require("async");
|
||||
const fs = require("fs");
|
||||
const fse = require("fs-extra");
|
||||
const Path = require("path");
|
||||
const logger = require("logger-sharelatex");
|
||||
const _ = require("underscore");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const crypto = require("crypto");
|
||||
|
||||
OutputFileOptimiser = require "./OutputFileOptimiser"
|
||||
const OutputFileOptimiser = require("./OutputFileOptimiser");
|
||||
|
||||
module.exports = OutputCacheManager =
|
||||
CACHE_SUBDIR: '.cache/clsi'
|
||||
ARCHIVE_SUBDIR: '.archive/clsi'
|
||||
# build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
|
||||
# for backwards compatibility, make the randombytes part optional
|
||||
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/
|
||||
CACHE_LIMIT: 2 # maximum number of cache directories
|
||||
CACHE_AGE: 60*60*1000 # up to one hour old
|
||||
module.exports = (OutputCacheManager = {
|
||||
CACHE_SUBDIR: '.cache/clsi',
|
||||
ARCHIVE_SUBDIR: '.archive/clsi',
|
||||
// build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
|
||||
// for backwards compatibility, make the randombytes part optional
|
||||
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
|
||||
CACHE_LIMIT: 2, // maximum number of cache directories
|
||||
CACHE_AGE: 60*60*1000, // up to one hour old
|
||||
|
||||
path: (buildId, file) ->
|
||||
# used by static server, given build id return '.cache/clsi/buildId'
|
||||
if buildId.match OutputCacheManager.BUILD_REGEX
|
||||
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
|
||||
else
|
||||
# for invalid build id, return top level
|
||||
return file
|
||||
path(buildId, file) {
|
||||
// used by static server, given build id return '.cache/clsi/buildId'
|
||||
if (buildId.match(OutputCacheManager.BUILD_REGEX)) {
|
||||
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file);
|
||||
} else {
|
||||
// for invalid build id, return top level
|
||||
return file;
|
||||
}
|
||||
},
|
||||
|
||||
generateBuildId: (callback = (error, buildId) ->) ->
|
||||
# generate a secure build id from Date.now() and 8 random bytes in hex
|
||||
crypto.randomBytes 8, (err, buf) ->
|
||||
return callback(err) if err?
|
||||
random = buf.toString('hex')
|
||||
date = Date.now().toString(16)
|
||||
callback err, "#{date}-#{random}"
|
||||
generateBuildId(callback) {
|
||||
// generate a secure build id from Date.now() and 8 random bytes in hex
|
||||
if (callback == null) { callback = function(error, buildId) {}; }
|
||||
return crypto.randomBytes(8, function(err, buf) {
|
||||
if (err != null) { return callback(err); }
|
||||
const random = buf.toString('hex');
|
||||
const date = Date.now().toString(16);
|
||||
return callback(err, `${date}-${random}`);
|
||||
});
|
||||
},
|
||||
|
||||
saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
|
||||
OutputCacheManager.generateBuildId (err, buildId) ->
|
||||
return callback(err) if err?
|
||||
OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback
|
||||
saveOutputFiles(outputFiles, compileDir, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return OutputCacheManager.generateBuildId(function(err, buildId) {
|
||||
if (err != null) { return callback(err); }
|
||||
return OutputCacheManager.saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback);
|
||||
});
|
||||
},
|
||||
|
||||
saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) ->
|
||||
# make a compileDir/CACHE_SUBDIR/build_id directory and
|
||||
# copy all the output files into it
|
||||
cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
|
||||
# Put the files into a new cache subdirectory
|
||||
cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
|
||||
# Is it a per-user compile? check if compile directory is PROJECTID-USERID
|
||||
perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/)
|
||||
saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) {
|
||||
// make a compileDir/CACHE_SUBDIR/build_id directory and
|
||||
// copy all the output files into it
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR);
|
||||
// Put the files into a new cache subdirectory
|
||||
const cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId);
|
||||
// Is it a per-user compile? check if compile directory is PROJECTID-USERID
|
||||
const perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/);
|
||||
|
||||
# Archive logs in background
|
||||
if Settings.clsi?.archive_logs or Settings.clsi?.strace
|
||||
OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) ->
|
||||
if err?
|
||||
logger.warn err:err, "erroring archiving log files"
|
||||
// Archive logs in background
|
||||
if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) || (Settings.clsi != null ? Settings.clsi.strace : undefined)) {
|
||||
OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(err) {
|
||||
if (err != null) {
|
||||
return logger.warn({err}, "erroring archiving log files");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
# make the new cache directory
|
||||
fse.ensureDir cacheDir, (err) ->
|
||||
if err?
|
||||
logger.error err: err, directory: cacheDir, "error creating cache directory"
|
||||
callback(err, outputFiles)
|
||||
else
|
||||
# copy all the output files into the new cache directory
|
||||
results = []
|
||||
async.mapSeries outputFiles, (file, cb) ->
|
||||
# don't send dot files as output, express doesn't serve them
|
||||
if OutputCacheManager._fileIsHidden(file.path)
|
||||
logger.debug compileDir: compileDir, path: file.path, "ignoring dotfile in output"
|
||||
return cb()
|
||||
# copy other files into cache directory if valid
|
||||
newFile = _.clone(file)
|
||||
[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
|
||||
OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
|
||||
return cb(err) if err?
|
||||
if !isSafe
|
||||
return cb()
|
||||
OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
|
||||
return cb(err) if err?
|
||||
if !shouldCopy
|
||||
return cb()
|
||||
OutputCacheManager._copyFile src, dst, (err) ->
|
||||
return cb(err) if err?
|
||||
newFile.build = buildId # attach a build id if we cached the file
|
||||
results.push newFile
|
||||
cb()
|
||||
, (err) ->
|
||||
if err?
|
||||
# pass back the original files if we encountered *any* error
|
||||
callback(err, outputFiles)
|
||||
# clean up the directory we just created
|
||||
fse.remove cacheDir, (err) ->
|
||||
if err?
|
||||
logger.error err: err, dir: cacheDir, "error removing cache dir after failure"
|
||||
else
|
||||
# pass back the list of new files in the cache
|
||||
callback(err, results)
|
||||
# let file expiry run in the background, expire all previous files if per-user
|
||||
OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null}
|
||||
// make the new cache directory
|
||||
return fse.ensureDir(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
logger.error({err, directory: cacheDir}, "error creating cache directory");
|
||||
return callback(err, outputFiles);
|
||||
} else {
|
||||
// copy all the output files into the new cache directory
|
||||
const results = [];
|
||||
return async.mapSeries(outputFiles, function(file, cb) {
|
||||
// don't send dot files as output, express doesn't serve them
|
||||
if (OutputCacheManager._fileIsHidden(file.path)) {
|
||||
logger.debug({compileDir, path: file.path}, "ignoring dotfile in output");
|
||||
return cb();
|
||||
}
|
||||
// copy other files into cache directory if valid
|
||||
const newFile = _.clone(file);
|
||||
const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]);
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!isSafe) {
|
||||
return cb();
|
||||
}
|
||||
return OutputCacheManager._checkIfShouldCopy(src, function(err, shouldCopy) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!shouldCopy) {
|
||||
return cb();
|
||||
}
|
||||
return OutputCacheManager._copyFile(src, dst, function(err) {
|
||||
if (err != null) { return cb(err); }
|
||||
newFile.build = buildId; // attach a build id if we cached the file
|
||||
results.push(newFile);
|
||||
return cb();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
, function(err) {
|
||||
if (err != null) {
|
||||
// pass back the original files if we encountered *any* error
|
||||
callback(err, outputFiles);
|
||||
// clean up the directory we just created
|
||||
return fse.remove(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
return logger.error({err, dir: cacheDir}, "error removing cache dir after failure");
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// pass back the list of new files in the cache
|
||||
callback(err, results);
|
||||
// let file expiry run in the background, expire all previous files if per-user
|
||||
return OutputCacheManager.expireOutputFiles(cacheRoot, {keep: buildId, limit: perUser ? 1 : null});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) ->
|
||||
archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
|
||||
logger.log {dir: archiveDir}, "archiving log files for project"
|
||||
fse.ensureDir archiveDir, (err) ->
|
||||
return callback(err) if err?
|
||||
async.mapSeries outputFiles, (file, cb) ->
|
||||
[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
|
||||
OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
|
||||
return cb(err) if err?
|
||||
return cb() if !isSafe
|
||||
OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
|
||||
return cb(err) if err?
|
||||
return cb() if !shouldArchive
|
||||
OutputCacheManager._copyFile src, dst, cb
|
||||
, callback
|
||||
archiveLogs(outputFiles, compileDir, buildId, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId);
|
||||
logger.log({dir: archiveDir}, "archiving log files for project");
|
||||
return fse.ensureDir(archiveDir, function(err) {
|
||||
if (err != null) { return callback(err); }
|
||||
return async.mapSeries(outputFiles, function(file, cb) {
|
||||
const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]);
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!isSafe) { return cb(); }
|
||||
return OutputCacheManager._checkIfShouldArchive(src, function(err, shouldArchive) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!shouldArchive) { return cb(); }
|
||||
return OutputCacheManager._copyFile(src, dst, cb);
|
||||
});
|
||||
});
|
||||
}
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
|
||||
# look in compileDir for build dirs and delete if > N or age of mod time > T
|
||||
fs.readdir cacheRoot, (err, results) ->
|
||||
if err?
|
||||
return callback(null) if err.code == 'ENOENT' # cache directory is empty
|
||||
logger.error err: err, project_id: cacheRoot, "error clearing cache"
|
||||
return callback(err)
|
||||
expireOutputFiles(cacheRoot, options, callback) {
|
||||
// look in compileDir for build dirs and delete if > N or age of mod time > T
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.readdir(cacheRoot, function(err, results) {
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') { return callback(null); } // cache directory is empty
|
||||
logger.error({err, project_id: cacheRoot}, "error clearing cache");
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
dirs = results.sort().reverse()
|
||||
currentTime = Date.now()
|
||||
const dirs = results.sort().reverse();
|
||||
const currentTime = Date.now();
|
||||
|
||||
isExpired = (dir, index) ->
|
||||
return false if options?.keep == dir
|
||||
# remove any directories over the requested (non-null) limit
|
||||
return true if options?.limit? and index > options.limit
|
||||
# remove any directories over the hard limit
|
||||
return true if index > OutputCacheManager.CACHE_LIMIT
|
||||
# we can get the build time from the first part of the directory name DDDD-RRRR
|
||||
# DDDD is date and RRRR is random bytes
|
||||
dirTime = parseInt(dir.split('-')?[0], 16)
|
||||
age = currentTime - dirTime
|
||||
return age > OutputCacheManager.CACHE_AGE
|
||||
const isExpired = function(dir, index) {
|
||||
if ((options != null ? options.keep : undefined) === dir) { return false; }
|
||||
// remove any directories over the requested (non-null) limit
|
||||
if (((options != null ? options.limit : undefined) != null) && (index > options.limit)) { return true; }
|
||||
// remove any directories over the hard limit
|
||||
if (index > OutputCacheManager.CACHE_LIMIT) { return true; }
|
||||
// we can get the build time from the first part of the directory name DDDD-RRRR
|
||||
// DDDD is date and RRRR is random bytes
|
||||
const dirTime = parseInt(__guard__(dir.split('-'), x => x[0]), 16);
|
||||
const age = currentTime - dirTime;
|
||||
return age > OutputCacheManager.CACHE_AGE;
|
||||
};
|
||||
|
||||
toRemove = _.filter(dirs, isExpired)
|
||||
const toRemove = _.filter(dirs, isExpired);
|
||||
|
||||
removeDir = (dir, cb) ->
|
||||
fse.remove Path.join(cacheRoot, dir), (err, result) ->
|
||||
logger.log cache: cacheRoot, dir: dir, "removed expired cache dir"
|
||||
if err?
|
||||
logger.error err: err, dir: dir, "cache remove error"
|
||||
cb(err, result)
|
||||
const removeDir = (dir, cb) =>
|
||||
fse.remove(Path.join(cacheRoot, dir), function(err, result) {
|
||||
logger.log({cache: cacheRoot, dir}, "removed expired cache dir");
|
||||
if (err != null) {
|
||||
logger.error({err, dir}, "cache remove error");
|
||||
}
|
||||
return cb(err, result);
|
||||
})
|
||||
;
|
||||
|
||||
async.eachSeries toRemove, (dir, cb) ->
|
||||
removeDir dir, cb
|
||||
, callback
|
||||
return async.eachSeries(toRemove, (dir, cb) => removeDir(dir, cb)
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_fileIsHidden: (path) ->
|
||||
return path?.match(/^\.|\/\./)?
|
||||
_fileIsHidden(path) {
|
||||
return ((path != null ? path.match(/^\.|\/\./) : undefined) != null);
|
||||
},
|
||||
|
||||
_checkFileIsSafe: (src, callback = (error, isSafe) ->) ->
|
||||
# check if we have a valid file to copy into the cache
|
||||
fs.stat src, (err, stats) ->
|
||||
if err?.code is 'ENOENT'
|
||||
logger.warn err: err, file: src, "file has disappeared before copying to build cache"
|
||||
callback(err, false)
|
||||
else if err?
|
||||
# some other problem reading the file
|
||||
logger.error err: err, file: src, "stat error for file in cache"
|
||||
callback(err, false)
|
||||
else if not stats.isFile()
|
||||
# other filetype - reject it
|
||||
logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache"
|
||||
callback(null, false)
|
||||
else
|
||||
# it's a plain file, ok to copy
|
||||
callback(null, true)
|
||||
_checkFileIsSafe(src, callback) {
|
||||
// check if we have a valid file to copy into the cache
|
||||
if (callback == null) { callback = function(error, isSafe) {}; }
|
||||
return fs.stat(src, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn({err, file: src}, "file has disappeared before copying to build cache");
|
||||
return callback(err, false);
|
||||
} else if (err != null) {
|
||||
// some other problem reading the file
|
||||
logger.error({err, file: src}, "stat error for file in cache");
|
||||
return callback(err, false);
|
||||
} else if (!stats.isFile()) {
|
||||
// other filetype - reject it
|
||||
logger.warn({src, stat: stats}, "nonfile output - refusing to copy to cache");
|
||||
return callback(null, false);
|
||||
} else {
|
||||
// it's a plain file, ok to copy
|
||||
return callback(null, true);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_copyFile: (src, dst, callback) ->
|
||||
# copy output file into the cache
|
||||
fse.copy src, dst, (err) ->
|
||||
if err?.code is 'ENOENT'
|
||||
logger.warn err: err, file: src, "file has disappeared when copying to build cache"
|
||||
callback(err, false)
|
||||
else if err?
|
||||
logger.error err: err, src: src, dst: dst, "copy error for file in cache"
|
||||
callback(err)
|
||||
else
|
||||
if Settings.clsi?.optimiseInDocker
|
||||
# don't run any optimisations on the pdf when they are done
|
||||
# in the docker container
|
||||
callback()
|
||||
else
|
||||
# call the optimiser for the file too
|
||||
OutputFileOptimiser.optimiseFile src, dst, callback
|
||||
_copyFile(src, dst, callback) {
|
||||
// copy output file into the cache
|
||||
return fse.copy(src, dst, function(err) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn({err, file: src}, "file has disappeared when copying to build cache");
|
||||
return callback(err, false);
|
||||
} else if (err != null) {
|
||||
logger.error({err, src, dst}, "copy error for file in cache");
|
||||
return callback(err);
|
||||
} else {
|
||||
if ((Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined)) {
|
||||
// don't run any optimisations on the pdf when they are done
|
||||
// in the docker container
|
||||
return callback();
|
||||
} else {
|
||||
// call the optimiser for the file too
|
||||
return OutputFileOptimiser.optimiseFile(src, dst, callback);
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) ->
|
||||
return callback(null, !Path.basename(src).match(/^strace/))
|
||||
_checkIfShouldCopy(src, callback) {
|
||||
if (callback == null) { callback = function(err, shouldCopy) {}; }
|
||||
return callback(null, !Path.basename(src).match(/^strace/));
|
||||
},
|
||||
|
||||
_checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) ->
|
||||
if Path.basename(src).match(/^strace/)
|
||||
return callback(null, true)
|
||||
if Settings.clsi?.archive_logs and Path.basename(src) in ["output.log", "output.blg"]
|
||||
return callback(null, true)
|
||||
return callback(null, false)
|
||||
_checkIfShouldArchive(src, callback) {
|
||||
let needle;
|
||||
if (callback == null) { callback = function(err, shouldCopy) {}; }
|
||||
if (Path.basename(src).match(/^strace/)) {
|
||||
return callback(null, true);
|
||||
}
|
||||
if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) && (needle = Path.basename(src), ["output.log", "output.blg"].includes(needle))) {
|
||||
return callback(null, true);
|
||||
}
|
||||
return callback(null, false);
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,50 +1,78 @@
|
|||
async = require "async"
|
||||
fs = require "fs"
|
||||
Path = require "path"
|
||||
spawn = require("child_process").spawn
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileFinder;
|
||||
const async = require("async");
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = OutputFileFinder =
|
||||
findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) ->
|
||||
incomingResources = {}
|
||||
for resource in resources
|
||||
incomingResources[resource.path] = true
|
||||
module.exports = (OutputFileFinder = {
|
||||
findOutputFiles(resources, directory, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles, allFiles) {}; }
|
||||
const incomingResources = {};
|
||||
for (let resource of Array.from(resources)) {
|
||||
incomingResources[resource.path] = true;
|
||||
}
|
||||
|
||||
OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
|
||||
if error?
|
||||
logger.err err:error, "error finding all output files"
|
||||
return callback(error)
|
||||
outputFiles = []
|
||||
for file in allFiles
|
||||
if !incomingResources[file]
|
||||
outputFiles.push {
|
||||
path: file
|
||||
type: file.match(/\.([^\.]+)$/)?[1]
|
||||
}
|
||||
callback null, outputFiles, allFiles
|
||||
return OutputFileFinder._getAllFiles(directory, function(error, allFiles) {
|
||||
if (allFiles == null) { allFiles = []; }
|
||||
if (error != null) {
|
||||
logger.err({err:error}, "error finding all output files");
|
||||
return callback(error);
|
||||
}
|
||||
const outputFiles = [];
|
||||
for (let file of Array.from(allFiles)) {
|
||||
if (!incomingResources[file]) {
|
||||
outputFiles.push({
|
||||
path: file,
|
||||
type: __guard__(file.match(/\.([^\.]+)$/), x => x[1])
|
||||
});
|
||||
}
|
||||
}
|
||||
return callback(null, outputFiles, allFiles);
|
||||
});
|
||||
},
|
||||
|
||||
_getAllFiles: (directory, _callback = (error, fileList) ->) ->
|
||||
callback = (error, fileList) ->
|
||||
_callback(error, fileList)
|
||||
_callback = () ->
|
||||
_getAllFiles(directory, _callback) {
|
||||
if (_callback == null) { _callback = function(error, fileList) {}; }
|
||||
const callback = function(error, fileList) {
|
||||
_callback(error, fileList);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
# don't include clsi-specific files/directories in the output list
|
||||
EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"]
|
||||
args = [directory, "(", EXCLUDE_DIRS..., ")", "-prune", "-o", "-type", "f", "-print"]
|
||||
logger.log args: args, "running find command"
|
||||
// don't include clsi-specific files/directories in the output list
|
||||
const EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"];
|
||||
const args = [directory, "(", ...Array.from(EXCLUDE_DIRS), ")", "-prune", "-o", "-type", "f", "-print"];
|
||||
logger.log({args}, "running find command");
|
||||
|
||||
proc = spawn("find", args)
|
||||
stdout = ""
|
||||
proc.stdout.on "data", (chunk) ->
|
||||
stdout += chunk.toString()
|
||||
proc.on "error", callback
|
||||
proc.on "close", (code) ->
|
||||
if code != 0
|
||||
logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
|
||||
return callback null, []
|
||||
fileList = stdout.trim().split("\n")
|
||||
fileList = fileList.map (file) ->
|
||||
# Strip leading directory
|
||||
path = Path.relative(directory, file)
|
||||
return callback null, fileList
|
||||
const proc = spawn("find", args);
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", chunk => stdout += chunk.toString());
|
||||
proc.on("error", callback);
|
||||
return proc.on("close", function(code) {
|
||||
if (code !== 0) {
|
||||
logger.warn({directory, code}, "find returned error, directory likely doesn't exist");
|
||||
return callback(null, []);
|
||||
}
|
||||
let fileList = stdout.trim().split("\n");
|
||||
fileList = fileList.map(function(file) {
|
||||
// Strip leading directory
|
||||
let path;
|
||||
return path = Path.relative(directory, file);
|
||||
});
|
||||
return callback(null, fileList);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,55 +1,77 @@
|
|||
fs = require "fs"
|
||||
Path = require "path"
|
||||
spawn = require("child_process").spawn
|
||||
logger = require "logger-sharelatex"
|
||||
Metrics = require "./Metrics"
|
||||
_ = require "underscore"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileOptimiser;
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const _ = require("underscore");
|
||||
|
||||
module.exports = OutputFileOptimiser =
|
||||
module.exports = (OutputFileOptimiser = {
|
||||
|
||||
optimiseFile: (src, dst, callback = (error) ->) ->
|
||||
# check output file (src) and see if we can optimise it, storing
|
||||
# the result in the build directory (dst)
|
||||
if src.match(/\/output\.pdf$/)
|
||||
OutputFileOptimiser.checkIfPDFIsOptimised src, (err, isOptimised) ->
|
||||
return callback(null) if err? or isOptimised
|
||||
OutputFileOptimiser.optimisePDF src, dst, callback
|
||||
else
|
||||
callback (null)
|
||||
optimiseFile(src, dst, callback) {
|
||||
// check output file (src) and see if we can optimise it, storing
|
||||
// the result in the build directory (dst)
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
if (src.match(/\/output\.pdf$/)) {
|
||||
return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(err, isOptimised) {
|
||||
if ((err != null) || isOptimised) { return callback(null); }
|
||||
return OutputFileOptimiser.optimisePDF(src, dst, callback);
|
||||
});
|
||||
} else {
|
||||
return callback((null));
|
||||
}
|
||||
},
|
||||
|
||||
checkIfPDFIsOptimised: (file, callback) ->
|
||||
SIZE = 16*1024 # check the header of the pdf
|
||||
result = new Buffer(SIZE)
|
||||
result.fill(0) # prevent leakage of uninitialised buffer
|
||||
fs.open file, "r", (err, fd) ->
|
||||
return callback(err) if err?
|
||||
fs.read fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) ->
|
||||
fs.close fd, (errClose) ->
|
||||
return callback(errRead) if errRead?
|
||||
return callback(errClose) if errReadClose?
|
||||
isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0
|
||||
callback(null, isOptimised)
|
||||
checkIfPDFIsOptimised(file, callback) {
|
||||
const SIZE = 16*1024; // check the header of the pdf
|
||||
const result = new Buffer(SIZE);
|
||||
result.fill(0); // prevent leakage of uninitialised buffer
|
||||
return fs.open(file, "r", function(err, fd) {
|
||||
if (err != null) { return callback(err); }
|
||||
return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
|
||||
fs.close(fd, function(errClose) {
|
||||
if (errRead != null) { return callback(errRead); }
|
||||
if (typeof errReadClose !== 'undefined' && errReadClose !== null) { return callback(errClose); }
|
||||
const isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0;
|
||||
return callback(null, isOptimised);
|
||||
})
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
optimisePDF: (src, dst, callback = (error) ->) ->
|
||||
tmpOutput = dst + '.opt'
|
||||
args = ["--linearize", src, tmpOutput]
|
||||
logger.log args: args, "running qpdf command"
|
||||
optimisePDF(src, dst, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const tmpOutput = dst + '.opt';
|
||||
const args = ["--linearize", src, tmpOutput];
|
||||
logger.log({args}, "running qpdf command");
|
||||
|
||||
timer = new Metrics.Timer("qpdf")
|
||||
proc = spawn("qpdf", args)
|
||||
stdout = ""
|
||||
proc.stdout.on "data", (chunk) ->
|
||||
stdout += chunk.toString()
|
||||
callback = _.once(callback) # avoid double call back for error and close event
|
||||
proc.on "error", (err) ->
|
||||
logger.warn {err, args}, "qpdf failed"
|
||||
callback(null) # ignore the error
|
||||
proc.on "close", (code) ->
|
||||
timer.done()
|
||||
if code != 0
|
||||
logger.warn {code, args}, "qpdf returned error"
|
||||
return callback(null) # ignore the error
|
||||
fs.rename tmpOutput, dst, (err) ->
|
||||
if err?
|
||||
logger.warn {tmpOutput, dst}, "failed to rename output of qpdf command"
|
||||
callback(null) # ignore the error
|
||||
const timer = new Metrics.Timer("qpdf");
|
||||
const proc = spawn("qpdf", args);
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", chunk => stdout += chunk.toString());
|
||||
callback = _.once(callback); // avoid double call back for error and close event
|
||||
proc.on("error", function(err) {
|
||||
logger.warn({err, args}, "qpdf failed");
|
||||
return callback(null);
|
||||
}); // ignore the error
|
||||
return proc.on("close", function(code) {
|
||||
timer.done();
|
||||
if (code !== 0) {
|
||||
logger.warn({code, args}, "qpdf returned error");
|
||||
return callback(null); // ignore the error
|
||||
}
|
||||
return fs.rename(tmpOutput, dst, function(err) {
|
||||
if (err != null) {
|
||||
logger.warn({tmpOutput, dst}, "failed to rename output of qpdf command");
|
||||
}
|
||||
return callback(null);
|
||||
});
|
||||
});
|
||||
} // ignore the error
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,84 +1,117 @@
|
|||
UrlCache = require "./UrlCache"
|
||||
CompileManager = require "./CompileManager"
|
||||
db = require "./db"
|
||||
dbQueue = require "./DbQueue"
|
||||
async = require "async"
|
||||
logger = require "logger-sharelatex"
|
||||
oneDay = 24 * 60 * 60 * 1000
|
||||
Settings = require "settings-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ProjectPersistenceManager;
|
||||
const UrlCache = require("./UrlCache");
|
||||
const CompileManager = require("./CompileManager");
|
||||
const db = require("./db");
|
||||
const dbQueue = require("./DbQueue");
|
||||
const async = require("async");
|
||||
const logger = require("logger-sharelatex");
|
||||
const oneDay = 24 * 60 * 60 * 1000;
|
||||
const Settings = require("settings-sharelatex");
|
||||
|
||||
module.exports = ProjectPersistenceManager =
|
||||
module.exports = (ProjectPersistenceManager = {
|
||||
|
||||
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
|
||||
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || (oneDay * 2.5),
|
||||
|
||||
markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
|
||||
job = (cb)->
|
||||
db.Project.findOrCreate(where: {project_id: project_id})
|
||||
markProjectAsJustAccessed(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.Project.findOrCreate({where: {project_id}})
|
||||
.spread(
|
||||
(project, created) ->
|
||||
project.updateAttributes(lastAccessed: new Date())
|
||||
.then(() -> cb())
|
||||
.error cb
|
||||
(project, created) =>
|
||||
project.updateAttributes({lastAccessed: new Date()})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error cb
|
||||
dbQueue.queue.push(job, callback)
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
clearExpiredProjects: (callback = (error) ->) ->
|
||||
ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) ->
|
||||
return callback(error) if error?
|
||||
logger.log project_ids: project_ids, "clearing expired projects"
|
||||
jobs = for project_id in (project_ids or [])
|
||||
do (project_id) ->
|
||||
(callback) ->
|
||||
ProjectPersistenceManager.clearProjectFromCache project_id, (err) ->
|
||||
if err?
|
||||
logger.error err: err, project_id: project_id, "error clearing project"
|
||||
callback()
|
||||
async.series jobs, (error) ->
|
||||
return callback(error) if error?
|
||||
CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) ->
|
||||
callback() # ignore any errors from deleting directories
|
||||
clearExpiredProjects(callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ProjectPersistenceManager._findExpiredProjectIds(function(error, project_ids) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_ids}, "clearing expired projects");
|
||||
const jobs = (Array.from(project_ids || [])).map((project_id) =>
|
||||
(project_id =>
|
||||
callback =>
|
||||
ProjectPersistenceManager.clearProjectFromCache(project_id, function(err) {
|
||||
if (err != null) {
|
||||
logger.error({err, project_id}, "error clearing project");
|
||||
}
|
||||
return callback();
|
||||
})
|
||||
|
||||
)(project_id));
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return CompileManager.clearExpiredProjects(ProjectPersistenceManager.EXPIRY_TIMEOUT, error => callback());
|
||||
});
|
||||
});
|
||||
}, // ignore any errors from deleting directories
|
||||
|
||||
clearProject: (project_id, user_id, callback = (error) ->) ->
|
||||
logger.log project_id: project_id, user_id:user_id, "clearing project for user"
|
||||
CompileManager.clearProject project_id, user_id, (error) ->
|
||||
return callback(error) if error?
|
||||
ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->
|
||||
return callback(error) if error?
|
||||
callback()
|
||||
clearProject(project_id, user_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id, user_id}, "clearing project for user");
|
||||
return CompileManager.clearProject(project_id, user_id, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ProjectPersistenceManager.clearProjectFromCache(project_id, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
clearProjectFromCache: (project_id, callback = (error) ->) ->
|
||||
logger.log project_id: project_id, "clearing project from cache"
|
||||
UrlCache.clearProject project_id, (error) ->
|
||||
if error?
|
||||
logger.err error:error, project_id: project_id, "error clearing project from cache"
|
||||
return callback(error)
|
||||
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
|
||||
if error?
|
||||
logger.err error:error, project_id:project_id, "error clearing project from database"
|
||||
callback(error)
|
||||
clearProjectFromCache(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id}, "clearing project from cache");
|
||||
return UrlCache.clearProject(project_id, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id}, "error clearing project from cache");
|
||||
return callback(error);
|
||||
}
|
||||
return ProjectPersistenceManager._clearProjectFromDatabase(project_id, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id}, "error clearing project from database");
|
||||
}
|
||||
return callback(error);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
|
||||
logger.log project_id:project_id, "clearing project from database"
|
||||
job = (cb)->
|
||||
db.Project.destroy(where: {project_id: project_id})
|
||||
.then(() -> cb())
|
||||
.error cb
|
||||
dbQueue.queue.push(job, callback)
|
||||
_clearProjectFromDatabase(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id}, "clearing project from database");
|
||||
const job = cb=>
|
||||
db.Project.destroy({where: {project_id}})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
|
||||
job = (cb)->
|
||||
keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)
|
||||
q = {}
|
||||
q[db.op.lt] = keepProjectsFrom
|
||||
db.Project.findAll(where:{lastAccessed:q})
|
||||
.then((projects) ->
|
||||
cb null, projects.map((project) -> project.project_id)
|
||||
).error cb
|
||||
_findExpiredProjectIds(callback) {
|
||||
if (callback == null) { callback = function(error, project_ids) {}; }
|
||||
const job = function(cb){
|
||||
const keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT);
|
||||
const q = {};
|
||||
q[db.op.lt] = keepProjectsFrom;
|
||||
return db.Project.findAll({where:{lastAccessed:q}})
|
||||
.then(projects => cb(null, projects.map(project => project.project_id))).error(cb);
|
||||
};
|
||||
|
||||
dbQueue.queue.push(job, callback)
|
||||
return dbQueue.queue.push(job, callback);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"
|
||||
logger.log({EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout");
|
||||
|
|
|
|||
|
|
@ -1,128 +1,182 @@
|
|||
settings = require("settings-sharelatex")
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let RequestParser;
|
||||
const settings = require("settings-sharelatex");
|
||||
|
||||
module.exports = RequestParser =
|
||||
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
|
||||
MAX_TIMEOUT: 600
|
||||
module.exports = (RequestParser = {
|
||||
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"],
|
||||
MAX_TIMEOUT: 600,
|
||||
|
||||
parse: (body, callback = (error, data) ->) ->
|
||||
response = {}
|
||||
parse(body, callback) {
|
||||
let resource;
|
||||
if (callback == null) { callback = function(error, data) {}; }
|
||||
const response = {};
|
||||
|
||||
if !body.compile?
|
||||
return callback "top level object should have a compile attribute"
|
||||
|
||||
compile = body.compile
|
||||
compile.options ||= {}
|
||||
|
||||
try
|
||||
response.compiler = @_parseAttribute "compiler",
|
||||
compile.options.compiler,
|
||||
validValues: @VALID_COMPILERS
|
||||
default: "pdflatex"
|
||||
type: "string"
|
||||
response.timeout = @_parseAttribute "timeout",
|
||||
compile.options.timeout
|
||||
default: RequestParser.MAX_TIMEOUT
|
||||
type: "number"
|
||||
response.imageName = @_parseAttribute "imageName",
|
||||
compile.options.imageName,
|
||||
type: "string"
|
||||
response.draft = @_parseAttribute "draft",
|
||||
compile.options.draft,
|
||||
default: false,
|
||||
type: "boolean"
|
||||
response.check = @_parseAttribute "check",
|
||||
compile.options.check,
|
||||
type: "string"
|
||||
response.flags = @_parseAttribute "flags",
|
||||
compile.options.flags,
|
||||
default: [],
|
||||
type: "object"
|
||||
|
||||
# The syncType specifies whether the request contains all
|
||||
# resources (full) or only those resources to be updated
|
||||
# in-place (incremental).
|
||||
response.syncType = @_parseAttribute "syncType",
|
||||
compile.options.syncType,
|
||||
validValues: ["full", "incremental"]
|
||||
type: "string"
|
||||
|
||||
# The syncState is an identifier passed in with the request
|
||||
# which has the property that it changes when any resource is
|
||||
# added, deleted, moved or renamed.
|
||||
#
|
||||
# on syncType full the syncState identifier is passed in and
|
||||
# stored
|
||||
#
|
||||
# on syncType incremental the syncState identifier must match
|
||||
# the stored value
|
||||
response.syncState = @_parseAttribute "syncState",
|
||||
compile.options.syncState,
|
||||
type: "string"
|
||||
|
||||
if response.timeout > RequestParser.MAX_TIMEOUT
|
||||
response.timeout = RequestParser.MAX_TIMEOUT
|
||||
response.timeout = response.timeout * 1000 # milliseconds
|
||||
|
||||
response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
|
||||
|
||||
rootResourcePath = @_parseAttribute "rootResourcePath",
|
||||
compile.rootResourcePath
|
||||
default: "main.tex"
|
||||
type: "string"
|
||||
originalRootResourcePath = rootResourcePath
|
||||
sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath)
|
||||
response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath)
|
||||
|
||||
for resource in response.resources
|
||||
if resource.path == originalRootResourcePath
|
||||
resource.path = sanitizedRootResourcePath
|
||||
catch error
|
||||
return callback error
|
||||
|
||||
callback null, response
|
||||
|
||||
_parseResource: (resource) ->
|
||||
if !resource.path? or typeof resource.path != "string"
|
||||
throw "all resources should have a path attribute"
|
||||
|
||||
if resource.modified?
|
||||
modified = new Date(resource.modified)
|
||||
if isNaN(modified.getTime())
|
||||
throw "resource modified date could not be understood: #{resource.modified}"
|
||||
|
||||
if !resource.url? and !resource.content?
|
||||
throw "all resources should have either a url or content attribute"
|
||||
if resource.content? and typeof resource.content != "string"
|
||||
throw "content attribute should be a string"
|
||||
if resource.url? and typeof resource.url != "string"
|
||||
throw "url attribute should be a string"
|
||||
|
||||
return {
|
||||
path: resource.path
|
||||
modified: modified
|
||||
url: resource.url
|
||||
content: resource.content
|
||||
if ((body.compile == null)) {
|
||||
return callback("top level object should have a compile attribute");
|
||||
}
|
||||
|
||||
_parseAttribute: (name, attribute, options) ->
|
||||
if attribute?
|
||||
if options.validValues?
|
||||
if options.validValues.indexOf(attribute) == -1
|
||||
throw "#{name} attribute should be one of: #{options.validValues.join(", ")}"
|
||||
if options.type?
|
||||
if typeof attribute != options.type
|
||||
throw "#{name} attribute should be a #{options.type}"
|
||||
else
|
||||
return options.default if options.default?
|
||||
return attribute
|
||||
const { compile } = body;
|
||||
if (!compile.options) { compile.options = {}; }
|
||||
|
||||
_sanitizePath: (path) ->
|
||||
# See http://php.net/manual/en/function.escapeshellcmd.php
|
||||
path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "")
|
||||
try {
|
||||
response.compiler = this._parseAttribute("compiler",
|
||||
compile.options.compiler, {
|
||||
validValues: this.VALID_COMPILERS,
|
||||
default: "pdflatex",
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
response.timeout = this._parseAttribute("timeout",
|
||||
compile.options.timeout, {
|
||||
default: RequestParser.MAX_TIMEOUT,
|
||||
type: "number"
|
||||
}
|
||||
);
|
||||
response.imageName = this._parseAttribute("imageName",
|
||||
compile.options.imageName,
|
||||
{type: "string"});
|
||||
response.draft = this._parseAttribute("draft",
|
||||
compile.options.draft, {
|
||||
default: false,
|
||||
type: "boolean"
|
||||
}
|
||||
);
|
||||
response.check = this._parseAttribute("check",
|
||||
compile.options.check,
|
||||
{type: "string"});
|
||||
response.flags = this._parseAttribute("flags",
|
||||
compile.options.flags, {
|
||||
default: [],
|
||||
type: "object"
|
||||
}
|
||||
);
|
||||
|
||||
_checkPath: (path) ->
|
||||
# check that the request does not use a relative path
|
||||
for dir in path.split('/')
|
||||
if dir == '..'
|
||||
throw "relative path in root resource"
|
||||
return path
|
||||
// The syncType specifies whether the request contains all
|
||||
// resources (full) or only those resources to be updated
|
||||
// in-place (incremental).
|
||||
response.syncType = this._parseAttribute("syncType",
|
||||
compile.options.syncType, {
|
||||
validValues: ["full", "incremental"],
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
|
||||
// The syncState is an identifier passed in with the request
|
||||
// which has the property that it changes when any resource is
|
||||
// added, deleted, moved or renamed.
|
||||
//
|
||||
// on syncType full the syncState identifier is passed in and
|
||||
// stored
|
||||
//
|
||||
// on syncType incremental the syncState identifier must match
|
||||
// the stored value
|
||||
response.syncState = this._parseAttribute("syncState",
|
||||
compile.options.syncState,
|
||||
{type: "string"});
|
||||
|
||||
if (response.timeout > RequestParser.MAX_TIMEOUT) {
|
||||
response.timeout = RequestParser.MAX_TIMEOUT;
|
||||
}
|
||||
response.timeout = response.timeout * 1000; // milliseconds
|
||||
|
||||
response.resources = ((() => {
|
||||
const result = [];
|
||||
for (resource of Array.from((compile.resources || []))) { result.push(this._parseResource(resource));
|
||||
}
|
||||
return result;
|
||||
})());
|
||||
|
||||
const rootResourcePath = this._parseAttribute("rootResourcePath",
|
||||
compile.rootResourcePath, {
|
||||
default: "main.tex",
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
const originalRootResourcePath = rootResourcePath;
|
||||
const sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath);
|
||||
response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath);
|
||||
|
||||
for (resource of Array.from(response.resources)) {
|
||||
if (resource.path === originalRootResourcePath) {
|
||||
resource.path = sanitizedRootResourcePath;
|
||||
}
|
||||
}
|
||||
} catch (error1) {
|
||||
const error = error1;
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
return callback(null, response);
|
||||
},
|
||||
|
||||
_parseResource(resource) {
|
||||
let modified;
|
||||
if ((resource.path == null) || (typeof resource.path !== "string")) {
|
||||
throw "all resources should have a path attribute";
|
||||
}
|
||||
|
||||
if (resource.modified != null) {
|
||||
modified = new Date(resource.modified);
|
||||
if (isNaN(modified.getTime())) {
|
||||
throw `resource modified date could not be understood: ${resource.modified}`;
|
||||
}
|
||||
}
|
||||
|
||||
if ((resource.url == null) && (resource.content == null)) {
|
||||
throw "all resources should have either a url or content attribute";
|
||||
}
|
||||
if ((resource.content != null) && (typeof resource.content !== "string")) {
|
||||
throw "content attribute should be a string";
|
||||
}
|
||||
if ((resource.url != null) && (typeof resource.url !== "string")) {
|
||||
throw "url attribute should be a string";
|
||||
}
|
||||
|
||||
return {
|
||||
path: resource.path,
|
||||
modified,
|
||||
url: resource.url,
|
||||
content: resource.content
|
||||
};
|
||||
},
|
||||
|
||||
_parseAttribute(name, attribute, options) {
|
||||
if (attribute != null) {
|
||||
if (options.validValues != null) {
|
||||
if (options.validValues.indexOf(attribute) === -1) {
|
||||
throw `${name} attribute should be one of: ${options.validValues.join(", ")}`;
|
||||
}
|
||||
}
|
||||
if (options.type != null) {
|
||||
if (typeof attribute !== options.type) {
|
||||
throw `${name} attribute should be a ${options.type}`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (options.default != null) { return options.default; }
|
||||
}
|
||||
return attribute;
|
||||
},
|
||||
|
||||
_sanitizePath(path) {
|
||||
// See http://php.net/manual/en/function.escapeshellcmd.php
|
||||
return path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "");
|
||||
},
|
||||
|
||||
_checkPath(path) {
|
||||
// check that the request does not use a relative path
|
||||
for (let dir of Array.from(path.split('/'))) {
|
||||
if (dir === '..') {
|
||||
throw "relative path in root resource";
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,72 +1,108 @@
|
|||
Path = require "path"
|
||||
fs = require "fs"
|
||||
logger = require "logger-sharelatex"
|
||||
settings = require("settings-sharelatex")
|
||||
Errors = require "./Errors"
|
||||
SafeReader = require "./SafeReader"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS201: Simplify complex destructure assignments
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceStateManager;
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
const Errors = require("./Errors");
|
||||
const SafeReader = require("./SafeReader");
|
||||
|
||||
module.exports = ResourceStateManager =
|
||||
module.exports = (ResourceStateManager = {
|
||||
|
||||
# The sync state is an identifier which must match for an
|
||||
# incremental update to be allowed.
|
||||
#
|
||||
# The initial value is passed in and stored on a full
|
||||
# compile, along with the list of resources..
|
||||
#
|
||||
# Subsequent incremental compiles must come with the same value - if
|
||||
# not they will be rejected with a 409 Conflict response. The
|
||||
# previous list of resources is returned.
|
||||
#
|
||||
# An incremental compile can only update existing files with new
|
||||
# content. The sync state identifier must change if any docs or
|
||||
# files are moved, added, deleted or renamed.
|
||||
// The sync state is an identifier which must match for an
|
||||
// incremental update to be allowed.
|
||||
//
|
||||
// The initial value is passed in and stored on a full
|
||||
// compile, along with the list of resources..
|
||||
//
|
||||
// Subsequent incremental compiles must come with the same value - if
|
||||
// not they will be rejected with a 409 Conflict response. The
|
||||
// previous list of resources is returned.
|
||||
//
|
||||
// An incremental compile can only update existing files with new
|
||||
// content. The sync state identifier must change if any docs or
|
||||
// files are moved, added, deleted or renamed.
|
||||
|
||||
SYNC_STATE_FILE: ".project-sync-state"
|
||||
SYNC_STATE_MAX_SIZE: 128*1024
|
||||
SYNC_STATE_FILE: ".project-sync-state",
|
||||
SYNC_STATE_MAX_SIZE: 128*1024,
|
||||
|
||||
saveProjectState: (state, resources, basePath, callback = (error) ->) ->
|
||||
stateFile = Path.join(basePath, @SYNC_STATE_FILE)
|
||||
if not state? # remove the file if no state passed in
|
||||
logger.log state:state, basePath:basePath, "clearing sync state"
|
||||
fs.unlink stateFile, (err) ->
|
||||
if err? and err.code isnt 'ENOENT'
|
||||
return callback(err)
|
||||
else
|
||||
return callback()
|
||||
else
|
||||
logger.log state:state, basePath:basePath, "writing sync state"
|
||||
resourceList = (resource.path for resource in resources)
|
||||
fs.writeFile stateFile, [resourceList..., "stateHash:#{state}"].join("\n"), callback
|
||||
saveProjectState(state, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
|
||||
if ((state == null)) { // remove the file if no state passed in
|
||||
logger.log({state, basePath}, "clearing sync state");
|
||||
return fs.unlink(stateFile, function(err) {
|
||||
if ((err != null) && (err.code !== 'ENOENT')) {
|
||||
return callback(err);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
logger.log({state, basePath}, "writing sync state");
|
||||
const resourceList = (Array.from(resources).map((resource) => resource.path));
|
||||
return fs.writeFile(stateFile, [...Array.from(resourceList), `stateHash:${state}`].join("\n"), callback);
|
||||
}
|
||||
},
|
||||
|
||||
checkProjectStateMatches: (state, basePath, callback = (error, resources) ->) ->
|
||||
stateFile = Path.join(basePath, @SYNC_STATE_FILE)
|
||||
size = @SYNC_STATE_MAX_SIZE
|
||||
SafeReader.readFile stateFile, size, 'utf8', (err, result, bytesRead) ->
|
||||
return callback(err) if err?
|
||||
if bytesRead is size
|
||||
logger.error file:stateFile, size:size, bytesRead:bytesRead, "project state file truncated"
|
||||
[resourceList..., oldState] = result?.toString()?.split("\n") or []
|
||||
newState = "stateHash:#{state}"
|
||||
logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: (newState is oldState), "checking sync state"
|
||||
if newState isnt oldState
|
||||
return callback new Errors.FilesOutOfSyncError("invalid state for incremental update")
|
||||
else
|
||||
resources = ({path: path} for path in resourceList)
|
||||
callback(null, resources)
|
||||
checkProjectStateMatches(state, basePath, callback) {
|
||||
if (callback == null) { callback = function(error, resources) {}; }
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
|
||||
const size = this.SYNC_STATE_MAX_SIZE;
|
||||
return SafeReader.readFile(stateFile, size, 'utf8', function(err, result, bytesRead) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (bytesRead === size) {
|
||||
logger.error({file:stateFile, size, bytesRead}, "project state file truncated");
|
||||
}
|
||||
const array = __guard__(result != null ? result.toString() : undefined, x => x.split("\n")) || [],
|
||||
adjustedLength = Math.max(array.length, 1),
|
||||
resourceList = array.slice(0, adjustedLength - 1),
|
||||
oldState = array[adjustedLength - 1];
|
||||
const newState = `stateHash:${state}`;
|
||||
logger.log({state, oldState, basePath, stateMatches: (newState === oldState)}, "checking sync state");
|
||||
if (newState !== oldState) {
|
||||
return callback(new Errors.FilesOutOfSyncError("invalid state for incremental update"));
|
||||
} else {
|
||||
const resources = (Array.from(resourceList).map((path) => ({path})));
|
||||
return callback(null, resources);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
checkResourceFiles: (resources, allFiles, basePath, callback = (error) ->) ->
|
||||
# check the paths are all relative to current directory
|
||||
for file in resources or []
|
||||
for dir in file?.path?.split('/')
|
||||
if dir == '..'
|
||||
return callback new Error("relative path in resource file list")
|
||||
# check if any of the input files are not present in list of files
|
||||
seenFile = {}
|
||||
for file in allFiles
|
||||
seenFile[file] = true
|
||||
missingFiles = (resource.path for resource in resources when not seenFile[resource.path])
|
||||
if missingFiles?.length > 0
|
||||
logger.err missingFiles:missingFiles, basePath:basePath, allFiles:allFiles, resources:resources, "missing input files for project"
|
||||
return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update")
|
||||
else
|
||||
callback()
|
||||
checkResourceFiles(resources, allFiles, basePath, callback) {
|
||||
// check the paths are all relative to current directory
|
||||
let file;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
for (file of Array.from(resources || [])) {
|
||||
for (let dir of Array.from(__guard__(file != null ? file.path : undefined, x => x.split('/')))) {
|
||||
if (dir === '..') {
|
||||
return callback(new Error("relative path in resource file list"));
|
||||
}
|
||||
}
|
||||
}
|
||||
// check if any of the input files are not present in list of files
|
||||
const seenFile = {};
|
||||
for (file of Array.from(allFiles)) {
|
||||
seenFile[file] = true;
|
||||
}
|
||||
const missingFiles = (Array.from(resources).filter((resource) => !seenFile[resource.path]).map((resource) => resource.path));
|
||||
if ((missingFiles != null ? missingFiles.length : undefined) > 0) {
|
||||
logger.err({missingFiles, basePath, allFiles, resources}, "missing input files for project");
|
||||
return callback(new Errors.FilesOutOfSyncError("resource files missing in incremental update"));
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,142 +1,206 @@
|
|||
UrlCache = require "./UrlCache"
|
||||
Path = require "path"
|
||||
fs = require "fs"
|
||||
async = require "async"
|
||||
mkdirp = require "mkdirp"
|
||||
OutputFileFinder = require "./OutputFileFinder"
|
||||
ResourceStateManager = require "./ResourceStateManager"
|
||||
Metrics = require "./Metrics"
|
||||
logger = require "logger-sharelatex"
|
||||
settings = require("settings-sharelatex")
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceWriter;
|
||||
const UrlCache = require("./UrlCache");
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const async = require("async");
|
||||
const mkdirp = require("mkdirp");
|
||||
const OutputFileFinder = require("./OutputFileFinder");
|
||||
const ResourceStateManager = require("./ResourceStateManager");
|
||||
const Metrics = require("./Metrics");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
|
||||
parallelFileDownloads = settings.parallelFileDownloads or 1
|
||||
const parallelFileDownloads = settings.parallelFileDownloads || 1;
|
||||
|
||||
module.exports = ResourceWriter =
|
||||
module.exports = (ResourceWriter = {
|
||||
|
||||
syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) ->
|
||||
if request.syncType is "incremental"
|
||||
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
|
||||
ResourceStateManager.checkProjectStateMatches request.syncState, basePath, (error, resourceList) ->
|
||||
return callback(error) if error?
|
||||
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) ->
|
||||
return callback(error) if error?
|
||||
ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) ->
|
||||
return callback(error) if error?
|
||||
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
|
||||
return callback(error) if error?
|
||||
callback(null, resourceList)
|
||||
else
|
||||
logger.log project_id: request.project_id, user_id: request.user_id, "full sync"
|
||||
@saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
|
||||
return callback(error) if error?
|
||||
ResourceStateManager.saveProjectState request.syncState, request.resources, basePath, (error) ->
|
||||
return callback(error) if error?
|
||||
callback(null, request.resources)
|
||||
syncResourcesToDisk(request, basePath, callback) {
|
||||
if (callback == null) { callback = function(error, resourceList) {}; }
|
||||
if (request.syncType === "incremental") {
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "incremental sync");
|
||||
return ResourceStateManager.checkProjectStateMatches(request.syncState, basePath, function(error, resourceList) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceWriter._removeExtraneousFiles(resourceList, basePath, function(error, outputFiles, allFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceStateManager.checkResourceFiles(resourceList, allFiles, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceWriter.saveIncrementalResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, resourceList);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "full sync");
|
||||
return this.saveAllResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceStateManager.saveProjectState(request.syncState, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, request.resources);
|
||||
});
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
|
||||
@_createDirectory basePath, (error) =>
|
||||
return callback(error) if error?
|
||||
jobs = for resource in resources
|
||||
do (resource) =>
|
||||
(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
|
||||
async.parallelLimit jobs, parallelFileDownloads, callback
|
||||
saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = Array.from(resources).map((resource) =>
|
||||
(resource => {
|
||||
return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
|
||||
})(resource));
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback);
|
||||
});
|
||||
},
|
||||
|
||||
saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
|
||||
@_createDirectory basePath, (error) =>
|
||||
return callback(error) if error?
|
||||
@_removeExtraneousFiles resources, basePath, (error) =>
|
||||
return callback(error) if error?
|
||||
jobs = for resource in resources
|
||||
do (resource) =>
|
||||
(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
|
||||
async.parallelLimit jobs, parallelFileDownloads, callback
|
||||
saveAllResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return this._removeExtraneousFiles(resources, basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = Array.from(resources).map((resource) =>
|
||||
(resource => {
|
||||
return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
|
||||
})(resource));
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_createDirectory: (basePath, callback = (error) ->) ->
|
||||
fs.mkdir basePath, (err) ->
|
||||
if err?
|
||||
if err.code is 'EEXIST'
|
||||
return callback()
|
||||
else
|
||||
logger.log {err: err, dir:basePath}, "error creating directory"
|
||||
return callback(err)
|
||||
else
|
||||
return callback()
|
||||
_createDirectory(basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.mkdir(basePath, function(err) {
|
||||
if (err != null) {
|
||||
if (err.code === 'EEXIST') {
|
||||
return callback();
|
||||
} else {
|
||||
logger.log({err, dir:basePath}, "error creating directory");
|
||||
return callback(err);
|
||||
}
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) ->
|
||||
timer = new Metrics.Timer("unlink-output-files")
|
||||
callback = (error, result...) ->
|
||||
timer.done()
|
||||
_callback(error, result...)
|
||||
_removeExtraneousFiles(resources, basePath, _callback) {
|
||||
if (_callback == null) { _callback = function(error, outputFiles, allFiles) {}; }
|
||||
const timer = new Metrics.Timer("unlink-output-files");
|
||||
const callback = function(error, ...result) {
|
||||
timer.done();
|
||||
return _callback(error, ...Array.from(result));
|
||||
};
|
||||
|
||||
OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) ->
|
||||
return callback(error) if error?
|
||||
return OutputFileFinder.findOutputFiles(resources, basePath, function(error, outputFiles, allFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
|
||||
jobs = []
|
||||
for file in outputFiles or []
|
||||
do (file) ->
|
||||
path = file.path
|
||||
should_delete = true
|
||||
if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache
|
||||
should_delete = false
|
||||
if path.match(/^output-.*/) # Tikz cached figures (default case)
|
||||
should_delete = false
|
||||
if path.match(/\.(pdf|dpth|md5)$/) # Tikz cached figures (by extension)
|
||||
should_delete = false
|
||||
if path.match(/\.(pygtex|pygstyle)$/) or path.match(/(^|\/)_minted-[^\/]+\//) # minted files/directory
|
||||
should_delete = false
|
||||
if path.match(/\.md\.tex$/) or path.match(/(^|\/)_markdown_[^\/]+\//) # markdown files/directory
|
||||
should_delete = false
|
||||
if path.match(/-eps-converted-to\.pdf$/) # Epstopdf generated files
|
||||
should_delete = false
|
||||
if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv"
|
||||
should_delete = true
|
||||
if path == "output.tex" # created by TikzManager if present in output files
|
||||
should_delete = true
|
||||
if should_delete
|
||||
jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
|
||||
const jobs = [];
|
||||
for (let file of Array.from(outputFiles || [])) {
|
||||
(function(file) {
|
||||
const { path } = file;
|
||||
let should_delete = true;
|
||||
if (path.match(/^output\./) || path.match(/\.aux$/) || path.match(/^cache\//)) { // knitr cache
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/^output-.*/)) { // Tikz cached figures (default case)
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.(pdf|dpth|md5)$/)) { // Tikz cached figures (by extension)
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.(pygtex|pygstyle)$/) || path.match(/(^|\/)_minted-[^\/]+\//)) { // minted files/directory
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) { // markdown files/directory
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/-eps-converted-to\.pdf$/)) { // Epstopdf generated files
|
||||
should_delete = false;
|
||||
}
|
||||
if ((path === "output.pdf") || (path === "output.dvi") || (path === "output.log") || (path === "output.xdv")) {
|
||||
should_delete = true;
|
||||
}
|
||||
if (path === "output.tex") { // created by TikzManager if present in output files
|
||||
should_delete = true;
|
||||
}
|
||||
if (should_delete) {
|
||||
return jobs.push(callback => ResourceWriter._deleteFileIfNotDirectory(Path.join(basePath, path), callback));
|
||||
}
|
||||
})(file);
|
||||
}
|
||||
|
||||
async.series jobs, (error) ->
|
||||
return callback(error) if error?
|
||||
callback(null, outputFiles, allFiles)
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, outputFiles, allFiles);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
|
||||
fs.stat path, (error, stat) ->
|
||||
if error? and error.code is 'ENOENT'
|
||||
return callback()
|
||||
else if error?
|
||||
logger.err {err: error, path: path}, "error stating file in deleteFileIfNotDirectory"
|
||||
return callback(error)
|
||||
else if stat.isFile()
|
||||
fs.unlink path, (error) ->
|
||||
if error?
|
||||
logger.err {err: error, path: path}, "error removing file in deleteFileIfNotDirectory"
|
||||
callback(error)
|
||||
else
|
||||
callback()
|
||||
else
|
||||
callback()
|
||||
_deleteFileIfNotDirectory(path, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.stat(path, function(error, stat) {
|
||||
if ((error != null) && (error.code === 'ENOENT')) {
|
||||
return callback();
|
||||
} else if (error != null) {
|
||||
logger.err({err: error, path}, "error stating file in deleteFileIfNotDirectory");
|
||||
return callback(error);
|
||||
} else if (stat.isFile()) {
|
||||
return fs.unlink(path, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({err: error, path}, "error removing file in deleteFileIfNotDirectory");
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
|
||||
ResourceWriter.checkPath basePath, resource.path, (error, path) ->
|
||||
return callback(error) if error?
|
||||
mkdirp Path.dirname(path), (error) ->
|
||||
return callback(error) if error?
|
||||
# TODO: Don't overwrite file if it hasn't been modified
|
||||
if resource.url?
|
||||
UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)->
|
||||
if err?
|
||||
logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources"
|
||||
callback() #try and continue compiling even if http resource can not be downloaded at this time
|
||||
else
|
||||
process = require("process")
|
||||
fs.writeFile path, resource.content, callback
|
||||
try
|
||||
result = fs.lstatSync(path)
|
||||
catch e
|
||||
_writeResourceToDisk(project_id, resource, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ResourceWriter.checkPath(basePath, resource.path, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return mkdirp(Path.dirname(path), function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
// TODO: Don't overwrite file if it hasn't been modified
|
||||
if (resource.url != null) {
|
||||
return UrlCache.downloadUrlToFile(project_id, resource.url, path, resource.modified, function(err){
|
||||
if (err != null) {
|
||||
logger.err({err, project_id, path, resource_url:resource.url, modified:resource.modified}, "error downloading file for resources");
|
||||
}
|
||||
return callback();
|
||||
}); //try and continue compiling even if http resource can not be downloaded at this time
|
||||
} else {
|
||||
const process = require("process");
|
||||
fs.writeFile(path, resource.content, callback);
|
||||
try {
|
||||
let result;
|
||||
return result = fs.lstatSync(path);
|
||||
} catch (e) {}
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
checkPath: (basePath, resourcePath, callback) ->
|
||||
path = Path.normalize(Path.join(basePath, resourcePath))
|
||||
if (path.slice(0, basePath.length + 1) != basePath + "/")
|
||||
return callback new Error("resource path is outside root directory")
|
||||
else
|
||||
return callback(null, path)
|
||||
checkPath(basePath, resourcePath, callback) {
|
||||
const path = Path.normalize(Path.join(basePath, resourcePath));
|
||||
if (path.slice(0, basePath.length + 1) !== (basePath + "/")) {
|
||||
return callback(new Error("resource path is outside root directory"));
|
||||
} else {
|
||||
return callback(null, path);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,25 +1,40 @@
|
|||
fs = require "fs"
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let SafeReader;
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = SafeReader =
|
||||
module.exports = (SafeReader = {
|
||||
|
||||
# safely read up to size bytes from a file and return result as a
|
||||
# string
|
||||
// safely read up to size bytes from a file and return result as a
|
||||
// string
|
||||
|
||||
readFile: (file, size, encoding, callback = (error, result) ->) ->
|
||||
fs.open file, 'r', (err, fd) ->
|
||||
return callback() if err? and err.code is 'ENOENT'
|
||||
return callback(err) if err?
|
||||
readFile(file, size, encoding, callback) {
|
||||
if (callback == null) { callback = function(error, result) {}; }
|
||||
return fs.open(file, 'r', function(err, fd) {
|
||||
if ((err != null) && (err.code === 'ENOENT')) { return callback(); }
|
||||
if (err != null) { return callback(err); }
|
||||
|
||||
# safely return always closing the file
|
||||
callbackWithClose = (err, result...) ->
|
||||
fs.close fd, (err1) ->
|
||||
return callback(err) if err?
|
||||
return callback(err1) if err1?
|
||||
callback(null, result...)
|
||||
// safely return always closing the file
|
||||
const callbackWithClose = (err, ...result) =>
|
||||
fs.close(fd, function(err1) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (err1 != null) { return callback(err1); }
|
||||
return callback(null, ...Array.from(result));
|
||||
})
|
||||
;
|
||||
|
||||
buff = new Buffer(size, 0) # fill with zeros
|
||||
fs.read fd, buff, 0, buff.length, 0, (err, bytesRead, buffer) ->
|
||||
return callbackWithClose(err) if err?
|
||||
result = buffer.toString(encoding, 0, bytesRead)
|
||||
callbackWithClose(null, result, bytesRead)
|
||||
const buff = new Buffer(size, 0); // fill with zeros
|
||||
return fs.read(fd, buff, 0, buff.length, 0, function(err, bytesRead, buffer) {
|
||||
if (err != null) { return callbackWithClose(err); }
|
||||
const result = buffer.toString(encoding, 0, bytesRead);
|
||||
return callbackWithClose(null, result, bytesRead);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,41 +1,64 @@
|
|||
Path = require("path")
|
||||
fs = require("fs")
|
||||
Settings = require("settings-sharelatex")
|
||||
logger = require("logger-sharelatex")
|
||||
url = require "url"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ForbidSymlinks;
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const url = require("url");
|
||||
|
||||
module.exports = ForbidSymlinks = (staticFn, root, options) ->
|
||||
expressStatic = staticFn root, options
|
||||
basePath = Path.resolve(root)
|
||||
return (req, res, next) ->
|
||||
path = url.parse(req.url)?.pathname
|
||||
# check that the path is of the form /project_id_or_name/path/to/file.log
|
||||
if result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/)
|
||||
project_id = result[1]
|
||||
file = result[2]
|
||||
else
|
||||
logger.warn path: path, "unrecognized file request"
|
||||
return res.sendStatus(404)
|
||||
# check that the file does not use a relative path
|
||||
for dir in file.split('/')
|
||||
if dir == '..'
|
||||
logger.warn path: path, "attempt to use a relative path"
|
||||
return res.sendStatus(404)
|
||||
# check that the requested path is normalized
|
||||
requestedFsPath = "#{basePath}/#{project_id}/#{file}"
|
||||
if requestedFsPath != Path.normalize(requestedFsPath)
|
||||
logger.error path: requestedFsPath, "requestedFsPath is not normalized"
|
||||
return res.sendStatus(404)
|
||||
# check that the requested path is not a symlink
|
||||
fs.realpath requestedFsPath, (err, realFsPath)->
|
||||
if err?
|
||||
if err.code == 'ENOENT'
|
||||
return res.sendStatus(404)
|
||||
else
|
||||
logger.error err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access"
|
||||
return res.sendStatus(500)
|
||||
else if requestedFsPath != realFsPath
|
||||
logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting"
|
||||
return res.sendStatus(404)
|
||||
else
|
||||
expressStatic(req, res, next)
|
||||
module.exports = (ForbidSymlinks = function(staticFn, root, options) {
|
||||
const expressStatic = staticFn(root, options);
|
||||
const basePath = Path.resolve(root);
|
||||
return function(req, res, next) {
|
||||
let file, project_id, result;
|
||||
const path = __guard__(url.parse(req.url), x => x.pathname);
|
||||
// check that the path is of the form /project_id_or_name/path/to/file.log
|
||||
if (result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/)) {
|
||||
project_id = result[1];
|
||||
file = result[2];
|
||||
} else {
|
||||
logger.warn({path}, "unrecognized file request");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
// check that the file does not use a relative path
|
||||
for (let dir of Array.from(file.split('/'))) {
|
||||
if (dir === '..') {
|
||||
logger.warn({path}, "attempt to use a relative path");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
}
|
||||
// check that the requested path is normalized
|
||||
const requestedFsPath = `${basePath}/${project_id}/${file}`;
|
||||
if (requestedFsPath !== Path.normalize(requestedFsPath)) {
|
||||
logger.error({path: requestedFsPath}, "requestedFsPath is not normalized");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
// check that the requested path is not a symlink
|
||||
return fs.realpath(requestedFsPath, function(err, realFsPath){
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') {
|
||||
return res.sendStatus(404);
|
||||
} else {
|
||||
logger.error({err, requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "error checking file access");
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
} else if (requestedFsPath !== realFsPath) {
|
||||
logger.warn({requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "trying to access a different file (symlink), aborting");
|
||||
return res.sendStatus(404);
|
||||
} else {
|
||||
return expressStatic(req, res, next);
|
||||
}
|
||||
});
|
||||
};
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
|
|
@ -1,37 +1,56 @@
|
|||
fs = require "fs"
|
||||
Path = require "path"
|
||||
ResourceWriter = require "./ResourceWriter"
|
||||
SafeReader = require "./SafeReader"
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let TikzManager;
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const ResourceWriter = require("./ResourceWriter");
|
||||
const SafeReader = require("./SafeReader");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
# for \tikzexternalize or pstool to work the main file needs to match the
|
||||
# jobname. Since we set the -jobname to output, we have to create a
|
||||
# copy of the main file as 'output.tex'.
|
||||
// for \tikzexternalize or pstool to work the main file needs to match the
|
||||
// jobname. Since we set the -jobname to output, we have to create a
|
||||
// copy of the main file as 'output.tex'.
|
||||
|
||||
module.exports = TikzManager =
|
||||
module.exports = (TikzManager = {
|
||||
|
||||
checkMainFile: (compileDir, mainFile, resources, callback = (error, needsMainFile) ->) ->
|
||||
# if there's already an output.tex file, we don't want to touch it
|
||||
for resource in resources
|
||||
if resource.path is "output.tex"
|
||||
logger.log compileDir: compileDir, mainFile: mainFile, "output.tex already in resources"
|
||||
return callback(null, false)
|
||||
# if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
|
||||
ResourceWriter.checkPath compileDir, mainFile, (error, path) ->
|
||||
return callback(error) if error?
|
||||
SafeReader.readFile path, 65536, "utf8", (error, content) ->
|
||||
return callback(error) if error?
|
||||
usesTikzExternalize = content?.indexOf("\\tikzexternalize") >= 0
|
||||
usesPsTool = content?.indexOf("{pstool}") >= 0
|
||||
logger.log compileDir: compileDir, mainFile: mainFile, usesTikzExternalize:usesTikzExternalize, usesPsTool: usesPsTool, "checked for packages needing main file as output.tex"
|
||||
needsMainFile = (usesTikzExternalize || usesPsTool)
|
||||
callback null, needsMainFile
|
||||
checkMainFile(compileDir, mainFile, resources, callback) {
|
||||
// if there's already an output.tex file, we don't want to touch it
|
||||
if (callback == null) { callback = function(error, needsMainFile) {}; }
|
||||
for (let resource of Array.from(resources)) {
|
||||
if (resource.path === "output.tex") {
|
||||
logger.log({compileDir, mainFile}, "output.tex already in resources");
|
||||
return callback(null, false);
|
||||
}
|
||||
}
|
||||
// if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return SafeReader.readFile(path, 65536, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
const usesTikzExternalize = (content != null ? content.indexOf("\\tikzexternalize") : undefined) >= 0;
|
||||
const usesPsTool = (content != null ? content.indexOf("{pstool}") : undefined) >= 0;
|
||||
logger.log({compileDir, mainFile, usesTikzExternalize, usesPsTool}, "checked for packages needing main file as output.tex");
|
||||
const needsMainFile = (usesTikzExternalize || usesPsTool);
|
||||
return callback(null, needsMainFile);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
injectOutputFile: (compileDir, mainFile, callback = (error) ->) ->
|
||||
ResourceWriter.checkPath compileDir, mainFile, (error, path) ->
|
||||
return callback(error) if error?
|
||||
fs.readFile path, "utf8", (error, content) ->
|
||||
return callback(error) if error?
|
||||
logger.log compileDir: compileDir, mainFile: mainFile, "copied file to output.tex as project uses packages which require it"
|
||||
# use wx flag to ensure that output file does not already exist
|
||||
fs.writeFile Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback
|
||||
injectOutputFile(compileDir, mainFile, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.readFile(path, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({compileDir, mainFile}, "copied file to output.tex as project uses packages which require it");
|
||||
// use wx flag to ensure that output file does not already exist
|
||||
return fs.writeFile(Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,134 +1,189 @@
|
|||
db = require("./db")
|
||||
dbQueue = require "./DbQueue"
|
||||
UrlFetcher = require("./UrlFetcher")
|
||||
Settings = require("settings-sharelatex")
|
||||
crypto = require("crypto")
|
||||
fs = require("fs")
|
||||
logger = require "logger-sharelatex"
|
||||
async = require "async"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlCache;
|
||||
const db = require("./db");
|
||||
const dbQueue = require("./DbQueue");
|
||||
const UrlFetcher = require("./UrlFetcher");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const crypto = require("crypto");
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const async = require("async");
|
||||
|
||||
module.exports = UrlCache =
|
||||
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
|
||||
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
|
||||
return callback(error) if error?
|
||||
UrlCache._copyFile pathToCachedUrl, destPath, (error) ->
|
||||
if error?
|
||||
UrlCache._clearUrlDetails project_id, url, () ->
|
||||
callback(error)
|
||||
else
|
||||
callback(error)
|
||||
module.exports = (UrlCache = {
|
||||
downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._ensureUrlIsInCache(project_id, url, lastModified, (error, pathToCachedUrl) => {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) {
|
||||
if (error != null) {
|
||||
return UrlCache._clearUrlDetails(project_id, url, () => callback(error));
|
||||
} else {
|
||||
return callback(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
clearProject: (project_id, callback = (error) ->) ->
|
||||
UrlCache._findAllUrlsInProject project_id, (error, urls) ->
|
||||
logger.log project_id: project_id, url_count: urls.length, "clearing project URLs"
|
||||
return callback(error) if error?
|
||||
jobs = for url in (urls or [])
|
||||
do (url) ->
|
||||
(callback) ->
|
||||
UrlCache._clearUrlFromCache project_id, url, (error) ->
|
||||
if error?
|
||||
logger.error err: error, project_id: project_id, url: url, "error clearing project URL"
|
||||
callback()
|
||||
async.series jobs, callback
|
||||
clearProject(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._findAllUrlsInProject(project_id, function(error, urls) {
|
||||
logger.log({project_id, url_count: urls.length}, "clearing project URLs");
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = (Array.from(urls || [])).map((url) =>
|
||||
(url =>
|
||||
callback =>
|
||||
UrlCache._clearUrlFromCache(project_id, url, function(error) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, project_id, url}, "error clearing project URL");
|
||||
}
|
||||
return callback();
|
||||
})
|
||||
|
||||
)(url));
|
||||
return async.series(jobs, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_ensureUrlIsInCache: (project_id, url, lastModified, callback = (error, pathOnDisk) ->) ->
|
||||
if lastModified?
|
||||
# MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
|
||||
# So round down to seconds
|
||||
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
|
||||
UrlCache._doesUrlNeedDownloading project_id, url, lastModified, (error, needsDownloading) =>
|
||||
return callback(error) if error?
|
||||
if needsDownloading
|
||||
logger.log url: url, lastModified: lastModified, "downloading URL"
|
||||
UrlFetcher.pipeUrlToFile url, UrlCache._cacheFilePathForUrl(project_id, url), (error) =>
|
||||
return callback(error) if error?
|
||||
UrlCache._updateOrCreateUrlDetails project_id, url, lastModified, (error) =>
|
||||
return callback(error) if error?
|
||||
callback null, UrlCache._cacheFilePathForUrl(project_id, url)
|
||||
else
|
||||
logger.log url: url, lastModified: lastModified, "URL is up to date in cache"
|
||||
callback null, UrlCache._cacheFilePathForUrl(project_id, url)
|
||||
_ensureUrlIsInCache(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error, pathOnDisk) {}; }
|
||||
if (lastModified != null) {
|
||||
// MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
|
||||
// So round down to seconds
|
||||
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000);
|
||||
}
|
||||
return UrlCache._doesUrlNeedDownloading(project_id, url, lastModified, (error, needsDownloading) => {
|
||||
if (error != null) { return callback(error); }
|
||||
if (needsDownloading) {
|
||||
logger.log({url, lastModified}, "downloading URL");
|
||||
return UrlFetcher.pipeUrlToFile(url, UrlCache._cacheFilePathForUrl(project_id, url), error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._updateOrCreateUrlDetails(project_id, url, lastModified, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.log({url, lastModified}, "URL is up to date in cache");
|
||||
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
|
||||
if !lastModified?
|
||||
return callback null, true
|
||||
UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
|
||||
return callback(error) if error?
|
||||
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
|
||||
return callback null, true
|
||||
else
|
||||
return callback null, false
|
||||
_doesUrlNeedDownloading(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error, needsDownloading) {}; }
|
||||
if ((lastModified == null)) {
|
||||
return callback(null, true);
|
||||
}
|
||||
return UrlCache._findUrlDetails(project_id, url, function(error, urlDetails) {
|
||||
if (error != null) { return callback(error); }
|
||||
if ((urlDetails == null) || (urlDetails.lastModified == null) || (urlDetails.lastModified.getTime() < lastModified.getTime())) {
|
||||
return callback(null, true);
|
||||
} else {
|
||||
return callback(null, false);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_cacheFileNameForUrl: (project_id, url) ->
|
||||
project_id + ":" + crypto.createHash("md5").update(url).digest("hex")
|
||||
_cacheFileNameForUrl(project_id, url) {
|
||||
return project_id + ":" + crypto.createHash("md5").update(url).digest("hex");
|
||||
},
|
||||
|
||||
_cacheFilePathForUrl: (project_id, url) ->
|
||||
"#{Settings.path.clsiCacheDir}/#{UrlCache._cacheFileNameForUrl(project_id, url)}"
|
||||
_cacheFilePathForUrl(project_id, url) {
|
||||
return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(project_id, url)}`;
|
||||
},
|
||||
|
||||
_copyFile: (from, to, _callback = (error) ->) ->
|
||||
callbackOnce = (error) ->
|
||||
if error?
|
||||
logger.error err: error, from:from, to:to, "error copying file from cache"
|
||||
_callback(error)
|
||||
_callback = () ->
|
||||
writeStream = fs.createWriteStream(to)
|
||||
readStream = fs.createReadStream(from)
|
||||
writeStream.on "error", callbackOnce
|
||||
readStream.on "error", callbackOnce
|
||||
writeStream.on "close", callbackOnce
|
||||
writeStream.on "open", () ->
|
||||
readStream.pipe(writeStream)
|
||||
_copyFile(from, to, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callbackOnce = function(error) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, from, to}, "error copying file from cache");
|
||||
}
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
const writeStream = fs.createWriteStream(to);
|
||||
const readStream = fs.createReadStream(from);
|
||||
writeStream.on("error", callbackOnce);
|
||||
readStream.on("error", callbackOnce);
|
||||
writeStream.on("close", callbackOnce);
|
||||
return writeStream.on("open", () => readStream.pipe(writeStream));
|
||||
},
|
||||
|
||||
_clearUrlFromCache: (project_id, url, callback = (error) ->) ->
|
||||
UrlCache._clearUrlDetails project_id, url, (error) ->
|
||||
return callback(error) if error?
|
||||
UrlCache._deleteUrlCacheFromDisk project_id, url, (error) ->
|
||||
return callback(error) if error?
|
||||
callback null
|
||||
_clearUrlFromCache(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._clearUrlDetails(project_id, url, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_deleteUrlCacheFromDisk: (project_id, url, callback = (error) ->) ->
|
||||
fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), (error) ->
|
||||
if error? and error.code != 'ENOENT' # no error if the file isn't present
|
||||
return callback(error)
|
||||
else
|
||||
return callback()
|
||||
_deleteUrlCacheFromDisk(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(error) {
|
||||
if ((error != null) && (error.code !== 'ENOENT')) { // no error if the file isn't present
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
|
||||
job = (cb)->
|
||||
db.UrlCache.find(where: { url: url, project_id: project_id })
|
||||
.then((urlDetails) -> cb null, urlDetails)
|
||||
.error cb
|
||||
dbQueue.queue.push job, callback
|
||||
_findUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error, urlDetails) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.find({where: { url, project_id }})
|
||||
.then(urlDetails => cb(null, urlDetails))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
|
||||
job = (cb)->
|
||||
db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
|
||||
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.findOrCreate({where: {url, project_id}})
|
||||
.spread(
|
||||
(urlDetails, created) ->
|
||||
urlDetails.updateAttributes(lastModified: lastModified)
|
||||
.then(() -> cb())
|
||||
(urlDetails, created) =>
|
||||
urlDetails.updateAttributes({lastModified})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error cb
|
||||
dbQueue.queue.push(job, callback)
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
_clearUrlDetails: (project_id, url, callback = (error) ->) ->
|
||||
job = (cb)->
|
||||
db.UrlCache.destroy(where: {url: url, project_id: project_id})
|
||||
.then(() -> cb null)
|
||||
.error cb
|
||||
dbQueue.queue.push(job, callback)
|
||||
_clearUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.destroy({where: {url, project_id}})
|
||||
.then(() => cb(null))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
|
||||
job = (cb)->
|
||||
db.UrlCache.findAll(where: { project_id: project_id })
|
||||
_findAllUrlsInProject(project_id, callback) {
|
||||
if (callback == null) { callback = function(error, urls) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.findAll({where: { project_id }})
|
||||
.then(
|
||||
(urlEntries) ->
|
||||
cb null, urlEntries.map((entry) -> entry.url)
|
||||
)
|
||||
.error cb
|
||||
dbQueue.queue.push(job, callback)
|
||||
urlEntries => cb(null, urlEntries.map(entry => entry.url)))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,70 +1,88 @@
|
|||
request = require("request").defaults(jar: false)
|
||||
fs = require("fs")
|
||||
logger = require "logger-sharelatex"
|
||||
settings = require("settings-sharelatex")
|
||||
URL = require('url');
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlFetcher;
|
||||
const request = require("request").defaults({jar: false});
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
const URL = require('url');
|
||||
|
||||
oneMinute = 60 * 1000
|
||||
const oneMinute = 60 * 1000;
|
||||
|
||||
module.exports = UrlFetcher =
|
||||
pipeUrlToFile: (url, filePath, _callback = (error) ->) ->
|
||||
callbackOnce = (error) ->
|
||||
clearTimeout timeoutHandler if timeoutHandler?
|
||||
_callback(error)
|
||||
_callback = () ->
|
||||
module.exports = (UrlFetcher = {
|
||||
pipeUrlToFile(url, filePath, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callbackOnce = function(error) {
|
||||
if (timeoutHandler != null) { clearTimeout(timeoutHandler); }
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
if settings.filestoreDomainOveride?
|
||||
p = URL.parse(url).path
|
||||
url = "#{settings.filestoreDomainOveride}#{p}"
|
||||
timeoutHandler = setTimeout () ->
|
||||
timeoutHandler = null
|
||||
logger.error url:url, filePath: filePath, "Timed out downloading file to cache"
|
||||
callbackOnce(new Error("Timed out downloading file to cache #{url}"))
|
||||
# FIXME: maybe need to close fileStream here
|
||||
, 3 * oneMinute
|
||||
if (settings.filestoreDomainOveride != null) {
|
||||
const p = URL.parse(url).path;
|
||||
url = `${settings.filestoreDomainOveride}${p}`;
|
||||
}
|
||||
var timeoutHandler = setTimeout(function() {
|
||||
timeoutHandler = null;
|
||||
logger.error({url, filePath}, "Timed out downloading file to cache");
|
||||
return callbackOnce(new Error(`Timed out downloading file to cache ${url}`));
|
||||
}
|
||||
// FIXME: maybe need to close fileStream here
|
||||
, 3 * oneMinute);
|
||||
|
||||
logger.log url:url, filePath: filePath, "started downloading url to cache"
|
||||
urlStream = request.get({url: url, timeout: oneMinute})
|
||||
urlStream.pause() # stop data flowing until we are ready
|
||||
logger.log({url, filePath}, "started downloading url to cache");
|
||||
const urlStream = request.get({url, timeout: oneMinute});
|
||||
urlStream.pause(); // stop data flowing until we are ready
|
||||
|
||||
# attach handlers before setting up pipes
|
||||
urlStream.on "error", (error) ->
|
||||
logger.error err: error, url:url, filePath: filePath, "error downloading url"
|
||||
callbackOnce(error or new Error("Something went wrong downloading the URL #{url}"))
|
||||
// attach handlers before setting up pipes
|
||||
urlStream.on("error", function(error) {
|
||||
logger.error({err: error, url, filePath}, "error downloading url");
|
||||
return callbackOnce(error || new Error(`Something went wrong downloading the URL ${url}`));
|
||||
});
|
||||
|
||||
urlStream.on "end", () ->
|
||||
logger.log url:url, filePath: filePath, "finished downloading file into cache"
|
||||
urlStream.on("end", () => logger.log({url, filePath}, "finished downloading file into cache"));
|
||||
|
||||
urlStream.on "response", (res) ->
|
||||
if res.statusCode >= 200 and res.statusCode < 300
|
||||
fileStream = fs.createWriteStream(filePath)
|
||||
return urlStream.on("response", function(res) {
|
||||
if ((res.statusCode >= 200) && (res.statusCode < 300)) {
|
||||
const fileStream = fs.createWriteStream(filePath);
|
||||
|
||||
# attach handlers before setting up pipes
|
||||
fileStream.on 'error', (error) ->
|
||||
logger.error err: error, url:url, filePath: filePath, "error writing file into cache"
|
||||
fs.unlink filePath, (err) ->
|
||||
if err?
|
||||
logger.err err: err, filePath: filePath, "error deleting file from cache"
|
||||
callbackOnce(error)
|
||||
// attach handlers before setting up pipes
|
||||
fileStream.on('error', function(error) {
|
||||
logger.error({err: error, url, filePath}, "error writing file into cache");
|
||||
return fs.unlink(filePath, function(err) {
|
||||
if (err != null) {
|
||||
logger.err({err, filePath}, "error deleting file from cache");
|
||||
}
|
||||
return callbackOnce(error);
|
||||
});
|
||||
});
|
||||
|
||||
fileStream.on 'finish', () ->
|
||||
logger.log url:url, filePath: filePath, "finished writing file into cache"
|
||||
callbackOnce()
|
||||
fileStream.on('finish', function() {
|
||||
logger.log({url, filePath}, "finished writing file into cache");
|
||||
return callbackOnce();
|
||||
});
|
||||
|
||||
fileStream.on 'pipe', () ->
|
||||
logger.log url:url, filePath: filePath, "piping into filestream"
|
||||
fileStream.on('pipe', () => logger.log({url, filePath}, "piping into filestream"));
|
||||
|
||||
urlStream.pipe(fileStream)
|
||||
urlStream.resume() # now we are ready to handle the data
|
||||
else
|
||||
logger.error statusCode: res.statusCode, url:url, filePath: filePath, "unexpected status code downloading url to cache"
|
||||
# https://nodejs.org/api/http.html#http_class_http_clientrequest
|
||||
# If you add a 'response' event handler, then you must consume
|
||||
# the data from the response object, either by calling
|
||||
# response.read() whenever there is a 'readable' event, or by
|
||||
# adding a 'data' handler, or by calling the .resume()
|
||||
# method. Until the data is consumed, the 'end' event will not
|
||||
# fire. Also, until the data is read it will consume memory
|
||||
# that can eventually lead to a 'process out of memory' error.
|
||||
urlStream.resume() # discard the data
|
||||
callbackOnce(new Error("URL returned non-success status code: #{res.statusCode} #{url}"))
|
||||
urlStream.pipe(fileStream);
|
||||
return urlStream.resume(); // now we are ready to handle the data
|
||||
} else {
|
||||
logger.error({statusCode: res.statusCode, url, filePath}, "unexpected status code downloading url to cache");
|
||||
// https://nodejs.org/api/http.html#http_class_http_clientrequest
|
||||
// If you add a 'response' event handler, then you must consume
|
||||
// the data from the response object, either by calling
|
||||
// response.read() whenever there is a 'readable' event, or by
|
||||
// adding a 'data' handler, or by calling the .resume()
|
||||
// method. Until the data is consumed, the 'end' event will not
|
||||
// fire. Also, until the data is read it will consume memory
|
||||
// that can eventually lead to a 'process out of memory' error.
|
||||
urlStream.resume(); // discard the data
|
||||
return callbackOnce(new Error(`URL returned non-success status code: ${res.statusCode} ${url}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,55 +1,59 @@
|
|||
Sequelize = require("sequelize")
|
||||
Settings = require("settings-sharelatex")
|
||||
_ = require("underscore")
|
||||
logger = require "logger-sharelatex"
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const Sequelize = require("sequelize");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const _ = require("underscore");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
options = _.extend {logging:false}, Settings.mysql.clsi
|
||||
const options = _.extend({logging:false}, Settings.mysql.clsi);
|
||||
|
||||
logger.log dbPath:Settings.mysql.clsi.storage, "connecting to db"
|
||||
logger.log({dbPath:Settings.mysql.clsi.storage}, "connecting to db");
|
||||
|
||||
sequelize = new Sequelize(
|
||||
const sequelize = new Sequelize(
|
||||
Settings.mysql.clsi.database,
|
||||
Settings.mysql.clsi.username,
|
||||
Settings.mysql.clsi.password,
|
||||
options
|
||||
)
|
||||
);
|
||||
|
||||
if Settings.mysql.clsi.dialect == "sqlite"
|
||||
logger.log "running PRAGMA journal_mode=WAL;"
|
||||
sequelize.query("PRAGMA journal_mode=WAL;")
|
||||
sequelize.query("PRAGMA synchronous=OFF;")
|
||||
sequelize.query("PRAGMA read_uncommitted = true;")
|
||||
if (Settings.mysql.clsi.dialect === "sqlite") {
|
||||
logger.log("running PRAGMA journal_mode=WAL;");
|
||||
sequelize.query("PRAGMA journal_mode=WAL;");
|
||||
sequelize.query("PRAGMA synchronous=OFF;");
|
||||
sequelize.query("PRAGMA read_uncommitted = true;");
|
||||
}
|
||||
|
||||
module.exports =
|
||||
module.exports = {
|
||||
UrlCache: sequelize.define("UrlCache", {
|
||||
url: Sequelize.STRING
|
||||
project_id: Sequelize.STRING
|
||||
url: Sequelize.STRING,
|
||||
project_id: Sequelize.STRING,
|
||||
lastModified: Sequelize.DATE
|
||||
}, {
|
||||
indexes: [
|
||||
{fields: ['url', 'project_id']},
|
||||
{fields: ['project_id']}
|
||||
]
|
||||
})
|
||||
}),
|
||||
|
||||
Project: sequelize.define("Project", {
|
||||
project_id: {type: Sequelize.STRING, primaryKey: true}
|
||||
project_id: {type: Sequelize.STRING, primaryKey: true},
|
||||
lastAccessed: Sequelize.DATE
|
||||
}, {
|
||||
indexes: [
|
||||
{fields: ['lastAccessed']}
|
||||
]
|
||||
})
|
||||
}),
|
||||
|
||||
op: Sequelize.Op
|
||||
op: Sequelize.Op,
|
||||
|
||||
sync: () ->
|
||||
logger.log dbPath:Settings.mysql.clsi.storage, "syncing db schema"
|
||||
sequelize.sync()
|
||||
.then(->
|
||||
logger.log "db sync complete"
|
||||
).catch((err)->
|
||||
console.log err, "error syncing"
|
||||
)
|
||||
sync() {
|
||||
logger.log({dbPath:Settings.mysql.clsi.storage}, "syncing db schema");
|
||||
return sequelize.sync()
|
||||
.then(() => logger.log("db sync complete")).catch(err=> console.log(err, "error syncing"));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue