From 2feea2592d8e73c93e0e42cc689fec1cf575fec2 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Mon, 30 Aug 2021 07:56:13 -0400 Subject: [PATCH] Merge pull request #4887 from overleaf/em-decaf Decaf cleanup for the CLSI CompileManager GitOrigin-RevId: 06bba5c8af8808d0fa04187b10c8f31e08cd8754 --- services/clsi/app/js/CompileManager.js | 1324 ++++++++--------- .../clsi/test/unit/js/CompileManagerTests.js | 496 +++--- 2 files changed, 839 insertions(+), 981 deletions(-) diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 07a9033b3d..66ef5caa86 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -1,21 +1,3 @@ -/* eslint-disable - camelcase, - handle-callback-err, - no-return-assign, - no-undef, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let CompileManager const ResourceWriter = require('./ResourceWriter') const LatexRunner = require('./LatexRunner') const OutputFileFinder = require('./OutputFileFinder') @@ -24,7 +6,7 @@ const Settings = require('@overleaf/settings') const Path = require('path') const logger = require('logger-sharelatex') const Metrics = require('./Metrics') -const child_process = require('child_process') +const childProcess = require('child_process') const DraftModeManager = require('./DraftModeManager') const TikzManager = require('./TikzManager') const LockManager = require('./LockManager') @@ -36,726 +18,654 @@ const Errors = require('./Errors') const CommandRunner = require('./CommandRunner') const { emitPdfStats } = require('./ContentCacheMetrics') -const getCompileName = function (project_id, user_id) { - if (user_id != null) { - return `${project_id}-${user_id}` +function getCompileName(projectId, userId) { + if (userId != null) { + return `${projectId}-${userId}` } else { - return project_id + return projectId } } -const getCompileDir = (project_id, user_id) => - Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) +function getCompileDir(projectId, userId) { + return Path.join(Settings.path.compilesDir, getCompileName(projectId, userId)) +} -const getOutputDir = (project_id, user_id) => - Path.join(Settings.path.outputDir, getCompileName(project_id, user_id)) +function getOutputDir(projectId, userId) { + return Path.join(Settings.path.outputDir, getCompileName(projectId, userId)) +} -module.exports = CompileManager = { - doCompileWithLock(request, callback) { - if (callback == null) { - callback = function (error, outputFiles) {} +function doCompileWithLock(request, callback) { + const compileDir = getCompileDir(request.project_id, request.user_id) + const lockFile = Path.join(compileDir, '.project-lock') + // use a .project-lock file in the compile directory to prevent + // simultaneous compiles + fse.ensureDir(compileDir, error => { + if (error) { + return callback(error) } - const compileDir = getCompileDir(request.project_id, request.user_id) - const lockFile = Path.join(compileDir, '.project-lock') - // use a .project-lock file in the compile directory to prevent - // simultaneous compiles - return fse.ensureDir(compileDir, function (error) { - if (error != null) { + LockManager.runWithLock( + lockFile, + releaseLock => doCompile(request, releaseLock), + callback + ) + }) +} + +function doCompile(request, callback) { + const compileDir = getCompileDir(request.project_id, request.user_id) + const outputDir = getOutputDir(request.project_id, request.user_id) + + const timerE2E = new Metrics.Timer('compile-e2e') + const timer = new Metrics.Timer('write-to-disk') + logger.log( + { projectId: request.project_id, userId: request.user_id }, + 'syncing resources to disk' + ) + ResourceWriter.syncResourcesToDisk( + request, + compileDir, + (error, resourceList) => { + // NOTE: resourceList is insecure, it should only be used to exclude files from the output list + if (error && error instanceof Errors.FilesOutOfSyncError) { + logger.warn( + { projectId: request.project_id, userId: request.user_id }, + 'files out of sync, please retry' + ) + return callback(error) + } else if (error) { + logger.err( + { + err: error, + projectId: request.project_id, + userId: request.user_id, + }, + 'error writing resources to disk' + ) return callback(error) } - return LockManager.runWithLock( - lockFile, - releaseLock => CompileManager.doCompile(request, releaseLock), - callback + logger.log( + { + projectId: request.project_id, + userId: request.user_id, + time_taken: Date.now() - timer.start, + }, + 'written files to disk' ) - }) - }, + const syncStage = timer.done() - doCompile(request, callback) { - if (callback == null) { - callback = function (error, outputFiles) {} - } - const compileDir = getCompileDir(request.project_id, request.user_id) - const outputDir = getOutputDir(request.project_id, request.user_id) - - const timerE2E = new Metrics.Timer('compile-e2e') - let timer = new Metrics.Timer('write-to-disk') - logger.log( - { project_id: request.project_id, user_id: request.user_id }, - 'syncing resources to disk' - ) - return ResourceWriter.syncResourcesToDisk( - request, - compileDir, - function (error, resourceList) { - // NOTE: resourceList is insecure, it should only be used to exclude files from the output list - if (error != null && error instanceof Errors.FilesOutOfSyncError) { - logger.warn( - { project_id: request.project_id, user_id: request.user_id }, - 'files out of sync, please retry' + function injectDraftModeIfRequired(callback) { + if (request.draft) { + DraftModeManager.injectDraftMode( + Path.join(compileDir, request.rootResourcePath), + callback ) - return callback(error) - } else if (error != null) { - logger.err( - { - err: error, - project_id: request.project_id, - user_id: request.user_id, - }, - 'error writing resources to disk' - ) - return callback(error) + } else { + callback() } - logger.log( - { - project_id: request.project_id, - user_id: request.user_id, - time_taken: Date.now() - timer.start, - }, - 'written files to disk' - ) - const syncStage = timer.done() + } - const injectDraftModeIfRequired = function (callback) { - if (request.draft) { - return DraftModeManager.injectDraftMode( - Path.join(compileDir, request.rootResourcePath), - callback - ) - } else { - return callback() - } - } - - const createTikzFileIfRequired = callback => - TikzManager.checkMainFile( - compileDir, - request.rootResourcePath, - resourceList, - function (error, needsMainFile) { - if (error != null) { - return callback(error) - } - if (needsMainFile) { - return TikzManager.injectOutputFile( - compileDir, - request.rootResourcePath, - callback - ) - } else { - return callback() - } - } - ) - // set up environment variables for chktex - const env = {} - if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') { - // override default texlive openout_any environment variable - env.openout_any = Settings.texliveOpenoutAny - } - // only run chktex on LaTeX files (not knitr .Rtex files or any others) - const isLaTeXFile = - request.rootResourcePath != null - ? request.rootResourcePath.match(/\.tex$/i) - : undefined - if (request.check != null && isLaTeXFile) { - env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16' - env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000' - if (request.check === 'error') { - env.CHKTEX_EXIT_ON_ERROR = 1 - } - if (request.check === 'validate') { - env.CHKTEX_VALIDATE = 1 - } - } - - // apply a series of file modifications/creations for draft mode and tikz - return async.series( - [injectDraftModeIfRequired, createTikzFileIfRequired], - function (error) { - if (error != null) { + const createTikzFileIfRequired = callback => + TikzManager.checkMainFile( + compileDir, + request.rootResourcePath, + resourceList, + (error, needsMainFile) => { + if (error) { return callback(error) } - timer = new Metrics.Timer('run-compile') - // find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite) - let tag = - __guard__( - __guard__( - request.imageName != null - ? request.imageName.match(/:(.*)/) - : undefined, - x1 => x1[1] - ), - x => x.replace(/\./g, '-') - ) || 'default' - if (!request.project_id.match(/^[0-9a-f]{24}$/)) { - tag = 'other' - } // exclude smoke test - Metrics.inc('compiles') - Metrics.inc(`compiles-with-image.${tag}`) - const compileName = getCompileName( - request.project_id, - request.user_id - ) - return LatexRunner.runLatex( - compileName, - { - directory: compileDir, - mainFile: request.rootResourcePath, - compiler: request.compiler, - timeout: request.timeout, - image: request.imageName, - flags: request.flags, - environment: env, - compileGroup: request.compileGroup, - }, - function (error, output, stats, timings) { - // request was for validation only - let metric_key, metric_value - if (request.check === 'validate') { - const result = (error != null ? error.code : undefined) - ? 'fail' - : 'pass' - error = new Error('validation') - error.validate = result - } - // request was for compile, and failed on validation - if ( - request.check === 'error' && - (error != null ? error.message : undefined) === 'exited' - ) { - error = new Error('compilation') - error.validate = 'fail' - } - // compile was killed by user, was a validation, or a compile which failed validation - if ( - (error != null ? error.terminated : undefined) || - (error != null ? error.validate : undefined) || - (error != null ? error.timedout : undefined) - ) { - OutputFileFinder.findOutputFiles( - resourceList, - compileDir, - function (err, outputFiles) { - if (err != null) { - return callback(err) - } - error.outputFiles = outputFiles // return output files so user can check logs - return callback(error) - } - ) - return - } - // compile completed normally - if (error != null) { - return callback(error) - } - Metrics.inc('compiles-succeeded') - stats = stats || {} - const object = stats || {} - for (metric_key in object) { - metric_value = object[metric_key] - Metrics.count(metric_key, metric_value) - } - timings = timings || {} - const object1 = timings || {} - for (metric_key in object1) { - metric_value = object1[metric_key] - Metrics.timing(metric_key, metric_value) - } - const loadavg = - typeof os.loadavg === 'function' ? os.loadavg() : undefined - if (loadavg != null) { - Metrics.gauge('load-avg', loadavg[0]) - } - const ts = timer.done() - logger.log( - { - project_id: request.project_id, - user_id: request.user_id, - time_taken: ts, - stats, - timings, - loadavg, - }, - 'done compile' - ) - if ((stats != null ? stats['latex-runs'] : undefined) > 0) { - Metrics.timing( - 'run-compile-per-pass', - ts / stats['latex-runs'] - ) - } - if ( - (stats != null ? stats['latex-runs'] : undefined) > 0 && - (timings != null ? timings['cpu-time'] : undefined) > 0 - ) { - Metrics.timing( - 'run-compile-cpu-time-per-pass', - timings['cpu-time'] / stats['latex-runs'] - ) - } - // Emit compile time. - timings.compile = ts - - timer = new Metrics.Timer('process-output-files') + if (needsMainFile) { + TikzManager.injectOutputFile( + compileDir, + request.rootResourcePath, + callback + ) + } else { + callback() + } + } + ) + // set up environment variables for chktex + const env = {} + if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') { + // override default texlive openout_any environment variable + env.openout_any = Settings.texliveOpenoutAny + } + // only run chktex on LaTeX files (not knitr .Rtex files or any others) + const isLaTeXFile = + request.rootResourcePath != null + ? request.rootResourcePath.match(/\.tex$/i) + : undefined + if (request.check != null && isLaTeXFile) { + env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16' + env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000' + if (request.check === 'error') { + env.CHKTEX_EXIT_ON_ERROR = 1 + } + if (request.check === 'validate') { + env.CHKTEX_VALIDATE = 1 + } + } + // apply a series of file modifications/creations for draft mode and tikz + async.series( + [injectDraftModeIfRequired, createTikzFileIfRequired], + error => { + if (error) { + return callback(error) + } + const timer = new Metrics.Timer('run-compile') + // find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite) + let tag = 'default' + if (request.imageName != null) { + const match = request.imageName.match(/:(.*)/) + if (match != null) { + tag = match[1].replace(/\./g, '-') + } + } + if (!request.project_id.match(/^[0-9a-f]{24}$/)) { + tag = 'other' + } // exclude smoke test + Metrics.inc('compiles') + Metrics.inc(`compiles-with-image.${tag}`) + const compileName = getCompileName( + request.project_id, + request.user_id + ) + LatexRunner.runLatex( + compileName, + { + directory: compileDir, + mainFile: request.rootResourcePath, + compiler: request.compiler, + timeout: request.timeout, + image: request.imageName, + flags: request.flags, + environment: env, + compileGroup: request.compileGroup, + }, + (error, output, stats, timings) => { + // request was for validation only + if (request.check === 'validate') { + const result = error && error.code ? 'fail' : 'pass' + error = new Error('validation') + error.validate = result + } + // request was for compile, and failed on validation + if ( + request.check === 'error' && + error && + error.message === 'exited' + ) { + error = new Error('compilation') + error.validate = 'fail' + } + // compile was killed by user, was a validation, or a compile which failed validation + if ( + error && + (error.terminated || error.validate || error.timedout) + ) { return OutputFileFinder.findOutputFiles( resourceList, compileDir, - function (error, outputFiles) { - if (error != null) { - return callback(error) + (err, outputFiles) => { + if (err) { + return callback(err) } - return OutputCacheManager.saveOutputFiles( - { request, stats, timings }, - outputFiles, - compileDir, - outputDir, - (err, newOutputFiles) => { - if (err) { - const { project_id: projectId, user_id: userId } = - request - logger.err( - { projectId, userId, err }, - 'failed to save output files' - ) - } - - const outputStage = timer.done() - timings.sync = syncStage - timings.output = outputStage - - // Emit e2e compile time. - timings.compileE2E = timerE2E.done() - - if (stats['pdf-size']) { - emitPdfStats(stats, timings) - } - - callback(null, newOutputFiles, stats, timings) - } - ) + error.outputFiles = outputFiles // return output files so user can check logs + callback(error) } ) } - ) - } - ) - } - ) - }, - - stopCompile(project_id, user_id, callback) { - if (callback == null) { - callback = function (error) {} - } - const compileName = getCompileName(project_id, user_id) - return LatexRunner.killLatex(compileName, callback) - }, - - clearProject(project_id, user_id, _callback) { - if (_callback == null) { - _callback = function (error) {} - } - const callback = function (error) { - _callback(error) - return (_callback = function () {}) - } - - const compileDir = getCompileDir(project_id, user_id) - const outputDir = getOutputDir(project_id, user_id) - - return CompileManager._checkDirectory(compileDir, function (err, exists) { - if (err != null) { - return callback(err) - } - if (!exists) { - return callback() - } // skip removal if no directory present - - const proc = child_process.spawn('rm', [ - '-r', - '-f', - '--', - compileDir, - outputDir, - ]) - - proc.on('error', callback) - - let stderr = '' - proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk)) - - return proc.on('close', function (code) { - if (code === 0) { - return callback(null) - } else { - return callback( - new Error(`rm -r ${compileDir} ${outputDir} failed: ${stderr}`) - ) - } - }) - }) - }, - - _findAllDirs(callback) { - if (callback == null) { - callback = function (error, allDirs) {} - } - const root = Settings.path.compilesDir - return fs.readdir(root, function (err, files) { - if (err != null) { - return callback(err) - } - const allDirs = Array.from(files).map(file => Path.join(root, file)) - return callback(null, allDirs) - }) - }, - - clearExpiredProjects(max_cache_age_ms, callback) { - if (callback == null) { - callback = function (error) {} - } - const now = Date.now() - // action for each directory - const expireIfNeeded = (checkDir, cb) => - fs.stat(checkDir, function (err, stats) { - if (err != null) { - return cb() - } // ignore errors checking directory - const age = now - stats.mtime - const hasExpired = age > max_cache_age_ms - if (hasExpired) { - return fse.remove(checkDir, cb) - } else { - return cb() - } - }) - // iterate over all project directories - return CompileManager._findAllDirs(function (error, allDirs) { - if (error != null) { - return callback() - } - return async.eachSeries(allDirs, expireIfNeeded, callback) - }) - }, - - _checkDirectory(compileDir, callback) { - if (callback == null) { - callback = function (error, exists) {} - } - return fs.lstat(compileDir, function (err, stats) { - if ((err != null ? err.code : undefined) === 'ENOENT') { - return callback(null, false) // directory does not exist - } else if (err != null) { - logger.err( - { dir: compileDir, err }, - 'error on stat of project directory for removal' - ) - return callback(err) - } else if (!(stats != null ? stats.isDirectory() : undefined)) { - logger.err( - { dir: compileDir, stats }, - 'bad project directory for removal' - ) - return callback(new Error('project directory is not directory')) - } else { - return callback(null, true) - } - }) - }, // directory exists - - syncFromCode( - project_id, - user_id, - file_name, - line, - column, - imageName, - callback - ) { - // If LaTeX was run in a virtual environment, the file path that synctex expects - // might not match the file path on the host. The .synctex.gz file however, will be accessed - // wherever it is on the host. - if (callback == null) { - callback = function (error, pdfPositions) {} - } - const compileName = getCompileName(project_id, user_id) - const base_dir = Settings.path.synctexBaseDir(compileName) - const file_path = base_dir + '/' + file_name - const compileDir = getCompileDir(project_id, user_id) - const synctex_path = `${base_dir}/output.pdf` - const command = ['code', synctex_path, file_path, line, column] - CompileManager._runSynctex( - project_id, - user_id, - command, - imageName, - function (error, stdout) { - if (error != null) { - return callback(error) - } - logger.log( - { project_id, user_id, file_name, line, column, command, stdout }, - 'synctex code output' - ) - return callback( - null, - CompileManager._parseSynctexFromCodeOutput(stdout) - ) - } - ) - }, - - syncFromPdf(project_id, user_id, page, h, v, imageName, callback) { - if (callback == null) { - callback = function (error, filePositions) {} - } - const compileName = getCompileName(project_id, user_id) - const compileDir = getCompileDir(project_id, user_id) - const base_dir = Settings.path.synctexBaseDir(compileName) - const synctex_path = `${base_dir}/output.pdf` - const command = ['pdf', synctex_path, page, h, v] - CompileManager._runSynctex( - project_id, - user_id, - command, - imageName, - function (error, stdout) { - if (error != null) { - return callback(error) - } - logger.log( - { project_id, user_id, page, h, v, stdout }, - 'synctex pdf output' - ) - return callback( - null, - CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) - ) - } - ) - }, - - _checkFileExists(dir, filename, callback) { - if (callback == null) { - callback = function (error) {} - } - const file = Path.join(dir, filename) - return fs.stat(dir, function (error, stats) { - if ((error != null ? error.code : undefined) === 'ENOENT') { - return callback(new Errors.NotFoundError('no output directory')) - } - if (error != null) { - return callback(error) - } - return fs.stat(file, function (error, stats) { - if ((error != null ? error.code : undefined) === 'ENOENT') { - return callback(new Errors.NotFoundError('no output file')) - } - if (error != null) { - return callback(error) - } - if (!(stats != null ? stats.isFile() : undefined)) { - return callback(new Error('not a file')) - } - return callback() - }) - }) - }, - - _runSynctex(project_id, user_id, command, imageName, callback) { - if (callback == null) { - callback = function (error, stdout) {} - } - const seconds = 1000 - - command.unshift('/opt/synctex') - - const directory = getCompileDir(project_id, user_id) - const timeout = 60 * 1000 // increased to allow for large projects - const compileName = getCompileName(project_id, user_id) - const compileGroup = 'synctex' - CompileManager._checkFileExists(directory, 'output.synctex.gz', error => { - if (error) { - return callback(error) - } - return CommandRunner.run( - compileName, - command, - directory, - imageName || - (Settings.clsi && Settings.clsi.docker - ? Settings.clsi.docker.image - : undefined), - timeout, - {}, - compileGroup, - function (error, output) { - if (error != null) { - logger.err( - { err: error, command, project_id, user_id }, - 'error running synctex' - ) - return callback(error) - } - return callback(null, output.stdout) - } - ) - }) - }, - - _parseSynctexFromCodeOutput(output) { - const results = [] - for (const line of Array.from(output.split('\n'))) { - const [node, page, h, v, width, height] = Array.from(line.split('\t')) - if (node === 'NODE') { - results.push({ - page: parseInt(page, 10), - h: parseFloat(h), - v: parseFloat(v), - height: parseFloat(height), - width: parseFloat(width), - }) - } - } - return results - }, - - _parseSynctexFromPdfOutput(output, base_dir) { - const results = [] - for (let line of Array.from(output.split('\n'))) { - let column, file_path, node - ;[node, file_path, line, column] = Array.from(line.split('\t')) - if (node === 'NODE') { - const file = file_path.slice(base_dir.length + 1) - results.push({ - file, - line: parseInt(line, 10), - column: parseInt(column, 10), - }) - } - } - return results - }, - - wordcount(project_id, user_id, file_name, image, callback) { - if (callback == null) { - callback = function (error, pdfPositions) {} - } - logger.log({ project_id, user_id, file_name, image }, 'running wordcount') - const file_path = `$COMPILE_DIR/${file_name}` - const command = [ - 'texcount', - '-nocol', - '-inc', - file_path, - `-out=${file_path}.wc`, - ] - const compileDir = getCompileDir(project_id, user_id) - const timeout = 60 * 1000 - const compileName = getCompileName(project_id, user_id) - const compileGroup = 'wordcount' - return fse.ensureDir(compileDir, function (error) { - if (error != null) { - logger.err( - { error, project_id, user_id, file_name }, - 'error ensuring dir for sync from code' - ) - return callback(error) - } - return CommandRunner.run( - compileName, - command, - compileDir, - image, - timeout, - {}, - compileGroup, - function (error) { - if (error != null) { - return callback(error) - } - return fs.readFile( - compileDir + '/' + file_name + '.wc', - 'utf-8', - function (err, stdout) { - if (err != null) { - // call it node_err so sentry doesn't use random path error as unique id so it can't be ignored - logger.err( - { node_err: err, command, compileDir, project_id, user_id }, - 'error reading word count output' - ) - return callback(err) + // compile completed normally + if (error) { + return callback(error) } - const results = CompileManager._parseWordcountFromOutput(stdout) + Metrics.inc('compiles-succeeded') + stats = stats || {} + for (const metricKey in stats) { + const metricValue = stats[metricKey] + Metrics.count(metricKey, metricValue) + } + timings = timings || {} + for (const metricKey in timings) { + const metricValue = timings[metricKey] + Metrics.timing(metricKey, metricValue) + } + const loadavg = + typeof os.loadavg === 'function' ? os.loadavg() : undefined + if (loadavg != null) { + Metrics.gauge('load-avg', loadavg[0]) + } + const ts = timer.done() logger.log( - { project_id, user_id, wordcount: results }, - 'word count results' + { + projectId: request.project_id, + userId: request.user_id, + time_taken: ts, + stats, + timings, + loadavg, + }, + 'done compile' + ) + if (stats['latex-runs'] > 0) { + Metrics.timing('run-compile-per-pass', ts / stats['latex-runs']) + } + if (stats['latex-runs'] > 0 && timings['cpu-time'] > 0) { + Metrics.timing( + 'run-compile-cpu-time-per-pass', + timings['cpu-time'] / stats['latex-runs'] + ) + } + // Emit compile time. + timings.compile = ts + + const outputStageTimer = new Metrics.Timer('process-output-files') + + OutputFileFinder.findOutputFiles( + resourceList, + compileDir, + (error, outputFiles) => { + if (error) { + return callback(error) + } + OutputCacheManager.saveOutputFiles( + { request, stats, timings }, + outputFiles, + compileDir, + outputDir, + (err, newOutputFiles) => { + if (err) { + const { project_id: projectId, user_id: userId } = + request + logger.err( + { projectId, userId, err }, + 'failed to save output files' + ) + } + + const outputStage = outputStageTimer.done() + timings.sync = syncStage + timings.output = outputStage + + // Emit e2e compile time. + timings.compileE2E = timerE2E.done() + + if (stats['pdf-size']) { + emitPdfStats(stats, timings) + } + + callback(null, newOutputFiles, stats, timings) + } + ) + } ) - return callback(null, results) } ) } ) + } + ) +} + +function stopCompile(projectId, userId, callback) { + const compileName = getCompileName(projectId, userId) + LatexRunner.killLatex(compileName, callback) +} + +function clearProject(projectId, userId, _callback) { + function callback(error) { + _callback(error) + _callback = function () {} + } + + const compileDir = getCompileDir(projectId, userId) + const outputDir = getOutputDir(projectId, userId) + + _checkDirectory(compileDir, (err, exists) => { + if (err) { + return callback(err) + } + if (!exists) { + return callback() + } // skip removal if no directory present + + const proc = childProcess.spawn('rm', [ + '-r', + '-f', + '--', + compileDir, + outputDir, + ]) + + proc.on('error', callback) + + let stderr = '' + proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk)) + + proc.on('close', code => { + if (code === 0) { + callback(null) + } else { + callback( + new Error(`rm -r ${compileDir} ${outputDir} failed: ${stderr}`) + ) + } }) - }, - - _parseWordcountFromOutput(output) { - const results = { - encode: '', - textWords: 0, - headWords: 0, - outside: 0, - headers: 0, - elements: 0, - mathInline: 0, - mathDisplay: 0, - errors: 0, - messages: '', - } - for (const line of Array.from(output.split('\n'))) { - const [data, info] = Array.from(line.split(':')) - if (data.indexOf('Encoding') > -1) { - results.encode = info.trim() - } - if (data.indexOf('in text') > -1) { - results.textWords = parseInt(info, 10) - } - if (data.indexOf('in head') > -1) { - results.headWords = parseInt(info, 10) - } - if (data.indexOf('outside') > -1) { - results.outside = parseInt(info, 10) - } - if (data.indexOf('of head') > -1) { - results.headers = parseInt(info, 10) - } - if (data.indexOf('Number of floats/tables/figures') > -1) { - results.elements = parseInt(info, 10) - } - if (data.indexOf('Number of math inlines') > -1) { - results.mathInline = parseInt(info, 10) - } - if (data.indexOf('Number of math displayed') > -1) { - results.mathDisplay = parseInt(info, 10) - } - if (data === '(errors') { - // errors reported as (errors:123) - results.errors = parseInt(info, 10) - } - if (line.indexOf('!!! ') > -1) { - // errors logged as !!! message !!! - results.messages += line + '\n' - } - } - return results - }, + }) } -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined +function _findAllDirs(callback) { + const root = Settings.path.compilesDir + fs.readdir(root, (err, files) => { + if (err) { + return callback(err) + } + const allDirs = files.map(file => Path.join(root, file)) + callback(null, allDirs) + }) +} + +function clearExpiredProjects(maxCacheAgeMs, callback) { + const now = Date.now() + // action for each directory + const expireIfNeeded = (checkDir, cb) => + fs.stat(checkDir, (err, stats) => { + if (err) { + return cb() + } // ignore errors checking directory + const age = now - stats.mtime + const hasExpired = age > maxCacheAgeMs + if (hasExpired) { + fse.remove(checkDir, cb) + } else { + cb() + } + }) + // iterate over all project directories + _findAllDirs((error, allDirs) => { + if (error) { + return callback() + } + async.eachSeries(allDirs, expireIfNeeded, callback) + }) +} + +function _checkDirectory(compileDir, callback) { + fs.lstat(compileDir, (err, stats) => { + if (err && err.code === 'ENOENT') { + callback(null, false) // directory does not exist + } else if (err) { + logger.err( + { dir: compileDir, err }, + 'error on stat of project directory for removal' + ) + callback(err) + } else if (!stats.isDirectory()) { + logger.err( + { dir: compileDir, stats }, + 'bad project directory for removal' + ) + callback(new Error('project directory is not directory')) + } else { + // directory exists + callback(null, true) + } + }) +} + +function syncFromCode( + projectId, + userId, + filename, + line, + column, + imageName, + callback +) { + // If LaTeX was run in a virtual environment, the file path that synctex expects + // might not match the file path on the host. The .synctex.gz file however, will be accessed + // wherever it is on the host. + const compileName = getCompileName(projectId, userId) + const baseDir = Settings.path.synctexBaseDir(compileName) + const filePath = baseDir + '/' + filename + const synctexPath = `${baseDir}/output.pdf` + const command = ['code', synctexPath, filePath, line, column] + _runSynctex(projectId, userId, command, imageName, (error, stdout) => { + if (error) { + return callback(error) + } + logger.log( + { projectId, userId, filename, line, column, command, stdout }, + 'synctex code output' + ) + callback(null, _parseSynctexFromCodeOutput(stdout)) + }) +} + +function syncFromPdf(projectId, userId, page, h, v, imageName, callback) { + const compileName = getCompileName(projectId, userId) + const baseDir = Settings.path.synctexBaseDir(compileName) + const synctexPath = `${baseDir}/output.pdf` + const command = ['pdf', synctexPath, page, h, v] + _runSynctex(projectId, userId, command, imageName, (error, stdout) => { + if (error != null) { + return callback(error) + } + logger.log({ projectId, userId, page, h, v, stdout }, 'synctex pdf output') + callback(null, _parseSynctexFromPdfOutput(stdout, baseDir)) + }) +} + +function _checkFileExists(dir, filename, callback) { + const file = Path.join(dir, filename) + fs.stat(dir, (error, stats) => { + if (error && error.code === 'ENOENT') { + return callback(new Errors.NotFoundError('no output directory')) + } + if (error) { + return callback(error) + } + fs.stat(file, (error, stats) => { + if (error && error.code === 'ENOENT') { + return callback(new Errors.NotFoundError('no output file')) + } + if (error) { + return callback(error) + } + if (!stats.isFile()) { + return callback(new Error('not a file')) + } + callback() + }) + }) +} + +function _runSynctex(projectId, userId, command, imageName, callback) { + command.unshift('/opt/synctex') + + const directory = getCompileDir(projectId, userId) + const timeout = 60 * 1000 // increased to allow for large projects + const compileName = getCompileName(projectId, userId) + const compileGroup = 'synctex' + _checkFileExists(directory, 'output.synctex.gz', error => { + if (error) { + return callback(error) + } + CommandRunner.run( + compileName, + command, + directory, + imageName || + (Settings.clsi && Settings.clsi.docker + ? Settings.clsi.docker.image + : undefined), + timeout, + {}, + compileGroup, + (error, output) => { + if (error) { + logger.err( + { err: error, command, projectId, userId }, + 'error running synctex' + ) + return callback(error) + } + callback(null, output.stdout) + } + ) + }) +} + +function _parseSynctexFromCodeOutput(output) { + const results = [] + for (const line of output.split('\n')) { + const [node, page, h, v, width, height] = line.split('\t') + if (node === 'NODE') { + results.push({ + page: parseInt(page, 10), + h: parseFloat(h), + v: parseFloat(v), + height: parseFloat(height), + width: parseFloat(width), + }) + } + } + return results +} + +function _parseSynctexFromPdfOutput(output, baseDir) { + const results = [] + for (const line of output.split('\n')) { + const [node, filePath, lineNum, column] = line.split('\t') + if (node === 'NODE') { + const file = filePath.slice(baseDir.length + 1) + results.push({ + file, + line: parseInt(lineNum, 10), + column: parseInt(column, 10), + }) + } + } + return results +} + +function wordcount(projectId, userId, filename, image, callback) { + logger.log({ projectId, userId, filename, image }, 'running wordcount') + const filePath = `$COMPILE_DIR/${filename}` + const command = [ + 'texcount', + '-nocol', + '-inc', + filePath, + `-out=${filePath}.wc`, + ] + const compileDir = getCompileDir(projectId, userId) + const timeout = 60 * 1000 + const compileName = getCompileName(projectId, userId) + const compileGroup = 'wordcount' + fse.ensureDir(compileDir, error => { + if (error) { + logger.err( + { error, projectId, userId, filename }, + 'error ensuring dir for sync from code' + ) + return callback(error) + } + CommandRunner.run( + compileName, + command, + compileDir, + image, + timeout, + {}, + compileGroup, + error => { + if (error) { + return callback(error) + } + fs.readFile( + compileDir + '/' + filename + '.wc', + 'utf-8', + (err, stdout) => { + if (err) { + // call it node_err so sentry doesn't use random path error as unique id so it can't be ignored + logger.err( + { node_err: err, command, compileDir, projectId, userId }, + 'error reading word count output' + ) + return callback(err) + } + const results = _parseWordcountFromOutput(stdout) + logger.log( + { projectId, userId, wordcount: results }, + 'word count results' + ) + callback(null, results) + } + ) + } + ) + }) +} + +function _parseWordcountFromOutput(output) { + const results = { + encode: '', + textWords: 0, + headWords: 0, + outside: 0, + headers: 0, + elements: 0, + mathInline: 0, + mathDisplay: 0, + errors: 0, + messages: '', + } + for (const line of output.split('\n')) { + const [data, info] = line.split(':') + if (data.indexOf('Encoding') > -1) { + results.encode = info.trim() + } + if (data.indexOf('in text') > -1) { + results.textWords = parseInt(info, 10) + } + if (data.indexOf('in head') > -1) { + results.headWords = parseInt(info, 10) + } + if (data.indexOf('outside') > -1) { + results.outside = parseInt(info, 10) + } + if (data.indexOf('of head') > -1) { + results.headers = parseInt(info, 10) + } + if (data.indexOf('Number of floats/tables/figures') > -1) { + results.elements = parseInt(info, 10) + } + if (data.indexOf('Number of math inlines') > -1) { + results.mathInline = parseInt(info, 10) + } + if (data.indexOf('Number of math displayed') > -1) { + results.mathDisplay = parseInt(info, 10) + } + if (data === '(errors') { + // errors reported as (errors:123) + results.errors = parseInt(info, 10) + } + if (line.indexOf('!!! ') > -1) { + // errors logged as !!! message !!! + results.messages += line + '\n' + } + } + return results +} + +module.exports = { + doCompileWithLock, + stopCompile, + clearProject, + clearExpiredProjects, + syncFromCode, + syncFromPdf, + wordcount, } diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js index dfedcf31ef..a9c7c4f039 100644 --- a/services/clsi/test/unit/js/CompileManagerTests.js +++ b/services/clsi/test/unit/js/CompileManagerTests.js @@ -1,161 +1,122 @@ -/* eslint-disable - camelcase, - chai-friendly/no-unused-expressions, - no-path-concat, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const SandboxedModule = require('sandboxed-module') const sinon = require('sinon') const modulePath = require('path').join( __dirname, '../../../app/js/CompileManager' ) -const tk = require('timekeeper') const { EventEmitter } = require('events') -const Path = require('path') describe('CompileManager', function () { beforeEach(function () { + this.callback = sinon.stub() + this.projectId = 'project-id-123' + this.userId = '1234' + this.resources = 'mock-resources' + this.outputFiles = [ + { + path: 'output.log', + type: 'log', + }, + { + path: 'output.pdf', + type: 'pdf', + }, + ] + this.buildFiles = [ + { + path: 'output.log', + type: 'log', + build: 1234, + }, + { + path: 'output.pdf', + type: 'pdf', + build: 1234, + }, + ] + this.proc = new EventEmitter() + this.proc.stdout = new EventEmitter() + this.proc.stderr = new EventEmitter() + this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr) + + this.LatexRunner = { + runLatex: sinon.stub().yields(), + } + this.ResourceWriter = { + syncResourcesToDisk: sinon.stub().yields(null, this.resources), + } + this.OutputFileFinder = { + findOutputFiles: sinon.stub().yields(null, this.outputFiles), + } + this.OutputCacheManager = { + saveOutputFiles: sinon.stub().yields(null, this.buildFiles), + } + this.Settings = { + path: { + compilesDir: '/compiles/dir', + outputDir: '/output/dir', + }, + synctexBaseDir() { + return '/compile' + }, + clsi: { + docker: { + image: 'SOMEIMAGE', + }, + }, + } + this.child_process = { + exec: sinon.stub(), + spawn: sinon.stub().returns(this.proc), + } + this.CommandRunner = { + run: sinon.stub().yields(), + } + this.DraftModeManager = { + injectDraftMode: sinon.stub().yields(), + } + this.TikzManager = { + checkMainFile: sinon.stub().yields(null, false), + } + this.LockManager = { + runWithLock: sinon.stub().callsFake((lockFile, runner, callback) => { + runner((err, ...result) => callback(err, ...result)) + }), + } + this.fs = { + lstat: sinon.stub(), + stat: sinon.stub(), + readFile: sinon.stub(), + } + this.fse = { + ensureDir: sinon.stub().yields(), + } + this.CompileManager = SandboxedModule.require(modulePath, { requires: { - './LatexRunner': (this.LatexRunner = {}), - './ResourceWriter': (this.ResourceWriter = {}), - './OutputFileFinder': (this.OutputFileFinder = {}), - './OutputCacheManager': (this.OutputCacheManager = {}), - '@overleaf/settings': (this.Settings = { - path: { - compilesDir: '/compiles/dir', - outputDir: '/output/dir', - }, - synctexBaseDir() { - return '/compile' - }, - clsi: { - docker: { - image: 'SOMEIMAGE', - }, - }, - }), - - child_process: (this.child_process = {}), - './CommandRunner': (this.CommandRunner = {}), - './DraftModeManager': (this.DraftModeManager = {}), - './TikzManager': (this.TikzManager = {}), - './LockManager': (this.LockManager = {}), - fs: (this.fs = {}), - 'fs-extra': (this.fse = { ensureDir: sinon.stub().callsArg(1) }), + './LatexRunner': this.LatexRunner, + './ResourceWriter': this.ResourceWriter, + './OutputFileFinder': this.OutputFileFinder, + './OutputCacheManager': this.OutputCacheManager, + '@overleaf/settings': this.Settings, + child_process: this.child_process, + './CommandRunner': this.CommandRunner, + './DraftModeManager': this.DraftModeManager, + './TikzManager': this.TikzManager, + './LockManager': this.LockManager, + fs: this.fs, + 'fs-extra': this.fse, }, }) - this.callback = sinon.stub() - this.project_id = 'project-id-123' - return (this.user_id = '1234') }) + describe('doCompileWithLock', function () { beforeEach(function () { this.request = { - resources: (this.resources = 'mock-resources'), - project_id: this.project_id, - user_id: this.user_id, - } - this.output_files = ['foo', 'bar'] - this.Settings.compileDir = 'compiles' - this.compileDir = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}` - this.CompileManager.doCompile = sinon - .stub() - .yields(null, this.output_files) - return (this.LockManager.runWithLock = (lockFile, runner, callback) => - runner((err, ...result) => callback(err, ...Array.from(result)))) - }) - - describe('when the project is not locked', function () { - beforeEach(function () { - return this.CompileManager.doCompileWithLock( - this.request, - this.callback - ) - }) - - it('should ensure that the compile directory exists', function () { - return this.fse.ensureDir.calledWith(this.compileDir).should.equal(true) - }) - - it('should call doCompile with the request', function () { - return this.CompileManager.doCompile - .calledWith(this.request) - .should.equal(true) - }) - - return it('should call the callback with the output files', function () { - return this.callback - .calledWithExactly(null, this.output_files) - .should.equal(true) - }) - }) - - return describe('when the project is locked', function () { - beforeEach(function () { - this.error = new Error('locked') - this.LockManager.runWithLock = (lockFile, runner, callback) => { - return callback(this.error) - } - return this.CompileManager.doCompileWithLock( - this.request, - this.callback - ) - }) - - it('should ensure that the compile directory exists', function () { - return this.fse.ensureDir.calledWith(this.compileDir).should.equal(true) - }) - - it('should not call doCompile with the request', function () { - return this.CompileManager.doCompile.called.should.equal(false) - }) - - return it('should call the callback with the error', function () { - return this.callback.calledWithExactly(this.error).should.equal(true) - }) - }) - }) - - describe('doCompile', function () { - beforeEach(function () { - this.output_files = [ - { - path: 'output.log', - type: 'log', - }, - { - path: 'output.pdf', - type: 'pdf', - }, - ] - this.build_files = [ - { - path: 'output.log', - type: 'log', - build: 1234, - }, - { - path: 'output.pdf', - type: 'pdf', - build: 1234, - }, - ] - this.request = { - resources: (this.resources = 'mock-resources'), + resources: this.resources, rootResourcePath: (this.rootResourcePath = 'main.tex'), - project_id: this.project_id, - user_id: this.user_id, + project_id: this.projectId, + user_id: this.userId, compiler: (this.compiler = 'pdflatex'), timeout: (this.timeout = 42000), imageName: (this.image = 'example.com/image'), @@ -164,36 +125,50 @@ describe('CompileManager', function () { } this.env = {} this.Settings.compileDir = 'compiles' - this.compileDir = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}` - this.outputDir = `${this.Settings.path.outputDir}/${this.project_id}-${this.user_id}` - this.ResourceWriter.syncResourcesToDisk = sinon - .stub() - .callsArgWith(2, null, this.resources) - this.LatexRunner.runLatex = sinon.stub().callsArg(2) - this.OutputFileFinder.findOutputFiles = sinon - .stub() - .yields(null, this.output_files) - this.OutputCacheManager.saveOutputFiles = sinon - .stub() - .yields(null, this.build_files) - this.DraftModeManager.injectDraftMode = sinon.stub().callsArg(1) - return (this.TikzManager.checkMainFile = sinon.stub().callsArg(3, false)) + this.compileDir = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}` + this.outputDir = `${this.Settings.path.outputDir}/${this.projectId}-${this.userId}` + }) + + describe('when the project is locked', function () { + beforeEach(function () { + this.error = new Error('locked') + this.LockManager.runWithLock.callsFake((lockFile, runner, callback) => { + callback(this.error) + }) + this.CompileManager.doCompileWithLock(this.request, this.callback) + }) + + it('should ensure that the compile directory exists', function () { + this.fse.ensureDir.calledWith(this.compileDir).should.equal(true) + }) + + it('should not run LaTeX', function () { + this.LatexRunner.runLatex.called.should.equal(false) + }) + + it('should call the callback with the error', function () { + this.callback.calledWithExactly(this.error).should.equal(true) + }) }) describe('normally', function () { beforeEach(function () { - return this.CompileManager.doCompile(this.request, this.callback) + this.CompileManager.doCompileWithLock(this.request, this.callback) + }) + + it('should ensure that the compile directory exists', function () { + this.fse.ensureDir.calledWith(this.compileDir).should.equal(true) }) it('should write the resources to disk', function () { - return this.ResourceWriter.syncResourcesToDisk + this.ResourceWriter.syncResourcesToDisk .calledWith(this.request, this.compileDir) .should.equal(true) }) it('should run LaTeX', function () { - return this.LatexRunner.runLatex - .calledWith(`${this.project_id}-${this.user_id}`, { + this.LatexRunner.runLatex + .calledWith(`${this.projectId}-${this.userId}`, { directory: this.compileDir, mainFile: this.rootResourcePath, compiler: this.compiler, @@ -207,30 +182,28 @@ describe('CompileManager', function () { }) it('should find the output files', function () { - return this.OutputFileFinder.findOutputFiles + this.OutputFileFinder.findOutputFiles .calledWith(this.resources, this.compileDir) .should.equal(true) }) it('should return the output files', function () { - return this.callback - .calledWith(null, this.build_files) - .should.equal(true) + this.callback.calledWith(null, this.buildFiles).should.equal(true) }) - return it('should not inject draft mode by default', function () { - return this.DraftModeManager.injectDraftMode.called.should.equal(false) + it('should not inject draft mode by default', function () { + this.DraftModeManager.injectDraftMode.called.should.equal(false) }) }) describe('with draft mode', function () { beforeEach(function () { this.request.draft = true - return this.CompileManager.doCompile(this.request, this.callback) + this.CompileManager.doCompileWithLock(this.request, this.callback) }) - return it('should inject the draft mode header', function () { - return this.DraftModeManager.injectDraftMode + it('should inject the draft mode header', function () { + this.DraftModeManager.injectDraftMode .calledWith(this.compileDir + '/' + this.rootResourcePath) .should.equal(true) }) @@ -239,12 +212,12 @@ describe('CompileManager', function () { describe('with a check option', function () { beforeEach(function () { this.request.check = 'error' - return this.CompileManager.doCompile(this.request, this.callback) + this.CompileManager.doCompileWithLock(this.request, this.callback) }) - return it('should run chktex', function () { - return this.LatexRunner.runLatex - .calledWith(`${this.project_id}-${this.user_id}`, { + it('should run chktex', function () { + this.LatexRunner.runLatex + .calledWith(`${this.projectId}-${this.userId}`, { directory: this.compileDir, mainFile: this.rootResourcePath, compiler: this.compiler, @@ -262,16 +235,16 @@ describe('CompileManager', function () { }) }) - return describe('with a knitr file and check options', function () { + describe('with a knitr file and check options', function () { beforeEach(function () { this.request.rootResourcePath = 'main.Rtex' this.request.check = 'error' - return this.CompileManager.doCompile(this.request, this.callback) + this.CompileManager.doCompileWithLock(this.request, this.callback) }) - return it('should not run chktex', function () { - return this.LatexRunner.runLatex - .calledWith(`${this.project_id}-${this.user_id}`, { + it('should not run chktex', function () { + this.LatexRunner.runLatex + .calledWith(`${this.projectId}-${this.userId}`, { directory: this.compileDir, mainFile: 'main.Rtex', compiler: this.compiler, @@ -290,71 +263,61 @@ describe('CompileManager', function () { describe('succesfully', function () { beforeEach(function () { this.Settings.compileDir = 'compiles' - this.fs.lstat = sinon.stub().callsArgWith(1, null, { + this.fs.lstat.yields(null, { isDirectory() { return true }, }) - this.proc = new EventEmitter() - this.proc.stdout = new EventEmitter() - this.proc.stderr = new EventEmitter() - this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr) - this.child_process.spawn = sinon.stub().returns(this.proc) this.CompileManager.clearProject( - this.project_id, - this.user_id, + this.projectId, + this.userId, this.callback ) - return this.proc.emit('close', 0) + this.proc.emit('close', 0) }) it('should remove the project directory', function () { - return this.child_process.spawn + this.child_process.spawn .calledWith('rm', [ '-r', '-f', '--', - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, - `${this.Settings.path.outputDir}/${this.project_id}-${this.user_id}`, + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, + `${this.Settings.path.outputDir}/${this.projectId}-${this.userId}`, ]) .should.equal(true) }) - return it('should call the callback', function () { - return this.callback.called.should.equal(true) + it('should call the callback', function () { + this.callback.called.should.equal(true) }) }) - return describe('with a non-success status code', function () { + describe('with a non-success status code', function () { beforeEach(function () { this.Settings.compileDir = 'compiles' - this.fs.lstat = sinon.stub().callsArgWith(1, null, { + this.fs.lstat.yields(null, { isDirectory() { return true }, }) - this.proc = new EventEmitter() - this.proc.stdout = new EventEmitter() - this.proc.stderr = new EventEmitter() - this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr) - this.child_process.spawn = sinon.stub().returns(this.proc) this.CompileManager.clearProject( - this.project_id, - this.user_id, + this.projectId, + this.userId, this.callback ) this.proc.stderr.emit('data', (this.error = 'oops')) - return this.proc.emit('close', 1) + this.proc.emit('close', 1) }) it('should remove the project directory', function () { - return this.child_process.spawn + this.child_process.spawn .calledWith('rm', [ '-r', '-f', '--', - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, - `${this.Settings.path.outputDir}/${this.project_id}-${this.user_id}`, + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, + `${this.Settings.path.outputDir}/${this.projectId}-${this.userId}`, ]) .should.equal(true) }) @@ -363,7 +326,7 @@ describe('CompileManager', function () { this.callback.calledWithExactly(sinon.match(Error)).should.equal(true) this.callback.args[0][0].message.should.equal( - `rm -r ${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id} ${this.Settings.path.outputDir}/${this.project_id}-${this.user_id} failed: ${this.error}` + `rm -r ${this.Settings.path.compilesDir}/${this.projectId}-${this.userId} ${this.Settings.path.outputDir}/${this.projectId}-${this.userId} failed: ${this.error}` ) }) }) @@ -379,25 +342,22 @@ describe('CompileManager', function () { this.line = 5 this.column = 3 this.file_name = 'main.tex' - this.child_process.execFile = sinon.stub() - return (this.Settings.path.synctexBaseDir = project_id => - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`) + this.Settings.path.synctexBaseDir = projectId => + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}` }) describe('syncFromCode', function () { beforeEach(function () { - this.fs.stat = sinon.stub().callsArgWith(1, null, { + this.fs.stat.yields(null, { isFile() { return true }, }) this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n` - this.CommandRunner.run = sinon - .stub() - .yields(null, { stdout: this.stdout }) - return this.CompileManager.syncFromCode( - this.project_id, - this.user_id, + this.CommandRunner.run.yields(null, { stdout: this.stdout }) + this.CompileManager.syncFromCode( + this.projectId, + this.userId, this.file_name, this.line, this.column, @@ -407,21 +367,20 @@ describe('CompileManager', function () { }) it('should execute the synctex binary', function () { - const bin_path = Path.resolve(__dirname + '/../../../bin/synctex') - const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf` - const file_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}` - return this.CommandRunner.run + const synctexPath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/output.pdf` + const filePath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/${this.file_name}` + this.CommandRunner.run .calledWith( - `${this.project_id}-${this.user_id}`, + `${this.projectId}-${this.userId}`, [ '/opt/synctex', 'code', - synctex_path, - file_path, + synctexPath, + filePath, this.line, this.column, ], - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, this.Settings.clsi.docker.image, 60000, {} @@ -430,7 +389,7 @@ describe('CompileManager', function () { }) it('should call the callback with the parsed output', function () { - return this.callback + this.callback .calledWith(null, [ { page: this.page, @@ -448,8 +407,8 @@ describe('CompileManager', function () { beforeEach(function () { this.CommandRunner.run.reset() this.CompileManager.syncFromCode( - this.project_id, - this.user_id, + this.projectId, + this.userId, this.file_name, this.line, this.column, @@ -459,20 +418,20 @@ describe('CompileManager', function () { }) it('should execute the synctex binary in a custom docker image', function () { - const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf` - const file_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}` + const synctexPath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/output.pdf` + const filePath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/${this.file_name}` this.CommandRunner.run .calledWith( - `${this.project_id}-${this.user_id}`, + `${this.projectId}-${this.userId}`, [ '/opt/synctex', 'code', - synctex_path, - file_path, + synctexPath, + filePath, this.line, this.column, ], - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, customImageName, 60000, {} @@ -482,20 +441,18 @@ describe('CompileManager', function () { }) }) - return describe('syncFromPdf', function () { + describe('syncFromPdf', function () { beforeEach(function () { - this.fs.stat = sinon.stub().callsArgWith(1, null, { + this.fs.stat.yields(null, { isFile() { return true }, }) - this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n` - this.CommandRunner.run = sinon - .stub() - .callsArgWith(7, null, { stdout: this.stdout }) - return this.CompileManager.syncFromPdf( - this.project_id, - this.user_id, + this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/${this.file_name}\t${this.line}\t${this.column}\n` + this.CommandRunner.run.yields(null, { stdout: this.stdout }) + this.CompileManager.syncFromPdf( + this.projectId, + this.userId, this.page, this.h, this.v, @@ -505,13 +462,12 @@ describe('CompileManager', function () { }) it('should execute the synctex binary', function () { - const bin_path = Path.resolve(__dirname + '/../../../bin/synctex') - const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf` - return this.CommandRunner.run + const synctexPath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/output.pdf` + this.CommandRunner.run .calledWith( - `${this.project_id}-${this.user_id}`, - ['/opt/synctex', 'pdf', synctex_path, this.page, this.h, this.v], - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, + `${this.projectId}-${this.userId}`, + ['/opt/synctex', 'pdf', synctexPath, this.page, this.h, this.v], + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, this.Settings.clsi.docker.image, 60000, {} @@ -520,7 +476,7 @@ describe('CompileManager', function () { }) it('should call the callback with the parsed output', function () { - return this.callback + this.callback .calledWith(null, [ { file: this.file_name, @@ -536,8 +492,8 @@ describe('CompileManager', function () { beforeEach(function () { this.CommandRunner.run.reset() this.CompileManager.syncFromPdf( - this.project_id, - this.user_id, + this.projectId, + this.userId, this.page, this.h, this.v, @@ -547,12 +503,12 @@ describe('CompileManager', function () { }) it('should execute the synctex binary in a custom docker image', function () { - const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf` + const synctexPath = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}/output.pdf` this.CommandRunner.run .calledWith( - `${this.project_id}-${this.user_id}`, - ['/opt/synctex', 'pdf', synctex_path, this.page, this.h, this.v], - `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`, + `${this.projectId}-${this.userId}`, + ['/opt/synctex', 'pdf', synctexPath, this.page, this.h, this.v], + `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}`, customImageName, 60000, {} @@ -563,27 +519,19 @@ describe('CompileManager', function () { }) }) - return describe('wordcount', function () { + describe('wordcount', function () { beforeEach(function () { - this.CommandRunner.run = sinon.stub().callsArg(7) - this.fs.readFile = sinon - .stub() - .callsArgWith( - 2, - null, - (this.stdout = 'Encoding: ascii\nWords in text: 2') - ) - this.callback = sinon.stub() + this.stdout = 'Encoding: ascii\nWords in text: 2' + this.fs.readFile.yields(null, this.stdout) - this.project_id this.timeout = 60 * 1000 this.file_name = 'main.tex' this.Settings.path.compilesDir = '/local/compile/directory' this.image = 'example.com/image' - return this.CompileManager.wordcount( - this.project_id, - this.user_id, + this.CompileManager.wordcount( + this.projectId, + this.userId, this.file_name, this.image, this.callback @@ -591,19 +539,19 @@ describe('CompileManager', function () { }) it('should run the texcount command', function () { - this.directory = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}` - this.file_path = `$COMPILE_DIR/${this.file_name}` + this.directory = `${this.Settings.path.compilesDir}/${this.projectId}-${this.userId}` + this.filePath = `$COMPILE_DIR/${this.file_name}` this.command = [ 'texcount', '-nocol', '-inc', - this.file_path, - `-out=${this.file_path}.wc`, + this.filePath, + `-out=${this.filePath}.wc`, ] - return this.CommandRunner.run + this.CommandRunner.run .calledWith( - `${this.project_id}-${this.user_id}`, + `${this.projectId}-${this.userId}`, this.command, this.directory, this.image, @@ -613,8 +561,8 @@ describe('CompileManager', function () { .should.equal(true) }) - return it('should call the callback with the parsed output', function () { - return this.callback + it('should call the callback with the parsed output', function () { + this.callback .calledWith(null, { encode: 'ascii', textWords: 2,