diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index 4be4353cb9..7eaa03f814 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -22,7 +22,7 @@ COPY . /app FROM base COPY --from=app /app /app -RUN mkdir -p cache compiles db \ -&& chown node:node cache compiles db +RUN mkdir -p cache compiles db output \ + && chown node:node cache compiles db output CMD ["node", "--expose-gc", "app.js"] diff --git a/services/clsi/app.js b/services/clsi/app.js index 05bcffce32..c87edeb3f2 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -138,6 +138,26 @@ const staticCompileServer = ForbidSymlinks( } ) +const staticOutputServer = ForbidSymlinks( + express.static, + Settings.path.outputDir, + { + setHeaders(res, path, stat) { + if (Path.basename(path) === 'output.pdf') { + // Calculate an etag in the same way as nginx + // https://github.com/tj/send/issues/65 + const etag = (path, stat) => + `"${Math.ceil(+stat.mtime / 1000).toString(16)}` + + '-' + + Number(stat.size).toString(16) + + '"' + res.set('Etag', etag(path, stat)) + } + return res.set('Content-Type', ContentTypeMapper.map(path)) + } + } +) + app.get( '/project/:project_id/user/:user_id/build/:build_id/output/*', function (req, res, next) { @@ -145,7 +165,7 @@ app.get( req.url = `/${req.params.project_id}-${req.params.user_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`) - return staticCompileServer(req, res, next) + return staticOutputServer(req, res, next) } ) @@ -158,7 +178,7 @@ app.get('/project/:project_id/build/:build_id/output/*', function ( req.url = `/${req.params.project_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`) - return staticCompileServer(req, res, next) + return staticOutputServer(req, res, next) }) app.get('/project/:project_id/user/:user_id/output/*', function ( diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 68edde49b0..5dab554ff2 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -46,6 +46,9 @@ const getCompileName = function (project_id, user_id) { const getCompileDir = (project_id, user_id) => Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) +const getOutputDir = (project_id, user_id) => + Path.join(Settings.path.outputDir, getCompileName(project_id, user_id)) + module.exports = CompileManager = { doCompileWithLock(request, callback) { if (callback == null) { @@ -72,6 +75,8 @@ module.exports = CompileManager = { callback = function (error, outputFiles) {} } const compileDir = getCompileDir(request.project_id, request.user_id) + const outputDir = getOutputDir(request.project_id, request.user_id) + let timer = new Metrics.Timer('write-to-disk') logger.log( { project_id: request.project_id, user_id: request.user_id }, @@ -294,6 +299,7 @@ module.exports = CompileManager = { return OutputCacheManager.saveOutputFiles( outputFiles, compileDir, + outputDir, (error, newOutputFiles) => callback(null, newOutputFiles) ) } diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js index b34dea7994..8aefb9b77a 100644 --- a/services/clsi/app/js/OutputCacheManager.js +++ b/services/clsi/app/js/OutputCacheManager.js @@ -26,8 +26,8 @@ const crypto = require('crypto') const OutputFileOptimiser = require('./OutputFileOptimiser') module.exports = OutputCacheManager = { - CACHE_SUBDIR: '.cache/clsi', - ARCHIVE_SUBDIR: '.archive/clsi', + CACHE_SUBDIR: 'generated-files', + ARCHIVE_SUBDIR: 'archived-logs', // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes // for backwards compatibility, make the randombytes part optional BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/, @@ -59,7 +59,7 @@ module.exports = OutputCacheManager = { }) }, - saveOutputFiles(outputFiles, compileDir, callback) { + saveOutputFiles(outputFiles, compileDir, outputDir, callback) { if (callback == null) { callback = function (error) {} } @@ -70,22 +70,29 @@ module.exports = OutputCacheManager = { return OutputCacheManager.saveOutputFilesInBuildDir( outputFiles, compileDir, + outputDir, buildId, callback ) }) }, - saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) { + saveOutputFilesInBuildDir( + outputFiles, + compileDir, + outputDir, + buildId, + callback + ) { // make a compileDir/CACHE_SUBDIR/build_id directory and // copy all the output files into it if (callback == null) { callback = function (error) {} } - const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR) + const cacheRoot = Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR) // Put the files into a new cache subdirectory const cacheDir = Path.join( - compileDir, + outputDir, OutputCacheManager.CACHE_SUBDIR, buildId ) @@ -102,6 +109,7 @@ module.exports = OutputCacheManager = { OutputCacheManager.archiveLogs( outputFiles, compileDir, + outputDir, buildId, function (err) { if (err != null) { @@ -198,12 +206,12 @@ module.exports = OutputCacheManager = { }) }, - archiveLogs(outputFiles, compileDir, buildId, callback) { + archiveLogs(outputFiles, compileDir, outputDir, buildId, callback) { if (callback == null) { callback = function (error) {} } const archiveDir = Path.join( - compileDir, + outputDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId ) diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index 823f1f737f..72c3471ba9 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -27,6 +27,7 @@ module.exports = { path: { compilesDir: Path.resolve(__dirname, '../compiles'), + outputDir: Path.resolve(__dirname, '../output'), clsiCacheDir: Path.resolve(__dirname, '../cache'), synctexBaseDir(projectId) { return Path.join(this.compilesDir, projectId)