github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #16426 from overleaf/em-decaf-output-cache-manager

Browse source 
Decaf cleanup OutputCacheManager

GitOrigin-RevId: 59d65930b4a88ab20e330dedac3d00d80c4140fd
This commit is contained in:
Jakob Ackermann 2024-01-08 08:10:58 +00:00 committed by Copybot
parent 353fdac850
commit 17d17612b3
1 changed files with 93 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

254
services/clsi/app/js/OutputCacheManager.js
View file

@ -1,15 +1,3 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS104: Avoid inline assignments
* DS204: Change includes calls to have a more natural evaluation order
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputCacheManager let OutputCacheManager
const { callbackify, promisify } = require('util') const { callbackify, promisify } = require('util')
const async = require('async') const async = require('async')
@ -133,16 +121,13 @@ module.exports = OutputCacheManager = {
generateBuildId(callback) { generateBuildId(callback) {
// generate a secure build id from Date.now() and 8 random bytes in hex // generate a secure build id from Date.now() and 8 random bytes in hex
if (callback == null) { crypto.randomBytes(8, function (err, buf) {
callback = function () {} if (err) {
}
return crypto.randomBytes(8, function (err, buf) {
if (err != null) {
return callback(err) return callback(err)
} }
const random = buf.toString('hex') const random = buf.toString('hex')
const date = Date.now().toString(16) const date = Date.now().toString(16)
return callback(err, `${date}-${random}`) callback(err, `${date}-${random}`)
}) })
}, },
@ -153,11 +138,8 @@ module.exports = OutputCacheManager = {
outputDir, outputDir,
callback callback
) { ) {
if (callback == null) { OutputCacheManager.generateBuildId(function (err, buildId) {
callback = function () {} if (err) {
}
return OutputCacheManager.generateBuildId(function (err, buildId) {
if (err != null) {
return callback(err) return callback(err)
} }
if (!OLDEST_BUILD_DIR.has(outputDir)) { if (!OLDEST_BUILD_DIR.has(outputDir)) {
@ -175,7 +157,7 @@ module.exports = OutputCacheManager = {
buildId buildId
), ),
function (err, result) { function (err, result) {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
OutputCacheManager.collectOutputPdfSize( OutputCacheManager.collectOutputPdfSize(
@ -231,9 +213,6 @@ module.exports = OutputCacheManager = {
) { ) {
// make a compileDir/CACHE_SUBDIR/build_id directory and // make a compileDir/CACHE_SUBDIR/build_id directory and
// copy all the output files into it // copy all the output files into it
if (callback == null) {
callback = function () {}
}
// Put the files into a new cache subdirectory // Put the files into a new cache subdirectory
const cacheDir = Path.join( const cacheDir = Path.join(
outputDir, outputDir,
@ -246,17 +225,14 @@ module.exports = OutputCacheManager = {
) )
// Archive logs in background // Archive logs in background
if ( if (Settings.clsi?.archive_logs || Settings.clsi?.strace) {
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) ||
(Settings.clsi != null ? Settings.clsi.strace : undefined)
) {
OutputCacheManager.archiveLogs( OutputCacheManager.archiveLogs(
outputFiles, outputFiles,
compileDir, compileDir,
outputDir, outputDir,
buildId, buildId,
function (err) { function (err) {
if (err != null) { if (err) {
return logger.warn({ err }, 'erroring archiving log files') return logger.warn({ err }, 'erroring archiving log files')
} }
} }
@ -264,17 +240,17 @@ module.exports = OutputCacheManager = {
} }
// make the new cache directory // make the new cache directory
return fse.ensureDir(cacheDir, function (err) { fse.ensureDir(cacheDir, function (err) {
if (err != null) { if (err) {
logger.error( logger.error(
{ err, directory: cacheDir }, { err, directory: cacheDir },
'error creating cache directory' 'error creating cache directory'
) )
return callback(err, outputFiles) callback(err, outputFiles)
} else { } else {
// copy all the output files into the new cache directory // copy all the output files into the new cache directory
const results = [] const results = []
return async.mapSeries( async.mapSeries(
outputFiles, outputFiles,
function (file, cb) { function (file, cb) {
// don't send dot files as output, express doesn't serve them // don't send dot files as output, express doesn't serve them
@ -287,52 +263,43 @@ module.exports = OutputCacheManager = {
} }
// copy other files into cache directory if valid // copy other files into cache directory if valid
const newFile = _.clone(file) const newFile = _.clone(file)
const [src, dst] = Array.from([ const src = Path.join(compileDir, file.path)
Path.join(compileDir, file.path), const dst = Path.join(cacheDir, file.path)
Path.join(cacheDir, file.path), OutputCacheManager._checkFileIsSafe(src, function (err, isSafe) {
]) if (err) {
return OutputCacheManager._checkFileIsSafe( return cb(err)
src, }
function (err, isSafe) { if (!isSafe) {
if (err != null) { return cb()
return cb(err) }
} OutputCacheManager._checkIfShouldCopy(
if (!isSafe) { src,
return cb() function (err, shouldCopy) {
} if (err) {
return OutputCacheManager._checkIfShouldCopy( return cb(err)
src, }
function (err, shouldCopy) { if (!shouldCopy) {
if (err != null) { return cb()
}
OutputCacheManager._copyFile(src, dst, function (err) {
if (err) {
return cb(err) return cb(err)
} }
if (!shouldCopy) { newFile.build = buildId // attach a build id if we cached the file
return cb() results.push(newFile)
} cb()
return OutputCacheManager._copyFile( })
src, }
dst, )
function (err) { })
if (err != null) {
return cb(err)
}
newFile.build = buildId // attach a build id if we cached the file
results.push(newFile)
return cb()
}
)
}
)
}
)
}, },
function (err) { function (err) {
if (err != null) { if (err) {
// pass back the original files if we encountered *any* error // pass back the original files if we encountered *any* error
callback(err, outputFiles) callback(err, outputFiles)
// clean up the directory we just created // clean up the directory we just created
return fse.remove(cacheDir, function (err) { fse.remove(cacheDir, function (err) {
if (err != null) { if (err) {
return logger.error( return logger.error(
{ err, dir: cacheDir }, { err, dir: cacheDir },
'error removing cache dir after failure' 'error removing cache dir after failure'
@ -481,7 +448,7 @@ module.exports = OutputCacheManager = {
ensureContentDir(contentRoot, callback) { ensureContentDir(contentRoot, callback) {
fse.ensureDir(contentRoot, function (err) { fse.ensureDir(contentRoot, function (err) {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
fs.readdir(contentRoot, function (err, results) { fs.readdir(contentRoot, function (err, results) {
@ -503,7 +470,7 @@ module.exports = OutputCacheManager = {
if (err) { if (err) {
return callback(err) return callback(err)
} }
return callback(null, contentDir) callback(null, contentDir)
}) })
}) })
} }
@ -512,49 +479,41 @@ module.exports = OutputCacheManager = {
}, },
archiveLogs(outputFiles, compileDir, outputDir, buildId, callback) { archiveLogs(outputFiles, compileDir, outputDir, buildId, callback) {
if (callback == null) {
callback = function () {}
}
const archiveDir = Path.join( const archiveDir = Path.join(
outputDir, outputDir,
OutputCacheManager.ARCHIVE_SUBDIR, OutputCacheManager.ARCHIVE_SUBDIR,
buildId buildId
) )
logger.debug({ dir: archiveDir }, 'archiving log files for project') logger.debug({ dir: archiveDir }, 'archiving log files for project')
return fse.ensureDir(archiveDir, function (err) { fse.ensureDir(archiveDir, function (err) {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return async.mapSeries( async.mapSeries(
outputFiles, outputFiles,
function (file, cb) { function (file, cb) {
const [src, dst] = Array.from([ const src = Path.join(compileDir, file.path)
Path.join(compileDir, file.path), const dst = Path.join(archiveDir, file.path)
Path.join(archiveDir, file.path), OutputCacheManager._checkFileIsSafe(src, function (err, isSafe) {
]) if (err) {
return OutputCacheManager._checkFileIsSafe( return cb(err)
src,
function (err, isSafe) {
if (err != null) {
return cb(err)
}
if (!isSafe) {
return cb()
}
return OutputCacheManager._checkIfShouldArchive(
src,
function (err, shouldArchive) {
if (err != null) {
return cb(err)
}
if (!shouldArchive) {
return cb()
}
return OutputCacheManager._copyFile(src, dst, cb)
}
)
} }
) if (!isSafe) {
return cb()
}
OutputCacheManager._checkIfShouldArchive(
src,
function (err, shouldArchive) {
if (err) {
return cb(err)
}
if (!shouldArchive) {
return cb()
}
OutputCacheManager._copyFile(src, dst, cb)
}
)
})
}, },
callback callback
) )
@ -563,9 +522,6 @@ module.exports = OutputCacheManager = {
expireOutputFiles(outputDir, options, callback) { expireOutputFiles(outputDir, options, callback) {
// look in compileDir for build dirs and delete if > N or age of mod time > T // look in compileDir for build dirs and delete if > N or age of mod time > T
if (callback == null) {
callback = function () {}
}
const cleanupAll = cb => { const cleanupAll = cb => {
fse.remove(outputDir, err => { fse.remove(outputDir, err => {
if (err) { if (err) {
@ -578,8 +534,8 @@ module.exports = OutputCacheManager = {
} }
const cacheRoot = Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR) const cacheRoot = Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR)
return fs.readdir(cacheRoot, function (err, results) { fs.readdir(cacheRoot, function (err, results) {
if (err != null) { if (err) {
if (err.code === 'ENOENT') { if (err.code === 'ENOENT') {
// cache directory is empty // cache directory is empty
return cleanupAll(callback) return cleanupAll(callback)
@ -594,16 +550,13 @@ module.exports = OutputCacheManager = {
let oldestDirTimeToKeep = 0 let oldestDirTimeToKeep = 0
const isExpired = function (dir, index) { const isExpired = function (dir, index) {
if ((options != null ? options.keep : undefined) === dir) { if (options?.keep === dir) {
// This is the directory we just created for the compile request. // This is the directory we just created for the compile request.
oldestDirTimeToKeep = currentTime oldestDirTimeToKeep = currentTime
return false return false
} }
// remove any directories over the requested (non-null) limit // remove any directories over the requested (non-null) limit
if ( if (options?.limit != null && index > options.limit) {
(options != null ? options.limit : undefined) != null &&
index > options.limit
) {
return true return true
} }
// remove any directories over the hard limit // remove any directories over the hard limit
@ -612,10 +565,7 @@ module.exports = OutputCacheManager = {
} }
// we can get the build time from the first part of the directory name DDDD-RRRR // we can get the build time from the first part of the directory name DDDD-RRRR
// DDDD is date and RRRR is random bytes // DDDD is date and RRRR is random bytes
const dirTime = parseInt( const dirTime = parseInt(dir.split('-')[0], 16)
__guard__(dir.split('-'), x => x[0]),
16
)
const age = currentTime - dirTime const age = currentTime - dirTime
const expired = age > OutputCacheManager.CACHE_AGE const expired = age > OutputCacheManager.CACHE_AGE
if (expired) { if (expired) {
@ -634,12 +584,12 @@ module.exports = OutputCacheManager = {
const removeDir = (dir, cb) => const removeDir = (dir, cb) =>
fse.remove(Path.join(cacheRoot, dir), function (err, result) { fse.remove(Path.join(cacheRoot, dir), function (err, result) {
logger.debug({ cache: cacheRoot, dir }, 'removed expired cache dir') logger.debug({ cache: cacheRoot, dir }, 'removed expired cache dir')
if (err != null) { if (err) {
logger.error({ err, dir }, 'cache remove error') logger.error({ err, dir }, 'cache remove error')
} }
return cb(err, result) cb(err, result)
}) })
return async.eachSeries( async.eachSeries(
toRemove, toRemove,
(dir, cb) => removeDir(dir, cb), (dir, cb) => removeDir(dir, cb),
err => { err => {
@ -657,22 +607,19 @@ module.exports = OutputCacheManager = {
}, },
_fileIsHidden(path) { _fileIsHidden(path) {
return (path != null ? path.match(/^\.|\/\./) : undefined) != null return path?.match(/^\.|\/\./) != null
}, },
_checkFileIsSafe(src, callback) { _checkFileIsSafe(src, callback) {
// check if we have a valid file to copy into the cache // check if we have a valid file to copy into the cache
if (callback == null) { fs.stat(src, function (err, stats) {
callback = function () {} if (err?.code === 'ENOENT') {
}
return fs.stat(src, function (err, stats) {
if ((err != null ? err.code : undefined) === 'ENOENT') {
logger.warn( logger.warn(
{ err, file: src }, { err, file: src },
'file has disappeared before copying to build cache' 'file has disappeared before copying to build cache'
) )
return callback(err, false) return callback(err, false)
} else if (err != null) { } else if (err) {
// some other problem reading the file // some other problem reading the file
logger.error({ err, file: src }, 'stat error for file in cache') logger.error({ err, file: src }, 'stat error for file in cache')
return callback(err, false) return callback(err, false)
@ -692,63 +639,48 @@ module.exports = OutputCacheManager = {
_copyFile(src, dst, callback) { _copyFile(src, dst, callback) {
// copy output file into the cache // copy output file into the cache
return fse.copy(src, dst, function (err) { fse.copy(src, dst, function (err) {
if ((err != null ? err.code : undefined) === 'ENOENT') { if (err?.code === 'ENOENT') {
logger.warn( logger.warn(
{ err, file: src }, { err, file: src },
'file has disappeared when copying to build cache' 'file has disappeared when copying to build cache'
) )
return callback(err, false) callback(err, false)
} else if (err != null) { } else if (err) {
logger.error({ err, src, dst }, 'copy error for file in cache') logger.error({ err, src, dst }, 'copy error for file in cache')
return callback(err) callback(err)
} else { } else {
if ( if (Settings.clsi?.optimiseInDocker) {
Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined
) {
// don't run any optimisations on the pdf when they are done // don't run any optimisations on the pdf when they are done
// in the docker container // in the docker container
return callback() callback()
} else { } else {
// call the optimiser for the file too // call the optimiser for the file too
return OutputFileOptimiser.optimiseFile(src, dst, callback) OutputFileOptimiser.optimiseFile(src, dst, callback)
} }
} }
}) })
}, },
_checkIfShouldCopy(src, callback) { _checkIfShouldCopy(src, callback) {
if (callback == null) { callback(null, !Path.basename(src).match(/^strace/))
callback = function () {}
}
return callback(null, !Path.basename(src).match(/^strace/))
}, },
_checkIfShouldArchive(src, callback) { _checkIfShouldArchive(src, callback) {
let needle
if (callback == null) {
callback = function () {}
}
if (Path.basename(src).match(/^strace/)) { if (Path.basename(src).match(/^strace/)) {
return callback(null, true) return callback(null, true)
} }
const basename = Path.basename(src)
if ( if (
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) && Settings.clsi?.archive_logs &&
((needle = Path.basename(src)), ['output.log', 'output.blg'].includes(basename)
['output.log', 'output.blg'].includes(needle))
) { ) {
return callback(null, true) return callback(null, true)
} }
return callback(null, false) callback(null, false)
}, },
} }
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}
OutputCacheManager.promises = { OutputCacheManager.promises = {
expireOutputFiles: promisify(OutputCacheManager.expireOutputFiles), expireOutputFiles: promisify(OutputCacheManager.expireOutputFiles),
saveOutputFiles: promisify(OutputCacheManager.saveOutputFiles), saveOutputFiles: promisify(OutputCacheManager.saveOutputFiles),