overleaf/services/clsi/app/js/ResourceWriter.js

382 lines
11 KiB
JavaScript
Raw Normal View History

/* eslint-disable
no-return-assign,
no-unused-vars,
no-useless-escape,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ResourceWriter
const { promisify } = require('util')
const UrlCache = require('./UrlCache')
const Path = require('path')
const fs = require('fs')
const async = require('async')
const OutputFileFinder = require('./OutputFileFinder')
const ResourceStateManager = require('./ResourceStateManager')
const Metrics = require('./Metrics')
const logger = require('@overleaf/logger')
const settings = require('@overleaf/settings')
const parallelFileDownloads = settings.parallelFileDownloads || 1
2014-02-12 12:27:43 -05:00
module.exports = ResourceWriter = {
syncResourcesToDisk(request, basePath, callback) {
if (callback == null) {
callback = function () {}
}
if (request.syncType === 'incremental') {
logger.debug(
{ projectId: request.project_id, userId: request.user_id },
'incremental sync'
)
return ResourceStateManager.checkProjectStateMatches(
request.syncState,
basePath,
2020-08-10 12:01:11 -04:00
function (error, resourceList) {
if (error != null) {
return callback(error)
}
return ResourceWriter._removeExtraneousFiles(
request,
resourceList,
basePath,
2020-08-10 12:01:11 -04:00
function (error, outputFiles, allFiles) {
if (error != null) {
return callback(error)
}
return ResourceStateManager.checkResourceFiles(
resourceList,
allFiles,
basePath,
2020-08-10 12:01:11 -04:00
function (error) {
if (error != null) {
return callback(error)
}
return ResourceWriter.saveIncrementalResourcesToDisk(
request.project_id,
request.resources,
basePath,
2020-08-10 12:01:11 -04:00
function (error) {
if (error != null) {
return callback(error)
}
return callback(null, resourceList)
}
)
}
)
}
)
}
)
}
logger.debug(
{ projectId: request.project_id, userId: request.user_id },
'full sync'
)
UrlCache.createProjectDir(request.project_id, error => {
if (error != null) {
return callback(error)
}
ResourceWriter.saveAllResourcesToDisk(
request,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return ResourceStateManager.saveProjectState(
request.syncState,
request.resources,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return callback(null, request.resources)
}
)
}
)
})
},
2017-08-01 09:35:55 -04:00
saveIncrementalResourcesToDisk(projectId, resources, basePath, callback) {
if (callback == null) {
callback = function () {}
}
return ResourceWriter._createDirectory(basePath, error => {
if (error != null) {
return callback(error)
}
2021-07-13 07:04:48 -04:00
const jobs = Array.from(resources).map(resource =>
(resource => {
return callback =>
ResourceWriter._writeResourceToDisk(
projectId,
resource,
basePath,
callback
)
})(resource)
)
return async.parallelLimit(jobs, parallelFileDownloads, callback)
})
},
2017-08-01 09:35:55 -04:00
saveAllResourcesToDisk(request, basePath, callback) {
if (callback == null) {
callback = function () {}
}
return ResourceWriter._createDirectory(basePath, error => {
if (error != null) {
return callback(error)
}
const { project_id: projectId, resources } = request
ResourceWriter._removeExtraneousFiles(
request,
resources,
basePath,
error => {
if (error != null) {
return callback(error)
}
const jobs = Array.from(resources).map(resource =>
(resource => {
return callback =>
ResourceWriter._writeResourceToDisk(
projectId,
resource,
basePath,
callback
)
})(resource)
)
return async.parallelLimit(jobs, parallelFileDownloads, callback)
}
)
})
},
2017-08-01 09:35:55 -04:00
_createDirectory(basePath, callback) {
if (callback == null) {
callback = function () {}
}
2020-08-10 12:01:11 -04:00
return fs.mkdir(basePath, function (err) {
if (err != null) {
if (err.code === 'EEXIST') {
return callback()
} else {
logger.debug({ err, dir: basePath }, 'error creating directory')
return callback(err)
}
} else {
return callback()
}
})
},
_removeExtraneousFiles(request, resources, basePath, _callback) {
if (_callback == null) {
_callback = function () {}
}
const timer = new Metrics.Timer(
'unlink-output-files',
1,
request.metricsOpts
)
2020-08-10 12:01:11 -04:00
const callback = function (error, ...result) {
timer.done()
return _callback(error, ...Array.from(result))
}
2014-02-12 12:27:43 -05:00
2021-07-13 07:04:48 -04:00
return OutputFileFinder.findOutputFiles(
resources,
basePath,
function (error, outputFiles, allFiles) {
if (error != null) {
return callback(error)
}
2014-02-12 12:27:43 -05:00
2021-07-13 07:04:48 -04:00
const jobs = []
for (const file of Array.from(outputFiles || [])) {
;(function (file) {
const { path } = file
let shouldDelete = true
2021-07-13 07:04:48 -04:00
if (
path.match(/^output\./) ||
path.match(/\.aux$/) ||
path.match(/^cache\//)
) {
// knitr cache
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (path.match(/^output-.*/)) {
// Tikz cached figures (default case)
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (path.match(/\.(pdf|dpth|md5)$/)) {
// Tikz cached figures (by extension)
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (
path.match(/\.(pygtex|pygstyle)$/) ||
path.match(/(^|\/)_minted-[^\/]+\//)
) {
// minted files/directory
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (
path.match(/\.md\.tex$/) ||
path.match(/(^|\/)_markdown_[^\/]+\//)
) {
// markdown files/directory
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (path.match(/-eps-converted-to\.pdf$/)) {
// Epstopdf generated files
shouldDelete = false
2021-07-13 07:04:48 -04:00
}
if (
path === 'output.pdf' ||
path === 'output.dvi' ||
path === 'output.log' ||
path === 'output.xdv' ||
path === 'output.stdout' ||
path === 'output.stderr'
) {
shouldDelete = true
2021-07-13 07:04:48 -04:00
}
if (path === 'output.tex') {
// created by TikzManager if present in output files
shouldDelete = true
2021-07-13 07:04:48 -04:00
}
if (shouldDelete) {
2021-07-13 07:04:48 -04:00
return jobs.push(callback =>
ResourceWriter._deleteFileIfNotDirectory(
Path.join(basePath, path),
callback
)
)
2021-07-13 07:04:48 -04:00
}
})(file)
}
return async.series(jobs, function (error) {
if (error != null) {
return callback(error)
}
2021-07-13 07:04:48 -04:00
return callback(null, outputFiles, allFiles)
})
}
2021-07-13 07:04:48 -04:00
)
},
2014-02-12 12:27:43 -05:00
_deleteFileIfNotDirectory(path, callback) {
if (callback == null) {
callback = function () {}
}
2020-08-10 12:01:11 -04:00
return fs.stat(path, function (error, stat) {
if (error != null && error.code === 'ENOENT') {
return callback()
} else if (error != null) {
logger.err(
{ err: error, path },
'error stating file in deleteFileIfNotDirectory'
)
return callback(error)
} else if (stat.isFile()) {
2020-08-10 12:01:11 -04:00
return fs.unlink(path, function (error) {
if (error != null) {
logger.err(
{ err: error, path },
'error removing file in deleteFileIfNotDirectory'
)
return callback(error)
} else {
return callback()
}
})
} else {
return callback()
}
})
},
2014-02-12 12:27:43 -05:00
_writeResourceToDisk(projectId, resource, basePath, callback) {
if (callback == null) {
callback = function () {}
}
2021-07-13 07:04:48 -04:00
return ResourceWriter.checkPath(
basePath,
resource.path,
function (error, path) {
if (error != null) {
return callback(error)
}
2021-07-13 07:04:48 -04:00
return fs.mkdir(
Path.dirname(path),
{ recursive: true },
function (error) {
if (error != null) {
return callback(error)
}
2021-07-13 07:04:48 -04:00
// TODO: Don't overwrite file if it hasn't been modified
if (resource.url != null) {
return UrlCache.downloadUrlToFile(
projectId,
2021-07-13 07:04:48 -04:00
resource.url,
path,
resource.modified,
function (err) {
if (err != null) {
logger.err(
{
err,
projectId,
2021-07-13 07:04:48 -04:00
path,
resourceUrl: resource.url,
2021-07-13 07:04:48 -04:00
modified: resource.modified,
},
'error downloading file for resources'
)
Metrics.inc('download-failed')
}
return callback()
}
) // try and continue compiling even if http resource can not be downloaded at this time
} else {
fs.writeFile(path, resource.content, callback)
}
}
)
}
)
},
2014-02-12 12:27:43 -05:00
checkPath(basePath, resourcePath, callback) {
const path = Path.normalize(Path.join(basePath, resourcePath))
if (path.slice(0, basePath.length + 1) !== basePath + '/') {
return callback(new Error('resource path is outside root directory'))
} else {
return callback(null, path)
}
2021-07-13 07:04:48 -04:00
},
}
module.exports.promises = {
syncResourcesToDisk: promisify(ResourceWriter.syncResourcesToDisk),
saveIncrementalResourcesToDisk: promisify(
ResourceWriter.saveIncrementalResourcesToDisk
),
saveAllResourcesToDisk: promisify(ResourceWriter.saveAllResourcesToDisk),
checkPath: promisify(ResourceWriter.checkPath),
}