overleaf/services/filestore/app/js/FileHandler.js

217 lines
6.7 KiB
JavaScript
Raw Normal View History

const Settings = require('@overleaf/settings')
const { callbackify } = require('util')
const fs = require('fs')
const PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
const ImageOptimiser = require('./ImageOptimiser')
const { ConversionError, InvalidParametersError } = require('./Errors')
const metrics = require('@overleaf/metrics')
module.exports = {
insertFile: callbackify(insertFile),
deleteFile: callbackify(deleteFile),
deleteProject: callbackify(deleteProject),
getFile: callbackify(getFile),
getRedirectUrl: callbackify(getRedirectUrl),
getFileSize: callbackify(getFileSize),
getDirectorySize: callbackify(getDirectorySize),
promises: {
getFile,
getRedirectUrl,
insertFile,
deleteFile,
deleteProject,
getFileSize,
2021-07-13 07:04:46 -04:00
getDirectorySize,
},
}
async function insertFile(bucket, key, stream) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
2020-04-30 08:20:40 -04:00
throw new InvalidParametersError('key does not match validation regex', {
bucket,
key,
2021-07-13 07:04:46 -04:00
convertedKey,
})
}
if (Settings.enableConversions) {
await PersistorManager.deleteDirectory(bucket, convertedKey)
}
await PersistorManager.sendStream(bucket, key, stream)
}
async function deleteFile(bucket, key) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
2020-04-30 08:20:40 -04:00
throw new InvalidParametersError('key does not match validation regex', {
bucket,
key,
2021-07-13 07:04:46 -04:00
convertedKey,
})
}
const jobs = [PersistorManager.deleteObject(bucket, key)]
if (Settings.enableConversions) {
jobs.push(PersistorManager.deleteDirectory(bucket, convertedKey))
}
await Promise.all(jobs)
}
async function deleteProject(bucket, key) {
if (!key.match(/^[0-9a-f]{24}\//i)) {
2020-04-30 08:20:40 -04:00
throw new InvalidParametersError('key does not match validation regex', {
bucket,
2021-07-13 07:04:46 -04:00
key,
})
}
await PersistorManager.deleteDirectory(bucket, key)
}
async function getFile(bucket, key, opts) {
opts = opts || {}
if (!opts.format && !opts.style) {
return PersistorManager.getObjectStream(bucket, key, opts)
} else {
return _getConvertedFile(bucket, key, opts)
}
}
let ACTIVE_SIGNED_URL_CALLS = 0
async function getRedirectUrl(bucket, key, opts) {
// if we're doing anything unusual with options, or the request isn't for
// one of the default buckets, return null so that we proxy the file
opts = opts || {}
if (
!opts.start &&
!opts.end &&
!opts.format &&
!opts.style &&
Object.values(Settings.filestore.stores).includes(bucket) &&
Settings.filestore.allowRedirects
) {
// record the number of in-flight calls to generate signed URLs
metrics.gauge('active_signed_url_calls', ++ACTIVE_SIGNED_URL_CALLS, {
path: bucket,
})
try {
const timer = new metrics.Timer('signed_url_call_time', {
path: bucket,
})
const redirectUrl = await PersistorManager.getRedirectUrl(bucket, key)
timer.done()
return redirectUrl
} finally {
metrics.gauge('active_signed_url_calls', --ACTIVE_SIGNED_URL_CALLS, {
path: bucket,
})
}
}
return null
}
async function getFileSize(bucket, key) {
return PersistorManager.getObjectSize(bucket, key)
}
async function getDirectorySize(bucket, projectId) {
return PersistorManager.directorySize(bucket, projectId)
}
async function _getConvertedFile(bucket, key, opts) {
const convertedKey = KeyBuilder.addCachingToKey(key, opts)
const exists = await PersistorManager.checkIfObjectExists(
bucket,
convertedKey
)
if (exists) {
return PersistorManager.getObjectStream(bucket, convertedKey, opts)
} else {
return _getConvertedFileAndCache(bucket, key, convertedKey, opts)
}
}
async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) {
let convertedFsPath
try {
convertedFsPath = await _convertFile(bucket, key, opts)
await ImageOptimiser.promises.compressPng(convertedFsPath)
await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath)
} catch (err) {
LocalFileWriter.deleteFile(convertedFsPath, () => {})
throw new ConversionError(
'failed to convert file',
{ opts, bucket, key, convertedKey },
err
)
}
// Send back the converted file from the local copy to avoid problems
// with the file not being present in S3 yet. As described in the
// documentation below, we have already made a 'HEAD' request in
// checkIfFileExists so we only have "eventual consistency" if we try
// to stream it from S3 here. This was a cause of many 403 errors.
//
// "Amazon S3 provides read-after-write consistency for PUTS of new
// objects in your S3 bucket in all regions with one caveat. The
// caveat is that if you make a HEAD or GET request to the key name
// (to find if the object exists) before creating the object, Amazon
// S3 provides eventual consistency for read-after-write.""
// https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
const readStream = fs.createReadStream(convertedFsPath)
readStream.on('error', function () {
LocalFileWriter.deleteFile(convertedFsPath, function () {})
})
2020-08-10 12:01:12 -04:00
readStream.on('end', function () {
LocalFileWriter.deleteFile(convertedFsPath, function () {})
})
return readStream
}
2014-02-14 11:39:05 -05:00
async function _convertFile(bucket, originalKey, opts) {
let originalFsPath
try {
originalFsPath = await _writeFileToDisk(bucket, originalKey, opts)
} catch (err) {
throw new ConversionError(
'unable to write file to disk',
{ bucket, originalKey, opts },
err
)
}
let promise
if (opts.format) {
promise = FileConverter.promises.convert(originalFsPath, opts.format)
} else if (opts.style === 'thumbnail') {
promise = FileConverter.promises.thumbnail(originalFsPath)
} else if (opts.style === 'preview') {
promise = FileConverter.promises.preview(originalFsPath)
} else {
2020-04-30 08:20:40 -04:00
throw new ConversionError('invalid file conversion options', {
bucket,
originalKey,
2021-07-13 07:04:46 -04:00
opts,
})
}
let destPath
try {
destPath = await promise
} catch (err) {
throw new ConversionError(
'error converting file',
{ bucket, originalKey, opts },
err
)
}
2020-08-10 12:01:12 -04:00
LocalFileWriter.deleteFile(originalFsPath, function () {})
return destPath
}
async function _writeFileToDisk(bucket, key, opts) {
const fileStream = await PersistorManager.getObjectStream(bucket, key, opts)
return LocalFileWriter.promises.writeStream(fileStream, key)
}