overleaf/services/filestore/app/js/FSPersistor.js

239 lines
6.3 KiB
JavaScript
Raw Normal View History

const fs = require('fs')
const glob = require('glob')
const path = require('path')
const rimraf = require('rimraf')
2020-01-02 06:29:28 -05:00
const Stream = require('stream')
const { promisify, callbackify } = require('util')
const LocalFileWriter = require('./LocalFileWriter').promises
const { NotFoundError, ReadError, WriteError } = require('./Errors')
const PersistorHelper = require('./PersistorHelper')
2020-01-02 06:29:28 -05:00
const pipeline = promisify(Stream.pipeline)
const fsUnlink = promisify(fs.unlink)
const fsOpen = promisify(fs.open)
const fsStat = promisify(fs.stat)
const fsGlob = promisify(glob)
2020-01-02 06:29:28 -05:00
const rmrf = promisify(rimraf)
const filterName = key => key.replace(/\//g, '_')
2020-01-02 06:29:28 -05:00
async function sendFile(location, target, source) {
const filteredTarget = filterName(target)
// actually copy the file (instead of moving it) to maintain consistent behaviour
// between the different implementations
try {
const sourceStream = fs.createReadStream(source)
const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`)
await pipeline(sourceStream, targetStream)
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to copy the specified file',
{ location, target, source },
WriteError
)
}
2020-01-02 06:29:28 -05:00
}
async function sendStream(location, target, sourceStream, sourceMd5) {
const fsPath = await LocalFileWriter.writeStream(sourceStream)
if (!sourceMd5) {
sourceMd5 = await _getFileMd5HashForPath(fsPath)
}
2020-01-02 06:29:28 -05:00
try {
await sendFile(location, target, fsPath)
const destMd5 = await getFileMd5Hash(location, target)
if (sourceMd5 !== destMd5) {
await LocalFileWriter.deleteFile(`${location}/${filterName(target)}`)
throw new WriteError({
message: 'md5 hash mismatch',
info: { sourceMd5, destMd5, location, target }
})
}
2020-01-02 06:29:28 -05:00
} finally {
await LocalFileWriter.deleteFile(fsPath)
}
}
// opts may be {start: Number, end: Number}
async function getFileStream(location, name, opts) {
const filteredName = filterName(name)
try {
opts.fd = await fsOpen(`${location}/${filteredName}`, 'r')
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to open file for streaming',
{ location, filteredName, opts },
ReadError
)
2020-01-02 06:29:28 -05:00
}
return fs.createReadStream(null, opts)
}
async function getFileSize(location, filename) {
const fullPath = path.join(location, filterName(filename))
try {
const stat = await fsStat(fullPath)
return stat.size
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to stat file',
{ location, filename },
ReadError
)
2020-01-02 06:29:28 -05:00
}
}
async function getFileMd5Hash(location, filename) {
const fullPath = path.join(location, filterName(filename))
try {
return await _getFileMd5HashForPath(fullPath)
} catch (err) {
throw new ReadError({
message: 'unable to get md5 hash from file',
info: { location, filename }
}).withCause(err)
}
}
2020-01-02 06:29:28 -05:00
async function copyFile(location, fromName, toName) {
const filteredFromName = filterName(fromName)
const filteredToName = filterName(toName)
try {
const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`)
const targetStream = fs.createWriteStream(`${location}/${filteredToName}`)
await pipeline(sourceStream, targetStream)
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to copy file',
{ location, filteredFromName, filteredToName },
WriteError
)
}
2020-01-02 06:29:28 -05:00
}
async function deleteFile(location, name) {
const filteredName = filterName(name)
try {
await fsUnlink(`${location}/${filteredName}`)
} catch (err) {
const wrappedError = PersistorHelper.wrapError(
err,
'failed to delete file',
{ location, filteredName },
WriteError
)
if (!(wrappedError instanceof NotFoundError)) {
// S3 doesn't give us a 404 when a file wasn't there to be deleted, so we
// should be consistent here as well
throw wrappedError
}
}
2020-01-02 06:29:28 -05:00
}
// this is only called internally for clean-up by `FileHandler` and isn't part of the external API
2020-01-02 06:29:28 -05:00
async function deleteDirectory(location, name) {
const filteredName = filterName(name.replace(/\/$/, ''))
try {
await rmrf(`${location}/${filteredName}`)
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to delete directory',
{ location, filteredName },
WriteError
)
}
2020-01-02 06:29:28 -05:00
}
async function checkIfFileExists(location, name) {
const filteredName = filterName(name)
try {
const stat = await fsStat(`${location}/${filteredName}`)
return !!stat
} catch (err) {
if (err.code === 'ENOENT') {
return false
}
throw PersistorHelper.wrapError(
err,
'failed to stat file',
{ location, filteredName },
ReadError
)
2020-01-02 06:29:28 -05:00
}
}
// note, does not recurse into subdirectories, as we use a flattened directory structure
2020-01-02 06:29:28 -05:00
async function directorySize(location, name) {
const filteredName = filterName(name.replace(/\/$/, ''))
let size = 0
try {
const files = await fsGlob(`${location}/${filteredName}_*`)
2020-01-02 06:29:28 -05:00
for (const file of files) {
try {
const stat = await fsStat(file)
if (stat.isFile()) {
size += stat.size
}
} catch (err) {
// ignore files that may have just been deleted
if (err.code !== 'ENOENT') {
throw err
}
}
}
2020-01-02 06:29:28 -05:00
} catch (err) {
throw PersistorHelper.wrapError(
err,
'failed to get directory size',
{ location, name },
ReadError
)
2020-01-02 06:29:28 -05:00
}
return size
}
module.exports = {
sendFile: callbackify(sendFile),
sendStream: callbackify(sendStream),
getFileStream: callbackify(getFileStream),
getFileSize: callbackify(getFileSize),
getFileMd5Hash: callbackify(getFileMd5Hash),
2020-01-02 06:29:28 -05:00
copyFile: callbackify(copyFile),
deleteFile: callbackify(deleteFile),
deleteDirectory: callbackify(deleteDirectory),
checkIfFileExists: callbackify(checkIfFileExists),
directorySize: callbackify(directorySize),
promises: {
sendFile,
sendStream,
getFileStream,
getFileSize,
getFileMd5Hash,
2020-01-02 06:29:28 -05:00
copyFile,
deleteFile,
deleteDirectory,
checkIfFileExists,
directorySize
}
}
async function _getFileMd5HashForPath(fullPath) {
const stream = fs.createReadStream(fullPath)
return PersistorHelper.calculateStreamMd5(stream)
}