overleaf/services/filestore/app/js/FSPersistorManager.js

const fs = require('fs')
const glob = require('glob')
const logger = require('logger-sharelatex')
const path = require('path')
const rimraf = require('rimraf')
const Stream = require('stream')
const { promisify, callbackify } = require('util')

const LocalFileWriter = require('./LocalFileWriter').promises
const { NotFoundError, ReadError } = require('./Errors')

const pipeline = promisify(Stream.pipeline)
const fsUnlink = promisify(fs.unlink)
const fsOpen = promisify(fs.open)
const fsStat = promisify(fs.stat)
const fsGlob = promisify(glob)
const rmrf = promisify(rimraf)

const filterName = key => key.replace(/\//g, '_')

async function sendFile(location, target, source) {
  const filteredTarget = filterName(target)
  logger.log({ location, target: filteredTarget, source }, 'sending file')

  // actually copy the file (instead of moving it) to maintain consistent behaviour
  // between the different implementations
  const sourceStream = fs.createReadStream(source)
  const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`)
  await pipeline(sourceStream, targetStream)
}

async function sendStream(location, target, sourceStream) {
  logger.log({ location, target }, 'sending file stream')

  let fsPath
  try {
    fsPath = await LocalFileWriter.writeStream(sourceStream)
    await sendFile(location, target, fsPath)
  } finally {
    await LocalFileWriter.deleteFile(fsPath)
  }
}

// opts may be {start: Number, end: Number}
async function getFileStream(location, name, opts) {
  const filteredName = filterName(name)
  logger.log({ location, filteredName }, 'getting file')

  try {
    opts.fd = await fsOpen(`${location}/${filteredName}`, 'r')
  } catch (err) {
    logger.err({ err, location, filteredName: name }, 'Error reading from file')

    if (err.code === 'ENOENT') {
      throw new NotFoundError({
        message: 'file not found',
        info: {
          location,
          filteredName
        }
      }).withCause(err)
    }
    throw new ReadError('failed to open file for streaming').withCause(err)
  }

  return fs.createReadStream(null, opts)
}

async function getFileSize(location, filename) {
  const fullPath = path.join(location, filterName(filename))

  try {
    const stat = await fsStat(fullPath)
    return stat.size
  } catch (err) {
    logger.err({ err, location, filename }, 'failed to stat file')

    if (err.code === 'ENOENT') {
      throw new NotFoundError({
        message: 'file not found',
        info: {
          location,
          fullPath
        }
      }).withCause(err)
    }
    throw new ReadError('failed to stat file').withCause(err)
  }
}

async function copyFile(location, fromName, toName) {
  const filteredFromName = filterName(fromName)
  const filteredToName = filterName(toName)
  logger.log({ location, filteredFromName, filteredToName }, 'copying file')

  const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`)
  const targetStream = fs.createWriteStream(`${location}/${filteredToName}`)
  await pipeline(sourceStream, targetStream)
}

async function deleteFile(location, name) {
  const filteredName = filterName(name)
  logger.log({ location, filteredName }, 'delete file')
  await fsUnlink(`${location}/${filteredName}`)
}

async function deleteDirectory(location, name) {
  const filteredName = filterName(name.replace(/\/$/, ''))

  logger.log({ location, filteredName }, 'deleting directory')

  await rmrf(`${location}/${filteredName}`)
}

async function checkIfFileExists(location, name) {
  const filteredName = filterName(name)
  try {
    const stat = await fsStat(`${location}/${filteredName}`)
    return !!stat
  } catch (err) {
    if (err.code === 'ENOENT') {
      return false
    }
    throw new ReadError('failed to stat file').withCause(err)
  }
}

// note, does not recurse into subdirectories, as we use a flattened directory structure
async function directorySize(location, name) {
  const filteredName = filterName(name.replace(/\/$/, ''))
  let size = 0

  try {
    const files = await fsGlob(`${location}/${filteredName}_*`)
    for (const file of files) {
      try {
        const stat = await fsStat(file)
        if (stat.isFile()) {
          size += stat.size
        }
      } catch (err) {
        // ignore files that may have just been deleted
        if (err.code !== 'ENOENT') {
          throw err
        }
      }
    }
  } catch (err) {
    throw new ReadError({
      message: 'failed to get directory size',
      info: { location, name }
    }).withCause(err)
  }

  return size
}

module.exports = {
  sendFile: callbackify(sendFile),
  sendStream: callbackify(sendStream),
  getFileStream: callbackify(getFileStream),
  getFileSize: callbackify(getFileSize),
  copyFile: callbackify(copyFile),
  deleteFile: callbackify(deleteFile),
  deleteDirectory: callbackify(deleteDirectory),
  checkIfFileExists: callbackify(checkIfFileExists),
  directorySize: callbackify(directorySize),
  promises: {
    sendFile,
    sendStream,
    getFileStream,
    getFileSize,
    copyFile,
    deleteFile,
    deleteDirectory,
    checkIfFileExists,
    directorySize
  }
}