overleaf/services/filestore/app/js/FileHandler.js

const { promisify } = require('util')
const fs = require('fs')
const PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
const async = require('async')
const ImageOptimiser = require('./ImageOptimiser')
const { ConversionError } = require('./Errors')

module.exports = {
  insertFile,
  deleteFile,
  getFile,
  getFileSize,
  getDirectorySize,
  promises: {
    getFile: promisify(getFile),
    insertFile: promisify(insertFile),
    deleteFile: promisify(deleteFile),
    getFileSize: promisify(getFileSize),
    getDirectorySize: promisify(getDirectorySize)
  }
}

function insertFile(bucket, key, stream, callback) {
  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
  PersistorManager.deleteDirectory(bucket, convertedKey, function(error) {
    if (error) {
      return callback(error)
    }
    PersistorManager.sendStream(bucket, key, stream, callback)
  })
}

function deleteFile(bucket, key, callback) {
  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
  async.parallel(
    [
      done => PersistorManager.deleteFile(bucket, key, done),
      done => PersistorManager.deleteDirectory(bucket, convertedKey, done)
    ],
    callback
  )
}

function getFile(bucket, key, opts, callback) {
  opts = opts || {}
  if (!opts.format && !opts.style) {
    PersistorManager.getFileStream(bucket, key, opts, callback)
  } else {
    _getConvertedFile(bucket, key, opts, callback)
  }
}

function getFileSize(bucket, key, callback) {
  PersistorManager.getFileSize(bucket, key, callback)
}

function getDirectorySize(bucket, projectId, callback) {
  PersistorManager.directorySize(bucket, projectId, callback)
}

function _getConvertedFile(bucket, key, opts, callback) {
  const convertedKey = KeyBuilder.addCachingToKey(key, opts)
  PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists) => {
    if (err) {
      return callback(err)
    }

    if (exists) {
      PersistorManager.getFileStream(bucket, convertedKey, opts, callback)
    } else {
      _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback)
    }
  })
}

function _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) {
  let convertedFsPath

  async.series(
    [
      cb => {
        _convertFile(bucket, key, opts, function(err, fileSystemPath) {
          convertedFsPath = fileSystemPath
          cb(err)
        })
      },
      cb => ImageOptimiser.compressPng(convertedFsPath, cb),
      cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb)
    ],
    function(err) {
      if (err) {
        LocalFileWriter.deleteFile(convertedFsPath, function() {})
        return callback(
          new ConversionError({
            message: 'failed to convert file',
            info: { opts, bucket, key, convertedKey }
          }).withCause(err)
        )
      }
      // Send back the converted file from the local copy to avoid problems
      // with the file not being present in S3 yet.  As described in the
      // documentation below, we have already made a 'HEAD' request in
      // checkIfFileExists so we only have "eventual consistency" if we try
      // to stream it from S3 here.  This was a cause of many 403 errors.
      //
      // "Amazon S3 provides read-after-write consistency for PUTS of new
      // objects in your S3 bucket in all regions with one caveat. The
      // caveat is that if you make a HEAD or GET request to the key name
      // (to find if the object exists) before creating the object, Amazon
      // S3 provides eventual consistency for read-after-write.""
      // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
      const readStream = fs.createReadStream(convertedFsPath)
      readStream.on('end', function() {
        LocalFileWriter.deleteFile(convertedFsPath, function() {})
      })
      callback(null, readStream)
    }
  )
}

function _convertFile(bucket, originalKey, opts, callback) {
  _writeFileToDisk(bucket, originalKey, opts, function(err, originalFsPath) {
    if (err) {
      return callback(
        new ConversionError({
          message: 'unable to write file to disk',
          info: { bucket, originalKey, opts }
        }).withCause(err)
      )
    }

    const done = function(err, destPath) {
      if (err) {
        return callback(
          new ConversionError({
            message: 'error converting file',
            info: { bucket, originalKey, opts }
          }).withCause(err)
        )
      }
      LocalFileWriter.deleteFile(originalFsPath, function() {})
      callback(err, destPath)
    }

    if (opts.format) {
      FileConverter.convert(originalFsPath, opts.format, done)
    } else if (opts.style === 'thumbnail') {
      FileConverter.thumbnail(originalFsPath, done)
    } else if (opts.style === 'preview') {
      FileConverter.preview(originalFsPath, done)
    } else {
      callback(
        new ConversionError({
          message: 'invalid file conversion options',
          info: {
            bucket,
            originalKey,
            opts
          }
        })
      )
    }
  })
}

function _writeFileToDisk(bucket, key, opts, callback) {
  PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) {
    if (err) {
      return callback(err)
    }
    LocalFileWriter.writeStream(fileStream, key, callback)
  })
}