overleaf/services/filestore/app/js/FileHandler.js

const Settings = require('@overleaf/settings')
const { callbackify } = require('util')
const fs = require('fs')
const PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
const ImageOptimiser = require('./ImageOptimiser')
const { ConversionError, InvalidParametersError } = require('./Errors')
const metrics = require('@overleaf/metrics')

module.exports = {
  insertFile: callbackify(insertFile),
  deleteFile: callbackify(deleteFile),
  deleteProject: callbackify(deleteProject),
  getFile: callbackify(getFile),
  getRedirectUrl: callbackify(getRedirectUrl),
  getFileSize: callbackify(getFileSize),
  getDirectorySize: callbackify(getDirectorySize),
  promises: {
    getFile,
    getRedirectUrl,
    insertFile,
    deleteFile,
    deleteProject,
    getFileSize,
    getDirectorySize,
  },
}

async function insertFile(bucket, key, stream) {
  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
  if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
    throw new InvalidParametersError('key does not match validation regex', {
      bucket,
      key,
      convertedKey,
    })
  }
  if (Settings.enableConversions) {
    await PersistorManager.deleteDirectory(bucket, convertedKey)
  }
  await PersistorManager.sendStream(bucket, key, stream)
}

async function deleteFile(bucket, key) {
  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
  if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
    throw new InvalidParametersError('key does not match validation regex', {
      bucket,
      key,
      convertedKey,
    })
  }
  const jobs = [PersistorManager.deleteObject(bucket, key)]
  if (Settings.enableConversions) {
    jobs.push(PersistorManager.deleteDirectory(bucket, convertedKey))
  }
  await Promise.all(jobs)
}

async function deleteProject(bucket, key) {
  if (!key.match(/^[0-9a-f]{24}\//i)) {
    throw new InvalidParametersError('key does not match validation regex', {
      bucket,
      key,
    })
  }
  await PersistorManager.deleteDirectory(bucket, key)
}

async function getFile(bucket, key, opts) {
  opts = opts || {}
  if (!opts.format && !opts.style) {
    return PersistorManager.getObjectStream(bucket, key, opts)
  } else {
    return _getConvertedFile(bucket, key, opts)
  }
}

let ACTIVE_SIGNED_URL_CALLS = 0

async function getRedirectUrl(bucket, key, opts) {
  // if we're doing anything unusual with options, or the request isn't for
  // one of the default buckets, return null so that we proxy the file
  opts = opts || {}
  if (
    !opts.start &&
    !opts.end &&
    !opts.format &&
    !opts.style &&
    Object.values(Settings.filestore.stores).includes(bucket) &&
    Settings.filestore.allowRedirects
  ) {
    // record the number of in-flight calls to generate signed URLs
    metrics.gauge('active_signed_url_calls', ++ACTIVE_SIGNED_URL_CALLS, {
      path: bucket,
    })
    try {
      const timer = new metrics.Timer('signed_url_call_time', {
        path: bucket,
      })
      const redirectUrl = await PersistorManager.getRedirectUrl(bucket, key)
      timer.done()
      return redirectUrl
    } finally {
      metrics.gauge('active_signed_url_calls', --ACTIVE_SIGNED_URL_CALLS, {
        path: bucket,
      })
    }
  }

  return null
}

async function getFileSize(bucket, key) {
  return PersistorManager.getObjectSize(bucket, key)
}

async function getDirectorySize(bucket, projectId) {
  return PersistorManager.directorySize(bucket, projectId)
}

async function _getConvertedFile(bucket, key, opts) {
  const convertedKey = KeyBuilder.addCachingToKey(key, opts)
  const exists = await PersistorManager.checkIfObjectExists(
    bucket,
    convertedKey
  )
  if (exists) {
    return PersistorManager.getObjectStream(bucket, convertedKey, opts)
  } else {
    return _getConvertedFileAndCache(bucket, key, convertedKey, opts)
  }
}

async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) {
  let convertedFsPath
  try {
    convertedFsPath = await _convertFile(bucket, key, opts)
    await ImageOptimiser.promises.compressPng(convertedFsPath)
    await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath)
  } catch (err) {
    LocalFileWriter.deleteFile(convertedFsPath, () => {})
    throw new ConversionError(
      'failed to convert file',
      { opts, bucket, key, convertedKey },
      err
    )
  }
  // Send back the converted file from the local copy to avoid problems
  // with the file not being present in S3 yet.  As described in the
  // documentation below, we have already made a 'HEAD' request in
  // checkIfFileExists so we only have "eventual consistency" if we try
  // to stream it from S3 here.  This was a cause of many 403 errors.
  //
  // "Amazon S3 provides read-after-write consistency for PUTS of new
  // objects in your S3 bucket in all regions with one caveat. The
  // caveat is that if you make a HEAD or GET request to the key name
  // (to find if the object exists) before creating the object, Amazon
  // S3 provides eventual consistency for read-after-write.""
  // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
  const readStream = fs.createReadStream(convertedFsPath)
  readStream.on('error', function () {
    LocalFileWriter.deleteFile(convertedFsPath, function () {})
  })
  readStream.on('end', function () {
    LocalFileWriter.deleteFile(convertedFsPath, function () {})
  })
  return readStream
}

async function _convertFile(bucket, originalKey, opts) {
  let originalFsPath
  try {
    originalFsPath = await _writeFileToDisk(bucket, originalKey, opts)
  } catch (err) {
    throw new ConversionError(
      'unable to write file to disk',
      { bucket, originalKey, opts },
      err
    )
  }

  let promise
  if (opts.format) {
    promise = FileConverter.promises.convert(originalFsPath, opts.format)
  } else if (opts.style === 'thumbnail') {
    promise = FileConverter.promises.thumbnail(originalFsPath)
  } else if (opts.style === 'preview') {
    promise = FileConverter.promises.preview(originalFsPath)
  } else {
    throw new ConversionError('invalid file conversion options', {
      bucket,
      originalKey,
      opts,
    })
  }
  let destPath
  try {
    destPath = await promise
  } catch (err) {
    throw new ConversionError(
      'error converting file',
      { bucket, originalKey, opts },
      err
    )
  }
  LocalFileWriter.deleteFile(originalFsPath, function () {})
  return destPath
}

async function _writeFileToDisk(bucket, key, opts) {
  const fileStream = await PersistorManager.getObjectStream(bucket, key, opts)
  return LocalFileWriter.promises.writeStream(fileStream, key)
}