overleaf/services/docstore/app/js/DocArchiveManager.js

/* eslint-disable
    camelcase,
    handle-callback-err,
    no-useless-escape,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
 * decaffeinate suggestions:
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
let DocArchive
const MongoManager = require('./MongoManager')
const Errors = require('./Errors')
const logger = require('logger-sharelatex')
const _ = require('underscore')
const async = require('async')
const settings = require('settings-sharelatex')
const request = require('request')
const crypto = require('crypto')
const RangeManager = require('./RangeManager')
const thirtySeconds = 30 * 1000

module.exports = DocArchive = {
  archiveAllDocs(project_id, callback) {
    if (callback == null) {
      callback = function(err, docs) {}
    }
    return MongoManager.getProjectsDocs(
      project_id,
      { include_deleted: true },
      { lines: true, ranges: true, rev: true, inS3: true },
      function(err, docs) {
        if (err != null) {
          return callback(err)
        } else if (docs == null) {
          return callback(
            new Errors.NotFoundError(`No docs for project ${project_id}`)
          )
        }
        docs = _.filter(docs, doc => doc.inS3 !== true)
        const jobs = _.map(docs, doc => cb =>
          DocArchive.archiveDoc(project_id, doc, cb)
        )
        return async.parallelLimit(jobs, 5, callback)
      }
    )
  },

  archiveDoc(project_id, doc, callback) {
    let options
    logger.log({ project_id, doc_id: doc._id }, 'sending doc to s3')
    try {
      options = DocArchive.buildS3Options(project_id + '/' + doc._id)
    } catch (e) {
      return callback(e)
    }
    return DocArchive._mongoDocToS3Doc(doc, function(error, json_doc) {
      if (error != null) {
        return callback(error)
      }
      options.body = json_doc
      options.headers = { 'Content-Type': 'application/json' }
      return request.put(options, function(err, res) {
        if (err != null || res.statusCode !== 200) {
          logger.err(
            {
              err,
              res,
              project_id,
              doc_id: doc._id,
              statusCode: res != null ? res.statusCode : undefined
            },
            'something went wrong archiving doc in aws'
          )
          return callback(new Error('Error in S3 request'))
        }
        const md5lines = crypto
          .createHash('md5')
          .update(json_doc, 'utf8')
          .digest('hex')
        const md5response = res.headers.etag.toString().replace(/\"/g, '')
        if (md5lines !== md5response) {
          logger.err(
            {
              responseMD5: md5response,
              linesMD5: md5lines,
              project_id,
              doc_id: doc != null ? doc._id : undefined
            },
            'err in response md5 from s3'
          )
          return callback(new Error('Error in S3 md5 response'))
        }
        return MongoManager.markDocAsArchived(doc._id, doc.rev, function(err) {
          if (err != null) {
            return callback(err)
          }
          return callback()
        })
      })
    })
  },

  unArchiveAllDocs(project_id, callback) {
    if (callback == null) {
      callback = function(err) {}
    }
    return MongoManager.getArchivedProjectDocs(project_id, function(err, docs) {
      if (err != null) {
        logger.err({ err, project_id }, 'error unarchiving all docs')
        return callback(err)
      } else if (docs == null) {
        return callback(
          new Errors.NotFoundError(`No docs for project ${project_id}`)
        )
      }
      const jobs = _.map(
        docs,
        doc =>
          function(cb) {
            if (doc.inS3 == null) {
              return cb()
            } else {
              return DocArchive.unarchiveDoc(project_id, doc._id, cb)
            }
          }
      )
      return async.parallelLimit(jobs, 5, callback)
    })
  },

  unarchiveDoc(project_id, doc_id, callback) {
    let options
    logger.log({ project_id, doc_id }, 'getting doc from s3')
    try {
      options = DocArchive.buildS3Options(project_id + '/' + doc_id)
    } catch (e) {
      return callback(e)
    }
    options.json = true
    return request.get(options, function(err, res, doc) {
      if (err != null || res.statusCode !== 200) {
        logger.err(
          { err, res, project_id, doc_id },
          'something went wrong unarchiving doc from aws'
        )
        return callback(new Errors.NotFoundError('Error in S3 request'))
      }
      return DocArchive._s3DocToMongoDoc(doc, function(error, mongo_doc) {
        if (error != null) {
          return callback(error)
        }
        return MongoManager.upsertIntoDocCollection(
          project_id,
          doc_id.toString(),
          mongo_doc,
          function(err) {
            if (err != null) {
              return callback(err)
            }
            logger.log({ project_id, doc_id }, 'deleting doc from s3')
            return DocArchive._deleteDocFromS3(project_id, doc_id, callback)
          }
        )
      })
    })
  },

  destroyAllDocs(project_id, callback) {
    if (callback == null) {
      callback = function(err) {}
    }
    return MongoManager.getProjectsDocs(
      project_id,
      { include_deleted: true },
      { _id: 1 },
      function(err, docs) {
        if (err != null) {
          logger.err({ err, project_id }, "error getting project's docs")
          return callback(err)
        } else if (docs == null) {
          return callback()
        }
        const jobs = _.map(docs, doc => cb =>
          DocArchive.destroyDoc(project_id, doc._id, cb)
        )
        return async.parallelLimit(jobs, 5, callback)
      }
    )
  },

  destroyDoc(project_id, doc_id, callback) {
    logger.log({ project_id, doc_id }, 'removing doc from mongo and s3')
    return MongoManager.findDoc(project_id, doc_id, { inS3: 1 }, function(
      error,
      doc
    ) {
      if (error != null) {
        return callback(error)
      }
      if (doc == null) {
        return callback(new Errors.NotFoundError('Doc not found in Mongo'))
      }
      if (doc.inS3 === true) {
        return DocArchive._deleteDocFromS3(project_id, doc_id, function(err) {
          if (err != null) {
            return err
          }
          return MongoManager.destroyDoc(doc_id, callback)
        })
      } else {
        return MongoManager.destroyDoc(doc_id, callback)
      }
    })
  },

  _deleteDocFromS3(project_id, doc_id, callback) {
    let options
    try {
      options = DocArchive.buildS3Options(project_id + '/' + doc_id)
    } catch (e) {
      return callback(e)
    }
    options.json = true
    return request.del(options, function(err, res, body) {
      if (err != null || res.statusCode !== 204) {
        logger.err(
          { err, res, project_id, doc_id },
          'something went wrong deleting doc from aws'
        )
        return callback(new Error('Error in S3 request'))
      }
      return callback()
    })
  },

  _s3DocToMongoDoc(doc, callback) {
    if (callback == null) {
      callback = function(error, mongo_doc) {}
    }
    const mongo_doc = {}
    if (doc.schema_v === 1 && doc.lines != null) {
      mongo_doc.lines = doc.lines
      if (doc.ranges != null) {
        mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges)
      }
    } else if (doc instanceof Array) {
      mongo_doc.lines = doc
    } else {
      return callback(new Error("I don't understand the doc format in s3"))
    }
    return callback(null, mongo_doc)
  },

  _mongoDocToS3Doc(doc, callback) {
    if (callback == null) {
      callback = function(error, s3_doc) {}
    }
    if (doc.lines == null) {
      return callback(new Error('doc has no lines'))
    }
    const json = JSON.stringify({
      lines: doc.lines,
      ranges: doc.ranges,
      schema_v: 1
    })
    if (json.indexOf('\u0000') !== -1) {
      const error = new Error('null bytes detected')
      logger.err({ err: error, doc, json }, error.message)
      return callback(error)
    }
    return callback(null, json)
  },

  buildS3Options(key) {
    if (settings.docstore.s3 == null) {
      throw new Error('S3 settings are not configured')
    }
    return {
      aws: {
        key: settings.docstore.s3.key,
        secret: settings.docstore.s3.secret,
        bucket: settings.docstore.s3.bucket
      },
      timeout: thirtySeconds,
      uri: `https://${settings.docstore.s3.bucket}.s3.amazonaws.com/${key}`
    }
  }
}