mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-16 02:52:31 -04:00
292 lines
8.5 KiB
JavaScript
292 lines
8.5 KiB
JavaScript
/* eslint-disable
|
|
camelcase,
|
|
handle-callback-err,
|
|
no-useless-escape,
|
|
*/
|
|
// TODO: This file was created by bulk-decaffeinate.
|
|
// Fix any style issues and re-enable lint.
|
|
/*
|
|
* decaffeinate suggestions:
|
|
* DS102: Remove unnecessary code created because of implicit returns
|
|
* DS207: Consider shorter variations of null checks
|
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
*/
|
|
let DocArchive
|
|
const MongoManager = require('./MongoManager')
|
|
const Errors = require('./Errors')
|
|
const logger = require('logger-sharelatex')
|
|
const _ = require('underscore')
|
|
const async = require('async')
|
|
const settings = require('settings-sharelatex')
|
|
const request = require('request')
|
|
const crypto = require('crypto')
|
|
const RangeManager = require('./RangeManager')
|
|
const thirtySeconds = 30 * 1000
|
|
|
|
module.exports = DocArchive = {
|
|
archiveAllDocs(project_id, callback) {
|
|
if (callback == null) {
|
|
callback = function(err, docs) {}
|
|
}
|
|
return MongoManager.getProjectsDocs(
|
|
project_id,
|
|
{ include_deleted: true },
|
|
{ lines: true, ranges: true, rev: true, inS3: true },
|
|
function(err, docs) {
|
|
if (err != null) {
|
|
return callback(err)
|
|
} else if (docs == null) {
|
|
return callback(
|
|
new Errors.NotFoundError(`No docs for project ${project_id}`)
|
|
)
|
|
}
|
|
docs = _.filter(docs, doc => doc.inS3 !== true)
|
|
const jobs = _.map(docs, doc => cb =>
|
|
DocArchive.archiveDoc(project_id, doc, cb)
|
|
)
|
|
return async.parallelLimit(jobs, 5, callback)
|
|
}
|
|
)
|
|
},
|
|
|
|
archiveDoc(project_id, doc, callback) {
|
|
let options
|
|
logger.log({ project_id, doc_id: doc._id }, 'sending doc to s3')
|
|
try {
|
|
options = DocArchive.buildS3Options(project_id + '/' + doc._id)
|
|
} catch (e) {
|
|
return callback(e)
|
|
}
|
|
return DocArchive._mongoDocToS3Doc(doc, function(error, json_doc) {
|
|
if (error != null) {
|
|
return callback(error)
|
|
}
|
|
options.body = json_doc
|
|
options.headers = { 'Content-Type': 'application/json' }
|
|
return request.put(options, function(err, res) {
|
|
if (err != null || res.statusCode !== 200) {
|
|
logger.err(
|
|
{
|
|
err,
|
|
res,
|
|
project_id,
|
|
doc_id: doc._id,
|
|
statusCode: res != null ? res.statusCode : undefined
|
|
},
|
|
'something went wrong archiving doc in aws'
|
|
)
|
|
return callback(new Error('Error in S3 request'))
|
|
}
|
|
const md5lines = crypto
|
|
.createHash('md5')
|
|
.update(json_doc, 'utf8')
|
|
.digest('hex')
|
|
const md5response = res.headers.etag.toString().replace(/\"/g, '')
|
|
if (md5lines !== md5response) {
|
|
logger.err(
|
|
{
|
|
responseMD5: md5response,
|
|
linesMD5: md5lines,
|
|
project_id,
|
|
doc_id: doc != null ? doc._id : undefined
|
|
},
|
|
'err in response md5 from s3'
|
|
)
|
|
return callback(new Error('Error in S3 md5 response'))
|
|
}
|
|
return MongoManager.markDocAsArchived(doc._id, doc.rev, function(err) {
|
|
if (err != null) {
|
|
return callback(err)
|
|
}
|
|
return callback()
|
|
})
|
|
})
|
|
})
|
|
},
|
|
|
|
unArchiveAllDocs(project_id, callback) {
|
|
if (callback == null) {
|
|
callback = function(err) {}
|
|
}
|
|
return MongoManager.getArchivedProjectDocs(project_id, function(err, docs) {
|
|
if (err != null) {
|
|
logger.err({ err, project_id }, 'error unarchiving all docs')
|
|
return callback(err)
|
|
} else if (docs == null) {
|
|
return callback(
|
|
new Errors.NotFoundError(`No docs for project ${project_id}`)
|
|
)
|
|
}
|
|
const jobs = _.map(
|
|
docs,
|
|
doc =>
|
|
function(cb) {
|
|
if (doc.inS3 == null) {
|
|
return cb()
|
|
} else {
|
|
return DocArchive.unarchiveDoc(project_id, doc._id, cb)
|
|
}
|
|
}
|
|
)
|
|
return async.parallelLimit(jobs, 5, callback)
|
|
})
|
|
},
|
|
|
|
unarchiveDoc(project_id, doc_id, callback) {
|
|
let options
|
|
logger.log({ project_id, doc_id }, 'getting doc from s3')
|
|
try {
|
|
options = DocArchive.buildS3Options(project_id + '/' + doc_id)
|
|
} catch (e) {
|
|
return callback(e)
|
|
}
|
|
options.json = true
|
|
return request.get(options, function(err, res, doc) {
|
|
if (err != null || res.statusCode !== 200) {
|
|
logger.err(
|
|
{ err, res, project_id, doc_id },
|
|
'something went wrong unarchiving doc from aws'
|
|
)
|
|
return callback(new Errors.NotFoundError('Error in S3 request'))
|
|
}
|
|
return DocArchive._s3DocToMongoDoc(doc, function(error, mongo_doc) {
|
|
if (error != null) {
|
|
return callback(error)
|
|
}
|
|
return MongoManager.upsertIntoDocCollection(
|
|
project_id,
|
|
doc_id.toString(),
|
|
mongo_doc,
|
|
function(err) {
|
|
if (err != null) {
|
|
return callback(err)
|
|
}
|
|
logger.log({ project_id, doc_id }, 'deleting doc from s3')
|
|
return DocArchive._deleteDocFromS3(project_id, doc_id, callback)
|
|
}
|
|
)
|
|
})
|
|
})
|
|
},
|
|
|
|
destroyAllDocs(project_id, callback) {
|
|
if (callback == null) {
|
|
callback = function(err) {}
|
|
}
|
|
return MongoManager.getProjectsDocs(
|
|
project_id,
|
|
{ include_deleted: true },
|
|
{ _id: 1 },
|
|
function(err, docs) {
|
|
if (err != null) {
|
|
logger.err({ err, project_id }, "error getting project's docs")
|
|
return callback(err)
|
|
} else if (docs == null) {
|
|
return callback()
|
|
}
|
|
const jobs = _.map(docs, doc => cb =>
|
|
DocArchive.destroyDoc(project_id, doc._id, cb)
|
|
)
|
|
return async.parallelLimit(jobs, 5, callback)
|
|
}
|
|
)
|
|
},
|
|
|
|
destroyDoc(project_id, doc_id, callback) {
|
|
logger.log({ project_id, doc_id }, 'removing doc from mongo and s3')
|
|
return MongoManager.findDoc(project_id, doc_id, { inS3: 1 }, function(
|
|
error,
|
|
doc
|
|
) {
|
|
if (error != null) {
|
|
return callback(error)
|
|
}
|
|
if (doc == null) {
|
|
return callback(new Errors.NotFoundError('Doc not found in Mongo'))
|
|
}
|
|
if (doc.inS3 === true) {
|
|
return DocArchive._deleteDocFromS3(project_id, doc_id, function(err) {
|
|
if (err != null) {
|
|
return err
|
|
}
|
|
return MongoManager.destroyDoc(doc_id, callback)
|
|
})
|
|
} else {
|
|
return MongoManager.destroyDoc(doc_id, callback)
|
|
}
|
|
})
|
|
},
|
|
|
|
_deleteDocFromS3(project_id, doc_id, callback) {
|
|
let options
|
|
try {
|
|
options = DocArchive.buildS3Options(project_id + '/' + doc_id)
|
|
} catch (e) {
|
|
return callback(e)
|
|
}
|
|
options.json = true
|
|
return request.del(options, function(err, res, body) {
|
|
if (err != null || res.statusCode !== 204) {
|
|
logger.err(
|
|
{ err, res, project_id, doc_id },
|
|
'something went wrong deleting doc from aws'
|
|
)
|
|
return callback(new Error('Error in S3 request'))
|
|
}
|
|
return callback()
|
|
})
|
|
},
|
|
|
|
_s3DocToMongoDoc(doc, callback) {
|
|
if (callback == null) {
|
|
callback = function(error, mongo_doc) {}
|
|
}
|
|
const mongo_doc = {}
|
|
if (doc.schema_v === 1 && doc.lines != null) {
|
|
mongo_doc.lines = doc.lines
|
|
if (doc.ranges != null) {
|
|
mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges)
|
|
}
|
|
} else if (doc instanceof Array) {
|
|
mongo_doc.lines = doc
|
|
} else {
|
|
return callback(new Error("I don't understand the doc format in s3"))
|
|
}
|
|
return callback(null, mongo_doc)
|
|
},
|
|
|
|
_mongoDocToS3Doc(doc, callback) {
|
|
if (callback == null) {
|
|
callback = function(error, s3_doc) {}
|
|
}
|
|
if (doc.lines == null) {
|
|
return callback(new Error('doc has no lines'))
|
|
}
|
|
const json = JSON.stringify({
|
|
lines: doc.lines,
|
|
ranges: doc.ranges,
|
|
schema_v: 1
|
|
})
|
|
if (json.indexOf('\u0000') !== -1) {
|
|
const error = new Error('null bytes detected')
|
|
logger.err({ err: error, doc, json }, error.message)
|
|
return callback(error)
|
|
}
|
|
return callback(null, json)
|
|
},
|
|
|
|
buildS3Options(key) {
|
|
if (settings.docstore.s3 == null) {
|
|
throw new Error('S3 settings are not configured')
|
|
}
|
|
return {
|
|
aws: {
|
|
key: settings.docstore.s3.key,
|
|
secret: settings.docstore.s3.secret,
|
|
bucket: settings.docstore.s3.bucket
|
|
},
|
|
timeout: thirtySeconds,
|
|
uri: `https://${settings.docstore.s3.bucket}.s3.amazonaws.com/${key}`
|
|
}
|
|
}
|
|
}
|