/* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ let DocArchive; const MongoManager = require("./MongoManager"); const Errors = require("./Errors"); const logger = require("logger-sharelatex"); const _ = require("underscore"); const async = require("async"); const settings = require("settings-sharelatex"); const request = require("request"); const crypto = require("crypto"); const RangeManager = require("./RangeManager"); const thirtySeconds = 30 * 1000; module.exports = (DocArchive = { archiveAllDocs(project_id, callback) { if (callback == null) { callback = function(err, docs) {}; } return MongoManager.getProjectsDocs(project_id, {include_deleted: true}, {lines: true, ranges: true, rev: true, inS3: true}, function(err, docs) { if (err != null) { return callback(err); } else if ((docs == null)) { return callback(new Errors.NotFoundError(`No docs for project ${project_id}`)); } docs = _.filter(docs, doc => doc.inS3 !== true); const jobs = _.map(docs, doc => cb => DocArchive.archiveDoc(project_id, doc, cb)); return async.parallelLimit(jobs, 5, callback); }); }, archiveDoc(project_id, doc, callback){ let options; logger.log({project_id, doc_id: doc._id}, "sending doc to s3"); try { options = DocArchive.buildS3Options(project_id+"/"+doc._id); } catch (e) { return callback(e); } return DocArchive._mongoDocToS3Doc(doc, function(error, json_doc) { if (error != null) { return callback(error); } options.body = json_doc; options.headers = {'Content-Type': "application/json"}; return request.put(options, function(err, res) { if ((err != null) || (res.statusCode !== 200)) { logger.err({err, res, project_id, doc_id: doc._id, statusCode: (res != null ? res.statusCode : undefined)}, "something went wrong archiving doc in aws"); return callback(new Error("Error in S3 request")); } const md5lines = crypto.createHash("md5").update(json_doc, "utf8").digest("hex"); const md5response = res.headers.etag.toString().replace(/\"/g, ''); if (md5lines !== md5response) { logger.err({responseMD5:md5response, linesMD5:md5lines, project_id, doc_id: (doc != null ? doc._id : undefined)}, "err in response md5 from s3"); return callback(new Error("Error in S3 md5 response")); } return MongoManager.markDocAsArchived(doc._id, doc.rev, function(err) { if (err != null) { return callback(err); } return callback(); }); }); }); }, unArchiveAllDocs(project_id, callback) { if (callback == null) { callback = function(err) {}; } return MongoManager.getArchivedProjectDocs(project_id, function(err, docs) { if (err != null) { logger.err({err, project_id}, "error unarchiving all docs"); return callback(err); } else if ((docs == null)) { return callback(new Errors.NotFoundError(`No docs for project ${project_id}`)); } const jobs = _.map(docs, doc => (function(cb) { if ((doc.inS3 == null)) { return cb(); } else { return DocArchive.unarchiveDoc(project_id, doc._id, cb); } })); return async.parallelLimit(jobs, 5, callback); }); }, unarchiveDoc(project_id, doc_id, callback){ let options; logger.log({project_id, doc_id}, "getting doc from s3"); try { options = DocArchive.buildS3Options(project_id+"/"+doc_id); } catch (e) { return callback(e); } options.json = true; return request.get(options, function(err, res, doc){ if ((err != null) || (res.statusCode !== 200)) { logger.err({err, res, project_id, doc_id}, "something went wrong unarchiving doc from aws"); return callback(new Errors.NotFoundError("Error in S3 request")); } return DocArchive._s3DocToMongoDoc(doc, function(error, mongo_doc) { if (error != null) { return callback(error); } return MongoManager.upsertIntoDocCollection(project_id, doc_id.toString(), mongo_doc, function(err) { if (err != null) { return callback(err); } logger.log({project_id, doc_id}, "deleting doc from s3"); return DocArchive._deleteDocFromS3(project_id, doc_id, callback); }); }); }); }, destroyAllDocs(project_id, callback) { if (callback == null) { callback = function(err) {}; } return MongoManager.getProjectsDocs(project_id, {include_deleted: true}, {_id: 1}, function(err, docs) { if (err != null) { logger.err({err, project_id}, "error getting project's docs"); return callback(err); } else if ((docs == null)) { return callback(); } const jobs = _.map(docs, doc => cb => DocArchive.destroyDoc(project_id, doc._id, cb)); return async.parallelLimit(jobs, 5, callback); }); }, destroyDoc(project_id, doc_id, callback){ logger.log({project_id, doc_id}, "removing doc from mongo and s3"); return MongoManager.findDoc(project_id, doc_id, {inS3: 1}, function(error, doc) { if (error != null) { return callback(error); } if (doc == null) { return callback(new Errors.NotFoundError("Doc not found in Mongo")); } if (doc.inS3 === true) { return DocArchive._deleteDocFromS3(project_id, doc_id, function(err) { if (err != null) { return err; } return MongoManager.destroyDoc(doc_id, callback); }); } else { return MongoManager.destroyDoc(doc_id, callback); } }); }, _deleteDocFromS3(project_id, doc_id, callback) { let options; try { options = DocArchive.buildS3Options(project_id+"/"+doc_id); } catch (e) { return callback(e); } options.json = true; return request.del(options, function(err, res, body){ if ((err != null) || (res.statusCode !== 204)) { logger.err({err, res, project_id, doc_id}, "something went wrong deleting doc from aws"); return callback(new Error("Error in S3 request")); } return callback(); }); }, _s3DocToMongoDoc(doc, callback) { if (callback == null) { callback = function(error, mongo_doc) {}; } const mongo_doc = {}; if ((doc.schema_v === 1) && (doc.lines != null)) { mongo_doc.lines = doc.lines; if (doc.ranges != null) { mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges); } } else if (doc instanceof Array) { mongo_doc.lines = doc; } else { return callback(new Error("I don't understand the doc format in s3")); } return callback(null, mongo_doc); }, _mongoDocToS3Doc(doc, callback) { if (callback == null) { callback = function(error, s3_doc) {}; } if ((doc.lines == null)) { return callback(new Error("doc has no lines")); } const json = JSON.stringify({ lines: doc.lines, ranges: doc.ranges, schema_v: 1 }); if (json.indexOf("\u0000") !== -1) { const error = new Error("null bytes detected"); logger.err({err: error, doc, json}, error.message); return callback(error); } return callback(null, json); }, buildS3Options(key){ if ((settings.docstore.s3 == null)) { throw new Error("S3 settings are not configured"); } return { aws: { key: settings.docstore.s3.key, secret: settings.docstore.s3.secret, bucket: settings.docstore.s3.bucket }, timeout: thirtySeconds, uri:`https://${settings.docstore.s3.bucket}.s3.amazonaws.com/${key}` }; } });