overleaf/services/track-changes/app/coffee/MongoAWS.js

/*
 * decaffeinate suggestions:
 * DS101: Remove unnecessary use of Array.from
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
let MongoAWS;
const settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const AWS = require('aws-sdk');
const S3S = require('s3-streams');
const {db, ObjectId} = require("./mongojs");
const JSONStream = require("JSONStream");
const ReadlineStream = require("byline");
const zlib = require("zlib");
const Metrics = require("metrics-sharelatex");

const DAYS = 24 * 3600 * 1000; // one day in milliseconds

const createStream = function(streamConstructor, project_id, doc_id, pack_id) {
	const AWS_CONFIG = {
		accessKeyId: settings.trackchanges.s3.key,
		secretAccessKey: settings.trackchanges.s3.secret,
		endpoint: settings.trackchanges.s3.endpoint,
		s3ForcePathStyle: settings.trackchanges.s3.pathStyle
	};

	return streamConstructor(new AWS.S3(AWS_CONFIG), {
		"Bucket": settings.trackchanges.stores.doc_history,
		"Key": project_id+"/changes-"+doc_id+"/pack-"+pack_id
	});
};

module.exports = (MongoAWS = {

	archivePack(project_id, doc_id, pack_id, _callback) {

		if (_callback == null) { _callback = function(error) {}; }
		const callback = function(...args) {
			_callback(...Array.from(args || []));
			return _callback = function() {};
		};

		const query = {
			_id: ObjectId(pack_id),
			doc_id: ObjectId(doc_id)
		};

		if ((project_id == null)) { return callback(new Error("invalid project id")); }
		if ((doc_id == null)) { return callback(new Error("invalid doc id")); }
		if ((pack_id == null)) { return callback(new Error("invalid pack id")); }

		logger.log({project_id, doc_id, pack_id}, "uploading data to s3");

		const upload = createStream(S3S.WriteStream, project_id, doc_id, pack_id);

		return db.docHistory.findOne(query, function(err, result) {
			if (err != null) { return callback(err); }
			if ((result == null)) { return callback(new Error("cannot find pack to send to s3")); }
			if (result.expiresAt != null) { return callback(new Error("refusing to send pack with TTL to s3")); }
			const uncompressedData = JSON.stringify(result);
			if (uncompressedData.indexOf("\u0000") !== -1) {
				const error = new Error("null bytes found in upload");
				logger.error({err: error, project_id, doc_id, pack_id}, error.message);
				return callback(error);
			}
			return zlib.gzip(uncompressedData, function(err, buf) {
				logger.log({project_id, doc_id, pack_id, origSize: uncompressedData.length, newSize: buf.length}, "compressed pack");
				if (err != null) { return callback(err); }
				upload.on('error', err => callback(err));
				upload.on('finish', function() {
					Metrics.inc("archive-pack");
					logger.log({project_id, doc_id, pack_id}, "upload to s3 completed");
					return callback(null);
				});
				upload.write(buf);
				return upload.end();
			});
		});
	},

	readArchivedPack(project_id, doc_id, pack_id, _callback) {
		if (_callback == null) { _callback = function(error, result) {}; }
		const callback = function(...args) {
			_callback(...Array.from(args || []));
			return _callback = function() {};
		};

		if ((project_id == null)) { return callback(new Error("invalid project id")); }
		if ((doc_id == null)) { return callback(new Error("invalid doc id")); }
		if ((pack_id == null)) { return callback(new Error("invalid pack id")); }

		logger.log({project_id, doc_id, pack_id}, "downloading data from s3");

		const download = createStream(S3S.ReadStream, project_id, doc_id, pack_id);

		const inputStream = download
			.on('open', obj => 1).on('error', err => callback(err));

		const gunzip = zlib.createGunzip();
		gunzip.setEncoding('utf8');
		gunzip.on('error', function(err) {
			logger.log({project_id, doc_id, pack_id, err}, "error uncompressing gzip stream");
			return callback(err);
		});

		const outputStream = inputStream.pipe(gunzip);
		const parts = [];
		outputStream.on('error', err => callback(err));
		outputStream.on('end', function() {
			let object;
			logger.log({project_id, doc_id, pack_id}, "download from s3 completed");
			try {
				object = JSON.parse(parts.join(''));
			} catch (e) {
				return callback(e);
			}
			object._id = ObjectId(object._id);
			object.doc_id = ObjectId(object.doc_id);
			object.project_id = ObjectId(object.project_id);
			for (let op of Array.from(object.pack)) {
				if (op._id != null) { op._id = ObjectId(op._id); }
			}
			return callback(null, object);
		});
		return outputStream.on('data', data => parts.push(data));
	},

	unArchivePack(project_id, doc_id, pack_id, callback) {
		if (callback == null) { callback = function(error) {}; }
		return MongoAWS.readArchivedPack(project_id, doc_id, pack_id, function(err, object) {
			if (err != null) { return callback(err); }
			Metrics.inc("unarchive-pack");
			// allow the object to expire, we can always retrieve it again
			object.expiresAt = new Date(Date.now() + (7 * DAYS));
			logger.log({project_id, doc_id, pack_id}, "inserting object from s3");
			return db.docHistory.insert(object, callback);
		});
	}
});