overleaf/services/filestore/app/js/S3PersistorManager.js

/* eslint-disable
    handle-callback-err,
    new-cap,
    no-return-assign,
    no-unused-vars,
    node/no-deprecated-api,
    standard/no-callback-literal,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
 * decaffeinate suggestions:
 * DS101: Remove unnecessary use of Array.from
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
// This module is the one which is used in production.  It needs to be migrated
// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
// details. The knox library is unmaintained and has bugs.

const http = require('http');
http.globalAgent.maxSockets = 300;
const https = require('https');
https.globalAgent.maxSockets = 300;
const settings = require("settings-sharelatex");
const request = require("request");
const logger = require("logger-sharelatex");
const metrics = require("metrics-sharelatex");
const fs = require("fs");
const knox = require("knox");
const path = require("path");
const LocalFileWriter = require("./LocalFileWriter");
const Errors = require("./Errors");
const _ = require("underscore");
const awsS3 = require("aws-sdk/clients/s3");
const URL = require('url');

const thirtySeconds = 30 * 1000;

const buildDefaultOptions = function(bucketName, method, key){
	let endpoint;
	if (settings.filestore.s3.endpoint) {
		endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`;
	} else {
		endpoint = `https://${bucketName}.s3.amazonaws.com`;
	}
	return {
			aws: {
				key: settings.filestore.s3.key,
				secret: settings.filestore.s3.secret,
				bucket: bucketName
			},
			method,
			timeout: thirtySeconds,
			uri:`${endpoint}/${key}`
	};
};

const getS3Options = function(credentials) {
	const options = {
		credentials: {
			accessKeyId: credentials.auth_key,
			secretAccessKey: credentials.auth_secret
		}
	};

	if (settings.filestore.s3.endpoint) {
		const endpoint = URL.parse(settings.filestore.s3.endpoint);
		options.endpoint = settings.filestore.s3.endpoint;
		options.sslEnabled = endpoint.protocol === 'https';
	}

	return options;
};

const defaultS3Client = new awsS3(getS3Options({
	auth_key: settings.filestore.s3.key,
	auth_secret: settings.filestore.s3.secret
}));

const getS3Client = function(credentials) {
	if (credentials != null) {
		return new awsS3(getS3Options(credentials));
	} else {
		return defaultS3Client;
	}
};

const getKnoxClient = bucketName => {
	const options = {
		key: settings.filestore.s3.key,
		secret: settings.filestore.s3.secret,
		bucket: bucketName
	};
	if (settings.filestore.s3.endpoint) {
		const endpoint = URL.parse(settings.filestore.s3.endpoint);
		options.endpoint = endpoint.hostname;
		options.port = endpoint.port;
	}
	return knox.createClient(options);
};

module.exports = {

	sendFile(bucketName, key, fsPath, callback){
		const s3Client = getKnoxClient(bucketName);
		let uploaded = 0;
		const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res){
			metrics.count('s3.egress', uploaded);
			if (err != null) {
				logger.err({err,  bucketName, key, fsPath},"something went wrong uploading file to s3");
				return callback(err);
			}
			if ((res == null)) {
				logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file");
				return callback("no response from put file");
			}
			if (res.statusCode !== 200) {
				logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file");
				return callback("non 200 response from s3 on put file");
			}
			logger.log({res,  bucketName, key, fsPath},"file uploaded to s3");
			return callback(err);
		});
		putEventEmiter.on("error", function(err){
			logger.err({err,  bucketName, key, fsPath}, "error emmited on put of file");
			return callback(err);
		});
		return putEventEmiter.on("progress", progress => uploaded = progress.written);
	},

	sendStream(bucketName, key, readStream, callback){
		logger.log({bucketName, key}, "sending file to s3");
		readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3"));
		return LocalFileWriter.writeStream(readStream, null, (err, fsPath)=> {
			if (err != null) {
				logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk");
				return callback(err);
			}
			return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error
            LocalFileWriter.deleteFile(fsPath, () => callback(err)));
		});
	},

	// opts may be {start: Number, end: Number}
	getFileStream(bucketName, key, opts, callback){
		if (callback == null) { callback = function(err, res){}; }
		opts = opts || {};
		callback = _.once(callback);
		logger.log({bucketName, key}, "getting file from s3");

		const s3 = getS3Client(opts.credentials);
		const s3Params = {
			Bucket: bucketName,
			Key: key
		};
		if ((opts.start != null) && (opts.end != null)) {
			s3Params.Range = `bytes=${opts.start}-${opts.end}`;
		}
		const s3Request = s3.getObject(s3Params);

		s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => {
			if ([403, 404].includes(statusCode)) {
				// S3 returns a 403 instead of a 404 when the user doesn't have
				// permission to list the bucket contents.
				logger.log({ bucketName, key }, "file not found in s3");
				return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null);
			}
			if (![200, 206].includes(statusCode)) {
				logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`);
				return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null);
			}
			const stream = response.httpResponse.createUnbufferedStream();
			stream.on('data', data => metrics.count('s3.ingress', data.byteLength));

			return callback(null, stream);
		});

		s3Request.on('error', err => {
			logger.err({ err, bucketName, key }, "error getting file stream from s3");
			return callback(err);
		});

		return s3Request.send();
	},

	getFileSize(bucketName, key, callback) {
		logger.log({ bucketName, key }, "getting file size from S3");
		const s3 = getS3Client();
		return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
			if (err != null) {
				if ([403, 404].includes(err.statusCode)) {
					// S3 returns a 403 instead of a 404 when the user doesn't have
					// permission to list the bucket contents.
					logger.log({
						bucketName,
						key
					}, "file not found in s3");
					callback(
						new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
					);
				} else {
					logger.err({
						bucketName,
						key,
						err
					}, "error performing S3 HeadObject");
					callback(err);
				}
				return;
			}
			return callback(null, data.ContentLength);
		});
	},

	copyFile(bucketName, sourceKey, destKey, callback){
		logger.log({bucketName, sourceKey, destKey}, "copying file in s3");
		const source = bucketName + '/' + sourceKey;
		// use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
		const s3 = getS3Client();
		return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) {
			if (err != null) {
				if (err.code === 'NoSuchKey') {
					logger.err({bucketName, sourceKey}, "original file not found in s3 when copying");
					return callback(new Errors.NotFoundError("original file not found in S3 when copying"));
				} else {
					logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws");
					return callback(err);
				}
			} else {
				return callback();
			}
		});
	},

	deleteFile(bucketName, key, callback){
		logger.log({bucketName, key}, "delete file in s3");
		const options = buildDefaultOptions(bucketName, "delete", key);
		return request(options, function(err, res){
			if (err != null) {
				logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws");
			}
			return callback(err);
		});
	},

	deleteDirectory(bucketName, key, _callback){
		// deleteMultiple can call the callback multiple times so protect against this.
		const callback = function(...args) {
			_callback(...Array.from(args || []));
			return _callback = function() {};
		};

		logger.log({key, bucketName}, "deleting directory");
		const s3Client = getKnoxClient(bucketName);
		return s3Client.list({prefix:key}, function(err, data){
			if (err != null) {
				logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
				return callback(err);
			}
			const keys = _.map(data.Contents, entry => entry.Key);
			return s3Client.deleteMultiple(keys, callback);
		});
	},

	checkIfFileExists(bucketName, key, callback){
		logger.log({bucketName, key}, "checking if file exists in s3");
		const options = buildDefaultOptions(bucketName, "head", key);
		return request(options, function(err, res){
			if (err != null) {
				logger.err({err, res, bucketName, key}, "something went wrong checking file in aws");
				return callback(err);
			}
			if ((res == null)) {
				logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists");
				err = new Error(`no response from s3 ${bucketName} ${key}`);
				return callback(err);
			}
			const exists = res.statusCode === 200;
			logger.log({bucketName, key, exists}, "checked if file exsists in s3");
			return callback(err, exists);
		});
	},

	directorySize(bucketName, key, callback){
		logger.log({bucketName, key}, "get project size in s3");
		const s3Client = getKnoxClient(bucketName);
		return s3Client.list({prefix:key}, function(err, data){
			if (err != null) {
				logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
				return callback(err);
			}
			let totalSize = 0;
			_.each(data.Contents, entry => totalSize += entry.Size);
			logger.log({totalSize}, "total size");
			return callback(null, totalSize);
		});
	}
};