mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-16 02:52:31 -04:00
300 lines
9.9 KiB
JavaScript
300 lines
9.9 KiB
JavaScript
/* eslint-disable
|
|
handle-callback-err,
|
|
new-cap,
|
|
no-return-assign,
|
|
no-unused-vars,
|
|
node/no-deprecated-api,
|
|
standard/no-callback-literal,
|
|
*/
|
|
// TODO: This file was created by bulk-decaffeinate.
|
|
// Fix any style issues and re-enable lint.
|
|
/*
|
|
* decaffeinate suggestions:
|
|
* DS101: Remove unnecessary use of Array.from
|
|
* DS102: Remove unnecessary code created because of implicit returns
|
|
* DS207: Consider shorter variations of null checks
|
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
*/
|
|
// This module is the one which is used in production. It needs to be migrated
|
|
// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
|
|
// details. The knox library is unmaintained and has bugs.
|
|
|
|
const http = require('http');
|
|
http.globalAgent.maxSockets = 300;
|
|
const https = require('https');
|
|
https.globalAgent.maxSockets = 300;
|
|
const settings = require("settings-sharelatex");
|
|
const request = require("request");
|
|
const logger = require("logger-sharelatex");
|
|
const metrics = require("metrics-sharelatex");
|
|
const fs = require("fs");
|
|
const knox = require("knox");
|
|
const path = require("path");
|
|
const LocalFileWriter = require("./LocalFileWriter");
|
|
const Errors = require("./Errors");
|
|
const _ = require("underscore");
|
|
const awsS3 = require("aws-sdk/clients/s3");
|
|
const URL = require('url');
|
|
|
|
const thirtySeconds = 30 * 1000;
|
|
|
|
const buildDefaultOptions = function(bucketName, method, key){
|
|
let endpoint;
|
|
if (settings.filestore.s3.endpoint) {
|
|
endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`;
|
|
} else {
|
|
endpoint = `https://${bucketName}.s3.amazonaws.com`;
|
|
}
|
|
return {
|
|
aws: {
|
|
key: settings.filestore.s3.key,
|
|
secret: settings.filestore.s3.secret,
|
|
bucket: bucketName
|
|
},
|
|
method,
|
|
timeout: thirtySeconds,
|
|
uri:`${endpoint}/${key}`
|
|
};
|
|
};
|
|
|
|
const getS3Options = function(credentials) {
|
|
const options = {
|
|
credentials: {
|
|
accessKeyId: credentials.auth_key,
|
|
secretAccessKey: credentials.auth_secret
|
|
}
|
|
};
|
|
|
|
if (settings.filestore.s3.endpoint) {
|
|
const endpoint = URL.parse(settings.filestore.s3.endpoint);
|
|
options.endpoint = settings.filestore.s3.endpoint;
|
|
options.sslEnabled = endpoint.protocol === 'https';
|
|
}
|
|
|
|
return options;
|
|
};
|
|
|
|
const defaultS3Client = new awsS3(getS3Options({
|
|
auth_key: settings.filestore.s3.key,
|
|
auth_secret: settings.filestore.s3.secret
|
|
}));
|
|
|
|
const getS3Client = function(credentials) {
|
|
if (credentials != null) {
|
|
return new awsS3(getS3Options(credentials));
|
|
} else {
|
|
return defaultS3Client;
|
|
}
|
|
};
|
|
|
|
const getKnoxClient = bucketName => {
|
|
const options = {
|
|
key: settings.filestore.s3.key,
|
|
secret: settings.filestore.s3.secret,
|
|
bucket: bucketName
|
|
};
|
|
if (settings.filestore.s3.endpoint) {
|
|
const endpoint = URL.parse(settings.filestore.s3.endpoint);
|
|
options.endpoint = endpoint.hostname;
|
|
options.port = endpoint.port;
|
|
}
|
|
return knox.createClient(options);
|
|
};
|
|
|
|
module.exports = {
|
|
|
|
sendFile(bucketName, key, fsPath, callback){
|
|
const s3Client = getKnoxClient(bucketName);
|
|
let uploaded = 0;
|
|
const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res){
|
|
metrics.count('s3.egress', uploaded);
|
|
if (err != null) {
|
|
logger.err({err, bucketName, key, fsPath},"something went wrong uploading file to s3");
|
|
return callback(err);
|
|
}
|
|
if ((res == null)) {
|
|
logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file");
|
|
return callback("no response from put file");
|
|
}
|
|
if (res.statusCode !== 200) {
|
|
logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file");
|
|
return callback("non 200 response from s3 on put file");
|
|
}
|
|
logger.log({res, bucketName, key, fsPath},"file uploaded to s3");
|
|
return callback(err);
|
|
});
|
|
putEventEmiter.on("error", function(err){
|
|
logger.err({err, bucketName, key, fsPath}, "error emmited on put of file");
|
|
return callback(err);
|
|
});
|
|
return putEventEmiter.on("progress", progress => uploaded = progress.written);
|
|
},
|
|
|
|
sendStream(bucketName, key, readStream, callback){
|
|
logger.log({bucketName, key}, "sending file to s3");
|
|
readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3"));
|
|
return LocalFileWriter.writeStream(readStream, null, (err, fsPath)=> {
|
|
if (err != null) {
|
|
logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk");
|
|
return callback(err);
|
|
}
|
|
return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error
|
|
LocalFileWriter.deleteFile(fsPath, () => callback(err)));
|
|
});
|
|
},
|
|
|
|
// opts may be {start: Number, end: Number}
|
|
getFileStream(bucketName, key, opts, callback){
|
|
if (callback == null) { callback = function(err, res){}; }
|
|
opts = opts || {};
|
|
callback = _.once(callback);
|
|
logger.log({bucketName, key}, "getting file from s3");
|
|
|
|
const s3 = getS3Client(opts.credentials);
|
|
const s3Params = {
|
|
Bucket: bucketName,
|
|
Key: key
|
|
};
|
|
if ((opts.start != null) && (opts.end != null)) {
|
|
s3Params.Range = `bytes=${opts.start}-${opts.end}`;
|
|
}
|
|
const s3Request = s3.getObject(s3Params);
|
|
|
|
s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => {
|
|
if ([403, 404].includes(statusCode)) {
|
|
// S3 returns a 403 instead of a 404 when the user doesn't have
|
|
// permission to list the bucket contents.
|
|
logger.log({ bucketName, key }, "file not found in s3");
|
|
return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null);
|
|
}
|
|
if (![200, 206].includes(statusCode)) {
|
|
logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`);
|
|
return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null);
|
|
}
|
|
const stream = response.httpResponse.createUnbufferedStream();
|
|
stream.on('data', data => metrics.count('s3.ingress', data.byteLength));
|
|
|
|
return callback(null, stream);
|
|
});
|
|
|
|
s3Request.on('error', err => {
|
|
logger.err({ err, bucketName, key }, "error getting file stream from s3");
|
|
return callback(err);
|
|
});
|
|
|
|
return s3Request.send();
|
|
},
|
|
|
|
getFileSize(bucketName, key, callback) {
|
|
logger.log({ bucketName, key }, "getting file size from S3");
|
|
const s3 = getS3Client();
|
|
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
|
|
if (err != null) {
|
|
if ([403, 404].includes(err.statusCode)) {
|
|
// S3 returns a 403 instead of a 404 when the user doesn't have
|
|
// permission to list the bucket contents.
|
|
logger.log({
|
|
bucketName,
|
|
key
|
|
}, "file not found in s3");
|
|
callback(
|
|
new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
|
|
);
|
|
} else {
|
|
logger.err({
|
|
bucketName,
|
|
key,
|
|
err
|
|
}, "error performing S3 HeadObject");
|
|
callback(err);
|
|
}
|
|
return;
|
|
}
|
|
return callback(null, data.ContentLength);
|
|
});
|
|
},
|
|
|
|
copyFile(bucketName, sourceKey, destKey, callback){
|
|
logger.log({bucketName, sourceKey, destKey}, "copying file in s3");
|
|
const source = bucketName + '/' + sourceKey;
|
|
// use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
|
|
const s3 = getS3Client();
|
|
return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) {
|
|
if (err != null) {
|
|
if (err.code === 'NoSuchKey') {
|
|
logger.err({bucketName, sourceKey}, "original file not found in s3 when copying");
|
|
return callback(new Errors.NotFoundError("original file not found in S3 when copying"));
|
|
} else {
|
|
logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws");
|
|
return callback(err);
|
|
}
|
|
} else {
|
|
return callback();
|
|
}
|
|
});
|
|
},
|
|
|
|
deleteFile(bucketName, key, callback){
|
|
logger.log({bucketName, key}, "delete file in s3");
|
|
const options = buildDefaultOptions(bucketName, "delete", key);
|
|
return request(options, function(err, res){
|
|
if (err != null) {
|
|
logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws");
|
|
}
|
|
return callback(err);
|
|
});
|
|
},
|
|
|
|
deleteDirectory(bucketName, key, _callback){
|
|
// deleteMultiple can call the callback multiple times so protect against this.
|
|
const callback = function(...args) {
|
|
_callback(...Array.from(args || []));
|
|
return _callback = function() {};
|
|
};
|
|
|
|
logger.log({key, bucketName}, "deleting directory");
|
|
const s3Client = getKnoxClient(bucketName);
|
|
return s3Client.list({prefix:key}, function(err, data){
|
|
if (err != null) {
|
|
logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
|
|
return callback(err);
|
|
}
|
|
const keys = _.map(data.Contents, entry => entry.Key);
|
|
return s3Client.deleteMultiple(keys, callback);
|
|
});
|
|
},
|
|
|
|
checkIfFileExists(bucketName, key, callback){
|
|
logger.log({bucketName, key}, "checking if file exists in s3");
|
|
const options = buildDefaultOptions(bucketName, "head", key);
|
|
return request(options, function(err, res){
|
|
if (err != null) {
|
|
logger.err({err, res, bucketName, key}, "something went wrong checking file in aws");
|
|
return callback(err);
|
|
}
|
|
if ((res == null)) {
|
|
logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists");
|
|
err = new Error(`no response from s3 ${bucketName} ${key}`);
|
|
return callback(err);
|
|
}
|
|
const exists = res.statusCode === 200;
|
|
logger.log({bucketName, key, exists}, "checked if file exsists in s3");
|
|
return callback(err, exists);
|
|
});
|
|
},
|
|
|
|
directorySize(bucketName, key, callback){
|
|
logger.log({bucketName, key}, "get project size in s3");
|
|
const s3Client = getKnoxClient(bucketName);
|
|
return s3Client.list({prefix:key}, function(err, data){
|
|
if (err != null) {
|
|
logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
|
|
return callback(err);
|
|
}
|
|
let totalSize = 0;
|
|
_.each(data.Contents, entry => totalSize += entry.Size);
|
|
logger.log({totalSize}, "total size");
|
|
return callback(null, totalSize);
|
|
});
|
|
}
|
|
};
|