/* eslint-disable handle-callback-err, new-cap, no-return-assign, no-unused-vars, node/no-deprecated-api, standard/no-callback-literal, */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from * DS102: Remove unnecessary code created because of implicit returns * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ // This module is the one which is used in production. It needs to be migrated // to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for // details. The knox library is unmaintained and has bugs. const http = require('http') http.globalAgent.maxSockets = 300 const https = require('https') https.globalAgent.maxSockets = 300 const settings = require('settings-sharelatex') const request = require('request') const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const fs = require('fs') const knox = require('knox') const path = require('path') const LocalFileWriter = require('./LocalFileWriter') const Errors = require('./Errors') const _ = require('underscore') const awsS3 = require('aws-sdk/clients/s3') const URL = require('url') const thirtySeconds = 30 * 1000 const buildDefaultOptions = function(bucketName, method, key) { let endpoint if (settings.filestore.s3.endpoint) { endpoint = `${settings.filestore.s3.endpoint}/${bucketName}` } else { endpoint = `https://${bucketName}.s3.amazonaws.com` } return { aws: { key: settings.filestore.s3.key, secret: settings.filestore.s3.secret, bucket: bucketName }, method, timeout: thirtySeconds, uri: `${endpoint}/${key}` } } const getS3Options = function(credentials) { const options = { credentials: { accessKeyId: credentials.auth_key, secretAccessKey: credentials.auth_secret } } if (settings.filestore.s3.endpoint) { const endpoint = URL.parse(settings.filestore.s3.endpoint) options.endpoint = settings.filestore.s3.endpoint options.sslEnabled = endpoint.protocol === 'https' } return options } const defaultS3Client = new awsS3( getS3Options({ auth_key: settings.filestore.s3.key, auth_secret: settings.filestore.s3.secret }) ) const getS3Client = function(credentials) { if (credentials != null) { return new awsS3(getS3Options(credentials)) } else { return defaultS3Client } } const getKnoxClient = bucketName => { const options = { key: settings.filestore.s3.key, secret: settings.filestore.s3.secret, bucket: bucketName } if (settings.filestore.s3.endpoint) { const endpoint = URL.parse(settings.filestore.s3.endpoint) options.endpoint = endpoint.hostname options.port = endpoint.port } return knox.createClient(options) } module.exports = { sendFile(bucketName, key, fsPath, callback) { const s3Client = getKnoxClient(bucketName) let uploaded = 0 const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) { metrics.count('s3.egress', uploaded) if (err != null) { logger.err( { err, bucketName, key, fsPath }, 'something went wrong uploading file to s3' ) return callback(err) } if (res == null) { logger.err( { err, res, bucketName, key, fsPath }, 'no response from s3 put file' ) return callback('no response from put file') } if (res.statusCode !== 200) { logger.err( { bucketName, key, fsPath }, 'non 200 response from s3 putting file' ) return callback('non 200 response from s3 on put file') } logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3') return callback(err) }) putEventEmiter.on('error', function(err) { logger.err( { err, bucketName, key, fsPath }, 'error emmited on put of file' ) return callback(err) }) return putEventEmiter.on( 'progress', progress => (uploaded = progress.written) ) }, sendStream(bucketName, key, readStream, callback) { logger.log({ bucketName, key }, 'sending file to s3') readStream.on('error', err => logger.err({ bucketName, key }, 'error on stream to send to s3') ) return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => { if (err != null) { logger.err( { bucketName, key, fsPath, err }, 'something went wrong writing stream to disk' ) return callback(err) } return this.sendFile(bucketName, key, fsPath, ( err // delete the temporary file created above and return the original error ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) }) }, // opts may be {start: Number, end: Number} getFileStream(bucketName, key, opts, callback) { if (callback == null) { callback = function(err, res) {} } opts = opts || {} callback = _.once(callback) logger.log({ bucketName, key }, 'getting file from s3') const s3 = getS3Client(opts.credentials) const s3Params = { Bucket: bucketName, Key: key } if (opts.start != null && opts.end != null) { s3Params.Range = `bytes=${opts.start}-${opts.end}` } const s3Request = s3.getObject(s3Params) s3Request.on( 'httpHeaders', (statusCode, headers, response, statusMessage) => { if ([403, 404].includes(statusCode)) { // S3 returns a 403 instead of a 404 when the user doesn't have // permission to list the bucket contents. logger.log({ bucketName, key }, 'file not found in s3') return callback( new Errors.NotFoundError( `File not found in S3: ${bucketName}:${key}` ), null ) } if (![200, 206].includes(statusCode)) { logger.log( { bucketName, key }, `error getting file from s3: ${statusCode}` ) return callback( new Error( `Got non-200 response from S3: ${statusCode} ${statusMessage}` ), null ) } const stream = response.httpResponse.createUnbufferedStream() stream.on('data', data => metrics.count('s3.ingress', data.byteLength)) return callback(null, stream) } ) s3Request.on('error', err => { logger.err({ err, bucketName, key }, 'error getting file stream from s3') return callback(err) }) return s3Request.send() }, getFileSize(bucketName, key, callback) { logger.log({ bucketName, key }, 'getting file size from S3') const s3 = getS3Client() return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { if (err != null) { if ([403, 404].includes(err.statusCode)) { // S3 returns a 403 instead of a 404 when the user doesn't have // permission to list the bucket contents. logger.log( { bucketName, key }, 'file not found in s3' ) callback( new Errors.NotFoundError( `File not found in S3: ${bucketName}:${key}` ) ) } else { logger.err( { bucketName, key, err }, 'error performing S3 HeadObject' ) callback(err) } return } return callback(null, data.ContentLength) }) }, copyFile(bucketName, sourceKey, destKey, callback) { logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') const source = bucketName + '/' + sourceKey // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) const s3 = getS3Client() return s3.copyObject( { Bucket: bucketName, Key: destKey, CopySource: source }, function(err) { if (err != null) { if (err.code === 'NoSuchKey') { logger.err( { bucketName, sourceKey }, 'original file not found in s3 when copying' ) return callback( new Errors.NotFoundError( 'original file not found in S3 when copying' ) ) } else { logger.err( { err, bucketName, sourceKey, destKey }, 'something went wrong copying file in aws' ) return callback(err) } } else { return callback() } } ) }, deleteFile(bucketName, key, callback) { logger.log({ bucketName, key }, 'delete file in s3') const options = buildDefaultOptions(bucketName, 'delete', key) return request(options, function(err, res) { if (err != null) { logger.err( { err, res, bucketName, key }, 'something went wrong deleting file in aws' ) } return callback(err) }) }, deleteDirectory(bucketName, key, _callback) { // deleteMultiple can call the callback multiple times so protect against this. const callback = function(...args) { _callback(...Array.from(args || [])) return (_callback = function() {}) } logger.log({ key, bucketName }, 'deleting directory') const s3Client = getKnoxClient(bucketName) return s3Client.list({ prefix: key }, function(err, data) { if (err != null) { logger.err( { err, bucketName, key }, 'something went wrong listing prefix in aws' ) return callback(err) } const keys = _.map(data.Contents, entry => entry.Key) return s3Client.deleteMultiple(keys, callback) }) }, checkIfFileExists(bucketName, key, callback) { logger.log({ bucketName, key }, 'checking if file exists in s3') const options = buildDefaultOptions(bucketName, 'head', key) return request(options, function(err, res) { if (err != null) { logger.err( { err, res, bucketName, key }, 'something went wrong checking file in aws' ) return callback(err) } if (res == null) { logger.err( { err, res, bucketName, key }, 'no response object returned when checking if file exists' ) err = new Error(`no response from s3 ${bucketName} ${key}`) return callback(err) } const exists = res.statusCode === 200 logger.log({ bucketName, key, exists }, 'checked if file exsists in s3') return callback(err, exists) }) }, directorySize(bucketName, key, callback) { logger.log({ bucketName, key }, 'get project size in s3') const s3Client = getKnoxClient(bucketName) return s3Client.list({ prefix: key }, function(err, data) { if (err != null) { logger.err( { err, bucketName, key }, 'something went wrong listing prefix in aws' ) return callback(err) } let totalSize = 0 _.each(data.Contents, entry => (totalSize += entry.Size)) logger.log({ totalSize }, 'total size') return callback(null, totalSize) }) } }