diff --git a/services/filestore/app/js/AWSSDKPersistorManager.js b/services/filestore/app/js/AWSSDKPersistorManager.js deleted file mode 100644 index 4dbc836280..0000000000 --- a/services/filestore/app/js/AWSSDKPersistorManager.js +++ /dev/null @@ -1,197 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This module is not used in production, which currently uses -// S3PersistorManager. The intention is to migrate S3PersistorManager to use the -// latest aws-sdk and delete this module so that PersistorManager would load the -// same backend for both the 's3' and 'aws-sdk' options. - -const logger = require('logger-sharelatex') -const aws = require('aws-sdk') -const _ = require('underscore') -const fs = require('fs') -const Errors = require('./Errors') - -const s3 = new aws.S3() - -module.exports = { - sendFile(bucketName, key, fsPath, callback) { - logger.log({ bucketName, key }, 'send file data to s3') - const stream = fs.createReadStream(fsPath) - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file data to s3' - ) - } - return callback(err) - }) - }, - - sendStream(bucketName, key, stream, callback) { - logger.log({ bucketName, key }, 'send file stream to s3') - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file stream to s3' - ) - } - return callback(err) - }) - }, - - getFileStream(bucketName, key, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - logger.log({ bucketName, key }, 'get file stream from s3') - callback = _.once(callback) - const params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - params.Range = `bytes=${opts.start}-${opts.end}` - } - const request = s3.getObject(params) - const stream = request.createReadStream() - stream.on('readable', () => callback(null, stream)) - return stream.on('error', function(err) { - logger.err({ err, bucketName, key }, 'error getting file stream from s3') - if (err.code === 'NoSuchKey') { - return callback( - new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) - ) - } - return callback(err) - }) - }, - - copyFile(bucketName, sourceKey, destKey, callback) { - logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') - const source = bucketName + '/' + sourceKey - return s3.copyObject( - { Bucket: bucketName, Key: destKey, CopySource: source }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, sourceKey, destKey }, - 'something went wrong copying file in s3' - ) - } - return callback(err) - } - ) - }, - - deleteFile(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete file in s3') - return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong deleting file in s3' - ) - } - return callback(err) - }) - }, - - deleteDirectory(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete directory in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - const keys = _.map(data.Contents, entry => ({ - Key: entry.Key - })) - return s3.deleteObjects( - { - Bucket: bucketName, - Delete: { - Objects: keys, - Quiet: true - } - }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, key: keys }, - 'something went wrong deleting directory in s3' - ) - } - return callback(err) - } - ) - }) - }, - - checkIfFileExists(bucketName, key, callback) { - logger.log({ bucketName, key }, 'check file existence in s3') - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if (err.code === 'NotFound') { - return callback(null, false) - } - logger.err( - { err, bucketName, key }, - 'something went wrong checking head in s3' - ) - return callback(err) - } - return callback(null, data.ETag != null) - }) - }, - - directorySize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'get project size in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - let totalSize = 0 - _.each(data.Contents, entry => (totalSize += entry.Size)) - return callback(null, totalSize) - }) - } -} diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index dbba9a93cc..eef441436c 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -17,7 +17,7 @@ module.exports = { directorySize } -function getFile(req, res) { +function getFile(req, res, next) { const { key, bucket } = req const { format, style } = req.query const options = { @@ -61,7 +61,9 @@ function getFile(req, res) { } logger.log({ key, bucket, format, style }, 'sending file to response') - pipeline(fileStream, res) + + // pass 'next' as a callback to 'pipeline' to receive any errors + pipeline(fileStream, res, next) }) } diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index 8124d66101..cca0cf0f36 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -13,8 +13,6 @@ if (!settings.filestore.backend) { switch (settings.filestore.backend) { case 'aws-sdk': - module.exports = require('./AWSSDKPersistorManager') - break case 's3': module.exports = require('./S3PersistorManager') break diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index cadf38172a..d0729b80b9 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -1,376 +1,258 @@ -/* eslint-disable - handle-callback-err, - new-cap, - no-return-assign, - no-unused-vars, - node/no-deprecated-api, - standard/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This module is the one which is used in production. It needs to be migrated -// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for -// details. The knox library is unmaintained and has bugs. - const http = require('http') -http.globalAgent.maxSockets = 300 const https = require('https') +http.globalAgent.maxSockets = 300 https.globalAgent.maxSockets = 300 + const settings = require('settings-sharelatex') -const request = require('request') const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') + +const meter = require('stream-meter') const fs = require('fs') -const knox = require('knox') -const path = require('path') -const LocalFileWriter = require('./LocalFileWriter') -const Errors = require('./Errors') -const _ = require('underscore') -const awsS3 = require('aws-sdk/clients/s3') -const URL = require('url') +const S3 = require('aws-sdk/clients/s3') +const { URL } = require('url') +const { callbackify } = require('util') +const { WriteError, ReadError, NotFoundError } = require('./Errors') -const thirtySeconds = 30 * 1000 - -const buildDefaultOptions = function(bucketName, method, key) { - let endpoint - if (settings.filestore.s3.endpoint) { - endpoint = `${settings.filestore.s3.endpoint}/${bucketName}` - } else { - endpoint = `https://${bucketName}.s3.amazonaws.com` - } - return { - aws: { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - }, - method, - timeout: thirtySeconds, - uri: `${endpoint}/${key}` +module.exports = { + sendFile: callbackify(sendFile), + sendStream: callbackify(sendStream), + getFileStream: callbackify(getFileStream), + deleteDirectory: callbackify(deleteDirectory), + getFileSize: callbackify(getFileSize), + deleteFile: callbackify(deleteFile), + copyFile: callbackify(copyFile), + checkIfFileExists: callbackify(checkIfFileExists), + directorySize: callbackify(directorySize), + promises: { + sendFile, + sendStream, + getFileStream, + deleteDirectory, + getFileSize, + deleteFile, + copyFile, + checkIfFileExists, + directorySize } } -const getS3Options = function(credentials) { +const _client = new S3(_defaultOptions()) + +async function sendFile(bucketName, key, fsPath) { + let readStream + try { + readStream = fs.createReadStream(fsPath) + } catch (err) { + throw _wrapError( + err, + 'error reading file from disk', + { bucketName, key, fsPath }, + ReadError + ) + } + return sendStream(bucketName, key, readStream) +} + +async function sendStream(bucketName, key, readStream) { + try { + const meteredStream = meter() + meteredStream.on('finish', () => { + metrics.count('s3.egress', meteredStream.bytes) + }) + + const response = await _client + .upload({ + Bucket: bucketName, + Key: key, + Body: readStream.pipe(meteredStream) + }) + .promise() + + logger.log({ response, bucketName, key }, 'data uploaded to s3') + } catch (err) { + throw _wrapError( + err, + 'upload to S3 failed', + { bucketName, key }, + WriteError + ) + } +} + +async function getFileStream(bucketName, key, opts) { + opts = opts || {} + + const params = { + Bucket: bucketName, + Key: key + } + if (opts.start != null && opts.end != null) { + params.Range = `bytes=${opts.start}-${opts.end}` + } + + return new Promise((resolve, reject) => { + const stream = _client.getObject(params).createReadStream() + + const meteredStream = meter() + meteredStream.on('finish', () => { + metrics.count('s3.ingress', meteredStream.bytes) + }) + + const onStreamReady = function() { + stream.removeListener('readable', onStreamReady) + resolve(stream.pipe(meteredStream)) + } + stream.on('readable', onStreamReady) + stream.on('error', err => { + reject(_wrapError(err, 'error reading from S3', params, ReadError)) + }) + }) +} + +async function deleteDirectory(bucketName, key) { + logger.log({ key, bucketName }, 'deleting directory') + let response + + try { + response = await _client + .listObjects({ Bucket: bucketName, Prefix: key }) + .promise() + } catch (err) { + throw _wrapError( + err, + 'failed to list objects in S3', + { bucketName, key }, + ReadError + ) + } + + const objects = response.Contents.map(item => ({ Key: item.Key })) + if (objects.length) { + try { + await _client + .deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true + } + }) + .promise() + } catch (err) { + throw _wrapError( + err, + 'failed to delete objects in S3', + { bucketName, key }, + WriteError + ) + } + } +} + +async function getFileSize(bucketName, key) { + try { + const response = await _client + .headObject({ Bucket: bucketName, Key: key }) + .promise() + return response.ContentLength + } catch (err) { + throw _wrapError( + err, + 'error getting size of s3 object', + { bucketName, key }, + ReadError + ) + } +} + +async function deleteFile(bucketName, key) { + try { + await _client.deleteObject({ Bucket: bucketName, Key: key }).promise() + } catch (err) { + throw _wrapError( + err, + 'failed to delete file in S3', + { bucketName, key }, + WriteError + ) + } +} + +async function copyFile(bucketName, sourceKey, destKey) { + const params = { + Bucket: bucketName, + Key: destKey, + CopySource: `${bucketName}/${sourceKey}` + } + try { + await _client.copyObject(params).promise() + } catch (err) { + throw _wrapError(err, 'failed to copy file in S3', params, WriteError) + } +} + +async function checkIfFileExists(bucketName, key) { + try { + await getFileSize(bucketName, key) + return true + } catch (err) { + if (err instanceof NotFoundError) { + return false + } + throw _wrapError( + err, + 'error checking whether S3 object exists', + { bucketName, key }, + ReadError + ) + } +} + +async function directorySize(bucketName, key) { + try { + const response = await _client + .listObjects({ Bucket: bucketName, Prefix: key }) + .promise() + + return response.Contents.reduce((acc, item) => item.Size + acc, 0) + } catch (err) { + throw _wrapError( + err, + 'error getting directory size in S3', + { bucketName, key }, + ReadError + ) + } +} + +function _wrapError(error, message, params, ErrorType) { + if (['NoSuchKey', 'NotFound', 'ENOENT'].includes(error.code)) { + return new NotFoundError({ + message: 'no such file', + info: params + }).withCause(error) + } else { + return new ErrorType({ + message: message, + info: params + }).withCause(error) + } +} + +function _defaultOptions() { const options = { credentials: { - accessKeyId: credentials.auth_key, - secretAccessKey: credentials.auth_secret + accessKeyId: settings.filestore.s3.key, + secretAccessKey: settings.filestore.s3.secret } } if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint) + const endpoint = new URL(settings.filestore.s3.endpoint) options.endpoint = settings.filestore.s3.endpoint options.sslEnabled = endpoint.protocol === 'https' } return options } - -const defaultS3Client = new awsS3( - getS3Options({ - auth_key: settings.filestore.s3.key, - auth_secret: settings.filestore.s3.secret - }) -) - -const getS3Client = function(credentials) { - if (credentials != null) { - return new awsS3(getS3Options(credentials)) - } else { - return defaultS3Client - } -} - -const getKnoxClient = bucketName => { - const options = { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - } - if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint) - options.endpoint = endpoint.hostname - options.port = endpoint.port - } - return knox.createClient(options) -} - -module.exports = { - sendFile(bucketName, key, fsPath, callback) { - const s3Client = getKnoxClient(bucketName) - let uploaded = 0 - const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) { - metrics.count('s3.egress', uploaded) - if (err != null) { - logger.err( - { err, bucketName, key, fsPath }, - 'something went wrong uploading file to s3' - ) - return callback(err) - } - if (res == null) { - logger.err( - { err, res, bucketName, key, fsPath }, - 'no response from s3 put file' - ) - return callback('no response from put file') - } - if (res.statusCode !== 200) { - logger.err( - { bucketName, key, fsPath }, - 'non 200 response from s3 putting file' - ) - return callback('non 200 response from s3 on put file') - } - logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3') - return callback(err) - }) - putEventEmiter.on('error', function(err) { - logger.err( - { err, bucketName, key, fsPath }, - 'error emmited on put of file' - ) - return callback(err) - }) - return putEventEmiter.on( - 'progress', - progress => (uploaded = progress.written) - ) - }, - - sendStream(bucketName, key, readStream, callback) { - logger.log({ bucketName, key }, 'sending file to s3') - readStream.on('error', err => - logger.err({ bucketName, key }, 'error on stream to send to s3') - ) - return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => { - if (err != null) { - logger.err( - { bucketName, key, fsPath, err }, - 'something went wrong writing stream to disk' - ) - return callback(err) - } - return this.sendFile(bucketName, key, fsPath, ( - err // delete the temporary file created above and return the original error - ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) - }) - }, - - // opts may be {start: Number, end: Number} - getFileStream(bucketName, key, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - opts = opts || {} - callback = _.once(callback) - logger.log({ bucketName, key }, 'getting file from s3') - - const s3 = getS3Client(opts.credentials) - const s3Params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - s3Params.Range = `bytes=${opts.start}-${opts.end}` - } - const s3Request = s3.getObject(s3Params) - - s3Request.on( - 'httpHeaders', - (statusCode, headers, response, statusMessage) => { - if ([403, 404].includes(statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log({ bucketName, key }, 'file not found in s3') - return callback( - new Errors.NotFoundError( - `File not found in S3: ${bucketName}:${key}` - ), - null - ) - } - if (![200, 206].includes(statusCode)) { - logger.log( - { bucketName, key }, - `error getting file from s3: ${statusCode}` - ) - return callback( - new Error( - `Got non-200 response from S3: ${statusCode} ${statusMessage}` - ), - null - ) - } - const stream = response.httpResponse.createUnbufferedStream() - stream.on('data', data => metrics.count('s3.ingress', data.byteLength)) - - return callback(null, stream) - } - ) - - s3Request.on('error', err => { - logger.err({ err, bucketName, key }, 'error getting file stream from s3') - return callback(err) - }) - - return s3Request.send() - }, - - getFileSize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'getting file size from S3') - const s3 = getS3Client() - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if ([403, 404].includes(err.statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log( - { - bucketName, - key - }, - 'file not found in s3' - ) - callback( - new Errors.NotFoundError( - `File not found in S3: ${bucketName}:${key}` - ) - ) - } else { - logger.err( - { - bucketName, - key, - err - }, - 'error performing S3 HeadObject' - ) - callback(err) - } - return - } - return callback(null, data.ContentLength) - }) - }, - - copyFile(bucketName, sourceKey, destKey, callback) { - logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') - const source = bucketName + '/' + sourceKey - // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) - const s3 = getS3Client() - return s3.copyObject( - { Bucket: bucketName, Key: destKey, CopySource: source }, - function(err) { - if (err != null) { - if (err.code === 'NoSuchKey') { - logger.err( - { bucketName, sourceKey }, - 'original file not found in s3 when copying' - ) - return callback( - new Errors.NotFoundError( - 'original file not found in S3 when copying' - ) - ) - } else { - logger.err( - { err, bucketName, sourceKey, destKey }, - 'something went wrong copying file in aws' - ) - return callback(err) - } - } else { - return callback() - } - } - ) - }, - - deleteFile(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete file in s3') - const options = buildDefaultOptions(bucketName, 'delete', key) - return request(options, function(err, res) { - if (err != null) { - logger.err( - { err, res, bucketName, key }, - 'something went wrong deleting file in aws' - ) - } - return callback(err) - }) - }, - - deleteDirectory(bucketName, key, _callback) { - // deleteMultiple can call the callback multiple times so protect against this. - const callback = function(...args) { - _callback(...Array.from(args || [])) - return (_callback = function() {}) - } - - logger.log({ key, bucketName }, 'deleting directory') - const s3Client = getKnoxClient(bucketName) - return s3Client.list({ prefix: key }, function(err, data) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in aws' - ) - return callback(err) - } - const keys = _.map(data.Contents, entry => entry.Key) - return s3Client.deleteMultiple(keys, callback) - }) - }, - - checkIfFileExists(bucketName, key, callback) { - logger.log({ bucketName, key }, 'checking if file exists in s3') - const options = buildDefaultOptions(bucketName, 'head', key) - return request(options, function(err, res) { - if (err != null) { - logger.err( - { err, res, bucketName, key }, - 'something went wrong checking file in aws' - ) - return callback(err) - } - if (res == null) { - logger.err( - { err, res, bucketName, key }, - 'no response object returned when checking if file exists' - ) - err = new Error(`no response from s3 ${bucketName} ${key}`) - return callback(err) - } - const exists = res.statusCode === 200 - logger.log({ bucketName, key, exists }, 'checked if file exsists in s3') - return callback(err, exists) - }) - }, - - directorySize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'get project size in s3') - const s3Client = getKnoxClient(bucketName) - return s3Client.list({ prefix: key }, function(err, data) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in aws' - ) - return callback(err) - } - let totalSize = 0 - _.each(data.Contents, entry => (totalSize += entry.Size)) - logger.log({ totalSize }, 'total size') - return callback(null, totalSize) - }) - } -} diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 3ed1400a61..8d78271caa 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -5018,6 +5018,38 @@ "resolved": "https://registry.npmjs.org/stream-counter/-/stream-counter-1.0.0.tgz", "integrity": "sha1-kc8lac5NxQYf6816yyY5SloRR1E=" }, + "stream-meter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", + "integrity": "sha1-Uq+Vql6nYKJJFxZwTb/5D3Ov3R0=", + "requires": { + "readable-stream": "^2.1.4" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "stream-shift": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", @@ -5531,7 +5563,7 @@ "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" diff --git a/services/filestore/package.json b/services/filestore/package.json index 5d7c3e3ec1..14e35cd8a2 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -44,6 +44,7 @@ "settings-sharelatex": "^1.1.0", "stream-browserify": "^2.0.1", "stream-buffers": "~0.2.5", + "stream-meter": "^1.0.4", "underscore": "~1.5.2" }, "devDependencies": { diff --git a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js deleted file mode 100644 index ea88da71c3..0000000000 --- a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js +++ /dev/null @@ -1,509 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-dupe-keys, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const sinon = require('sinon') -const chai = require('chai') - -const should = chai.should() -const { expect } = chai - -const modulePath = '../../../app/js/AWSSDKPersistorManager.js' -const SandboxedModule = require('sandboxed-module') - -describe('AWSSDKPersistorManager', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 'aws-sdk' - } - } - this.s3 = { - upload: sinon.stub(), - getObject: sinon.stub(), - copyObject: sinon.stub(), - deleteObject: sinon.stub(), - listObjects: sinon.stub(), - deleteObjects: sinon.stub(), - headObject: sinon.stub() - } - this.awssdk = { S3: sinon.stub().returns(this.s3) } - - this.requires = { - 'aws-sdk': this.awssdk, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - }, - fs: (this.fs = { createReadStream: sinon.stub() }), - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) - } - this.key = 'my/key' - this.bucketName = 'my-bucket' - this.error = 'my error' - return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) - }) - - describe('sendFile', function() { - beforeEach(function() { - this.stream = {} - this.fsPath = '/usr/local/some/file' - return this.fs.createReadStream.returns(this.stream) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('sendStream', function() { - beforeEach(function() { - return (this.stream = {}) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('getFileStream', function() { - beforeEach(function() { - this.opts = {} - this.stream = {} - this.read_stream = { on: (this.read_stream_on = sinon.stub()) } - this.object = { createReadStream: sinon.stub().returns(this.read_stream) } - return this.s3.getObject.returns(this.object) - }) - - it('should return a stream from s3.getObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(this.read_stream_on.calledTwice) - expect(err).to.not.be.ok - expect(stream, 'returned the stream').to.equal(this.read_stream) - expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - describe('with start and end options', function() { - beforeEach(function() { - return (this.opts = { - start: 0, - end: 8 - }) - }) - return it('should pass headers to the s3.GetObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - return expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Range: 'bytes=0-8' - }), - 'called with correct arguments' - ).to.be.true - } - ) - return done() - }) - }) - - return describe('error conditions', function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - this.error = new Error() - return (this.error.code = 'NoSuchKey') - }) - return it('should produce a NotFoundError', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect( - err instanceof this.Errors.NotFoundError, - 'error is a correct instance' - ).to.equal(true) - return done() - } - ) - }) - }) - - return describe('when there is some other error', function() { - beforeEach(function() { - return (this.error = new Error()) - }) - return it('should dispatch the error from s3 object stream', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - }) - - describe('copyFile', function() { - beforeEach(function() { - this.destKey = 'some/key' - return (this.stream = {}) - }) - - it('should copy the file with s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.not.be.ok - expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.copyObject.calledWith({ - Bucket: this.bucketName, - Key: this.destKey, - CopySource: this.bucketName + '/' + this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteFile', function() { - it('should delete the file with s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.deleteObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteDirectory', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return describe('with directory content', function() { - beforeEach(function() { - return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }]) - }) - - it('should forward the file keys to s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be - .true - expect( - this.s3.deleteObjects.calledWith({ - Bucket: this.bucketName, - Delete: { - Quiet: true, - Objects: this.fileList - } - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - - describe('checkIfFileExists', function() { - it('should check for the file with s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.not.be.ok - expect(this.s3.headObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.headObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should return false on an inexistant file', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.false - return done() - } - ) - }) - - it('should return true on an existing file', function(done) { - this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' }) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - return describe('directorySize', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return it('should sum directory files sizes', function(done) { - this.s3.listObjects.callsArgWith(1, null, { - Contents: [{ Size: 1024 }, { Size: 2048 }] - }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - (err, size) => { - expect(size).to.equal(3072) - return done() - } - ) - }) - }) -}) diff --git a/services/filestore/test/unit/js/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js index d8fd887265..0ecbb22078 100644 --- a/services/filestore/test/unit/js/PersistorManagerTests.js +++ b/services/filestore/test/unit/js/PersistorManagerTests.js @@ -43,6 +43,14 @@ describe('PersistorManager', function() { expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager') }) + it("should implement the S3 wrapped method when 'aws-sdk' is configured", function() { + settings.filestore.backend = 'aws-sdk' + PersistorManager = SandboxedModule.require(modulePath, { requires }) + + expect(PersistorManager).to.respondTo('wrappedMethod') + expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager') + }) + it('should implement the FS wrapped method when FS is configured', function() { settings.filestore.backend = 'fs' PersistorManager = SandboxedModule.require(modulePath, { requires }) diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index 777c9c5a32..2faa8b52c7 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -1,618 +1,718 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/S3PersistorManager.js' const SandboxedModule = require('sandboxed-module') +const Errors = require('../../../app/js/Errors') + describe('S3PersistorManagerTests', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 's3', - s3: { - secret: 'secret', - key: 'this_key' - }, - stores: { - user_files: 'sl_user_files' - } - } - } - this.knoxClient = { - putFile: sinon.stub(), - copyFile: sinon.stub(), - list: sinon.stub(), - deleteMultiple: sinon.stub(), - get: sinon.stub() - } - this.knox = { createClient: sinon.stub().returns(this.knoxClient) } - this.s3EventHandlers = {} - this.s3Request = { - on: sinon.stub().callsFake((event, callback) => { - return (this.s3EventHandlers[event] = callback) - }), - send: sinon.stub() - } - this.s3Response = { - httpResponse: { - createUnbufferedStream: sinon.stub() - } - } - this.s3Client = { - copyObject: sinon.stub(), - headObject: sinon.stub(), - getObject: sinon.stub().returns(this.s3Request) - } - this.awsS3 = sinon.stub().returns(this.s3Client) - this.LocalFileWriter = { - writeStream: sinon.stub(), - deleteFile: sinon.stub() - } - this.request = sinon.stub() - this.requires = { - knox: this.knox, - 'aws-sdk/clients/s3': this.awsS3, - 'settings-sharelatex': this.settings, - './LocalFileWriter': this.LocalFileWriter, - 'logger-sharelatex': { - log() {}, - err() {} + const settings = { + filestore: { + backend: 's3', + s3: { + secret: 'secret', + key: 'this_key' }, - request: this.request, - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) + stores: { + user_files: 'sl_user_files' + } } - this.key = 'my/key' - this.bucketName = 'my-bucket' - this.error = 'my errror' - return (this.S3PersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) + } + const filename = '/wombat/potato.tex' + const bucket = 'womBucket' + const key = 'monKey' + const destKey = 'donKey' + const objectSize = 5555 + const genericError = new Error('guru meditation error') + const files = [ + { Key: 'llama', Size: 11 }, + { Key: 'hippo', Size: 22 } + ] + const filesSize = 33 + + let Metrics, + S3, + Fs, + Meter, + MeteredStream, + ReadStream, + S3PersistorManager, + S3Client, + S3ReadStream, + S3NotFoundError, + FileNotFoundError, + EmptyPromise + + beforeEach(function() { + EmptyPromise = { + promise: sinon.stub().resolves() + } + + Metrics = { + count: sinon.stub() + } + + ReadStream = { + pipe: sinon.stub().returns('readStream') + } + + FileNotFoundError = new Error('File not found') + FileNotFoundError.code = 'ENOENT' + + Fs = { + createReadStream: sinon.stub().returns(ReadStream) + } + + MeteredStream = { + on: sinon.stub(), + bytes: objectSize + } + MeteredStream.on.withArgs('finish').yields() + Meter = sinon.stub().returns(MeteredStream) + + S3NotFoundError = new Error('not found') + S3NotFoundError.code = 'NoSuchKey' + + S3ReadStream = { + on: sinon.stub(), + pipe: sinon.stub().returns('s3Stream'), + removeListener: sinon.stub() + } + S3ReadStream.on.withArgs('readable').yields() + S3Client = { + getObject: sinon.stub().returns({ + createReadStream: sinon.stub().returns(S3ReadStream) + }), + headObject: sinon.stub().returns({ + promise: sinon.stub().resolves({ + ContentLength: objectSize + }) + }), + listObjects: sinon.stub().returns({ + promise: sinon.stub().resolves({ + Contents: files + }) + }), + upload: sinon.stub().returns(EmptyPromise), + copyObject: sinon.stub().returns(EmptyPromise), + deleteObject: sinon.stub().returns(EmptyPromise), + deleteObjects: sinon.stub().returns(EmptyPromise) + } + S3 = sinon.stub().returns(S3Client) + + S3PersistorManager = SandboxedModule.require(modulePath, { + requires: { + 'aws-sdk/clients/s3': S3, + 'settings-sharelatex': settings, + './Errors': Errors, + fs: Fs, + 'stream-meter': Meter, + 'logger-sharelatex': { + log() {}, + err() {} + }, + 'metrics-sharelatex': Metrics + }, + globals: { console } + }) }) describe('getFileStream', function() { - describe('success', function() { - beforeEach(function() { - this.expectedStream = { expectedStream: true } - this.expectedStream.on = sinon.stub() - this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 200, - {}, - this.s3Response, - 'OK' - ) - }) - return this.s3Response.httpResponse.createUnbufferedStream.returns( - this.expectedStream - ) + describe('when called with valid parameters', function() { + let stream + + beforeEach(async function() { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) }) - it('returns a stream', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + it('returns a stream', function() { + expect(stream).to.equal('s3Stream') }) - it('sets the AWS client up with credentials from settings', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.awsS3.lastCall.args).to.deep.equal([ - { - credentials: { - accessKeyId: this.settings.filestore.s3.key, - secretAccessKey: this.settings.filestore.s3.secret - } - } - ]) - return done() - } - ) - }) - - it('fetches the right key from the right bucket', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ - { - Bucket: this.bucketName, - Key: this.key - } - ]) - return done() - } - ) - }) - - it('accepts alternative credentials', function(done) { - const accessKeyId = 'that_key' - const secret = 'that_secret' - const opts = { + it('sets the AWS client up with credentials from settings', function() { + expect(S3).to.have.been.calledWith({ credentials: { - auth_key: accessKeyId, - auth_secret: secret + accessKeyId: settings.filestore.s3.key, + secretAccessKey: settings.filestore.s3.secret } - } - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - opts, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.awsS3.lastCall.args).to.deep.equal([ - { - credentials: { - accessKeyId, - secretAccessKey: secret - } - } - ]) - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + }) }) - return it('accepts byte range', function(done) { - const start = 0 - const end = 8 - const opts = { start, end } - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - opts, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ - { - Bucket: this.bucketName, - Key: this.key, - Range: `bytes=${start}-${end}` - } - ]) - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + it('fetches the right key from the right bucket', function() { + expect(S3Client.getObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('pipes the stream through the meter', function() { + expect(S3ReadStream.pipe).to.have.been.calledWith(MeteredStream) + }) + + it('records an ingress metric', function() { + expect(Metrics.count).to.have.been.calledWith('s3.ingress', objectSize) }) }) - return describe('errors', function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 404, - {}, - this.s3Response, - 'Not found' - ) - }) - }) + describe('when called with a byte range', function() { + let stream - return it('returns a NotFoundError that indicates the bucket and key', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(this.Errors.NotFoundError) - const errMsg = this.Errors.NotFoundError.lastCall.args[0] - expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`)) - expect(errMsg).to.match(new RegExp(`.*${this.key}.*`)) - return done() - } - ) + beforeEach(async function() { + stream = await S3PersistorManager.promises.getFileStream(bucket, key, { + start: 5, + end: 10 }) }) - describe('when S3 encounters an unkown error', function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 500, - {}, - this.s3Response, - 'Internal server error' - ) - }) - }) - - return it('returns an error', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(Error) - return done() - } - ) - }) + it('returns a stream', function() { + expect(stream).to.equal('s3Stream') }) - return describe('when the S3 request errors out before receiving HTTP headers', function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.error(new Error('connection failed')) - }) + it('passes the byte range on to S3', function() { + expect(S3Client.getObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Range: 'bytes=5-10' }) + }) + }) - return it('returns an error', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(Error) - return done() - } - ) - }) + describe("when the file doesn't exist", function() { + let error, stream + + beforeEach(async function() { + S3ReadStream.on = sinon.stub() + S3ReadStream.on.withArgs('error').yields(S3NotFoundError) + try { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + } catch (err) { + error = err + } + }) + + it('does not return a stream', function() { + expect(stream).not.to.exist + }) + + it('throws a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('wraps the error from S3', function() { + expect(error.cause).to.equal(S3NotFoundError) + }) + + it('stores the bucket and key in the error', function() { + expect(error.info).to.deep.equal({ Bucket: bucket, Key: key }) + }) + }) + + describe('when S3 encounters an unkown error', function() { + let error, stream + + beforeEach(async function() { + S3ReadStream.on = sinon.stub() + S3ReadStream.on.withArgs('error').yields(genericError) + try { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + } catch (err) { + error = err + } + }) + + it('does not return a stream', function() { + expect(stream).not.to.exist + }) + + it('throws a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('wraps the error from S3', function() { + expect(error.cause).to.equal(genericError) + }) + + it('stores the bucket and key in the error', function() { + expect(error.info).to.deep.equal({ Bucket: bucket, Key: key }) }) }) }) describe('getFileSize', function() { - it('should obtain the file size from S3', function(done) { - const expectedFileSize = 123 - this.s3Client.headObject.yields( - new Error('s3Client.headObject got unexpected arguments') - ) - this.s3Client.headObject - .withArgs({ - Bucket: this.bucketName, - Key: this.key - }) - .yields(null, { ContentLength: expectedFileSize }) + describe('when called with valid parameters', function() { + let size - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - if (err != null) { - return done(err) - } - expect(fileSize).to.equal(expectedFileSize) - return done() - } - ) - }) - ;[403, 404].forEach(statusCode => - it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { - const error = new Error() - error.statusCode = statusCode - this.s3Client.headObject.yields(error) - - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - expect(err).to.be.an.instanceof(this.Errors.NotFoundError) - return done() - } - ) + beforeEach(async function() { + size = await S3PersistorManager.promises.getFileSize(bucket, key) }) - ) - return it('should rethrow any other error', function(done) { - const error = new Error() - this.s3Client.headObject.yields(error) - this.s3Client.headObject.yields(error) + it('should return the object size', function() { + expect(size).to.equal(objectSize) + }) - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - expect(err).to.equal(error) - return done() - } - ) - }) - }) - - describe('sendFile', function() { - beforeEach(function() { - return this.knoxClient.putFile.returns({ on() {} }) + it('should pass the bucket and key to S3', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) }) - it('should put file with knox', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.knoxClient.putFile.callsArgWith(2, this.error) - return this.S3PersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - this.knoxClient.putFile - .calledWith(this.fsPath, this.key) - .should.equal(true) - err.should.equal(this.error) - return done() + describe('when the object is not found', function() { + let error + + beforeEach(async function() { + S3Client.headObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.getFileSize(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should return a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(S3NotFoundError) + }) }) - return it('should delete the file and pass the error with it', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.knoxClient.putFile.callsArgWith(2, this.error) - return this.S3PersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - this.knoxClient.putFile - .calledWith(this.fsPath, this.key) - .should.equal(true) - err.should.equal(this.error) - return done() + describe('when S3 returns an error', function() { + let error + + beforeEach(async function() { + S3Client.headObject = sinon.stub().returns({ + promise: sinon.stub().rejects(genericError) + }) + try { + await S3PersistorManager.promises.getFileSize(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should return a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('sendStream', function() { - beforeEach(function() { - this.fsPath = 'to/some/where' - this.origin = { on() {} } - return (this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)) + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.sendStream(bucket, key, ReadStream) + }) + + it('should upload the stream', function() { + expect(S3Client.upload).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Body: 'readStream' + }) + }) + + it('should meter the stream', function() { + expect(ReadStream.pipe).to.have.been.calledWith(MeteredStream) + }) + + it('should record an egress metric', function() { + expect(Metrics.count).to.have.been.calledWith('s3.egress', objectSize) + }) }) - it('should send stream to LocalFileWriter', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - () => { - this.LocalFileWriter.writeStream - .calledWith(this.origin) - .should.equal(true) - return done() + describe('when the upload fails', function() { + let error + beforeEach(async function() { + S3Client.upload = sinon.stub().returns({ + promise: sinon.stub().rejects(genericError) + }) + try { + await S3PersistorManager.promises.sendStream(bucket, key, ReadStream) + } catch (err) { + error = err } - ) + }) + + it('throws a WriteError', function() { + expect(error).to.be.an.instanceOf(Errors.WriteError) + }) + }) + }) + + describe('sendFile', function() { + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.sendFile(bucket, key, filename) + }) + + it('should create a read stream for the file', function() { + expect(Fs.createReadStream).to.have.been.calledWith(filename) + }) + + it('should upload the stream', function() { + expect(S3Client.upload).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Body: 'readStream' + }) + }) }) - it('should return the error from LocalFileWriter', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2, this.error) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - err => { - err.should.equal(this.error) - return done() + describe('when the file does not exist', function() { + let error + + beforeEach(async function() { + Fs.createReadStream = sinon.stub().throws(FileNotFoundError) + try { + await S3PersistorManager.promises.sendFile(bucket, key, filename) + } catch (err) { + error = err } - ) + }) + + it('returns a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('wraps the error', function() { + expect(error.cause).to.equal(FileNotFoundError) + }) }) - return it('should send the file to the filestore', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - err => { - this.S3PersistorManager.sendFile.called.should.equal(true) - return done() + describe('when reading the file throws an error', function() { + let error + + beforeEach(async function() { + Fs.createReadStream = sinon.stub().throws(genericError) + try { + await S3PersistorManager.promises.sendFile(bucket, key, filename) + } catch (err) { + error = err } - ) + }) + + it('returns a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('wraps the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('copyFile', function() { - beforeEach(function() { - this.sourceKey = 'my/key' - return (this.destKey = 'my/dest/key') + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.copyFile(bucket, key, destKey) + }) + + it('should copy the object', function() { + expect(S3Client.copyObject).to.have.been.calledWith({ + Bucket: bucket, + Key: destKey, + CopySource: `${bucket}/${key}` + }) + }) }) - it('should use AWS SDK to copy file', function(done) { - this.s3Client.copyObject.callsArgWith(1, this.error) - return this.S3PersistorManager.copyFile( - this.bucketName, - this.sourceKey, - this.destKey, - err => { - err.should.equal(this.error) - this.s3Client.copyObject - .calledWith({ - Bucket: this.bucketName, - Key: this.destKey, - CopySource: this.bucketName + '/' + this.key - }) - .should.equal(true) - return done() - } - ) - }) + describe('when the file does not exist', function() { + let error - return it('should return a NotFoundError object if the original file does not exist', function(done) { - const NoSuchKeyError = { code: 'NoSuchKey' } - this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError) - return this.S3PersistorManager.copyFile( - this.bucketName, - this.sourceKey, - this.destKey, - err => { - expect(err instanceof this.Errors.NotFoundError).to.equal(true) - return done() + beforeEach(async function() { + S3Client.copyObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.copyFile(bucket, key, destKey) + } catch (err) { + error = err } - ) + }) + + it('should throw a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) }) }) - describe('deleteDirectory', () => - it('should list the contents passing them onto multi delete', function(done) { - const data = { Contents: [{ Key: '1234' }, { Key: '456' }] } - this.knoxClient.list.callsArgWith(1, null, data) - this.knoxClient.deleteMultiple.callsArgWith(1) - return this.S3PersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - this.knoxClient.deleteMultiple - .calledWith(['1234', '456']) - .should.equal(true) - return done() - } - ) - })) - describe('deleteFile', function() { - it('should use correct options', function(done) { - this.request.callsArgWith(1) + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.deleteFile(bucket, key) + }) - return this.S3PersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - const opts = this.request.args[0][0] - assert.deepEqual(opts.aws, { - key: this.settings.filestore.s3.key, - secret: this.settings.filestore.s3.secret, - bucket: this.bucketName - }) - opts.method.should.equal('delete') - opts.timeout.should.equal(30 * 1000) - opts.uri.should.equal( - `https://${this.bucketName}.s3.amazonaws.com/${this.key}` - ) - return done() - } - ) + it('should delete the object', function() { + expect(S3Client.deleteObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) }) - return it('should return the error', function(done) { - this.request.callsArgWith(1, this.error) + describe('when the file does not exist', function() { + let error - return this.S3PersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - err.should.equal(this.error) - return done() + beforeEach(async function() { + S3Client.deleteObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.deleteFile(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should throw a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + }) + }) + + describe('deleteDirectory', function() { + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.deleteDirectory(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should delete the objects using their keys', function() { + expect(S3Client.deleteObjects).to.have.been.calledWith({ + Bucket: bucket, + Delete: { + Objects: [{ Key: 'llama' }, { Key: 'hippo' }], + Quiet: true + } + }) + }) + }) + + describe('when there are no files', function() { + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) + return S3PersistorManager.promises.deleteDirectory(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should not try to delete any objects', function() { + expect(S3Client.deleteObjects).not.to.have.been.called + }) + }) + + describe('when there is an error listing the objects', function() { + let error + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.deleteDirectory(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) + + it('should not try to delete any objects', function() { + expect(S3Client.deleteObjects).not.to.have.been.called + }) + }) + + describe('when there is an error deleting the objects', function() { + let error + + beforeEach(async function() { + S3Client.deleteObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.deleteDirectory(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a WriteError', function() { + expect(error).to.be.an.instanceOf(Errors.WriteError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) + }) + }) + + describe('directorySize', function() { + describe('with valid parameters', function() { + let size + + beforeEach(async function() { + size = await S3PersistorManager.promises.directorySize(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should return the directory size', function() { + expect(size).to.equal(filesSize) + }) + }) + + describe('when there are no files', function() { + let size + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) + size = await S3PersistorManager.promises.directorySize(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should return zero', function() { + expect(size).to.equal(0) + }) + }) + + describe('when there is an error listing the objects', function() { + let error + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.directorySize(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('checkIfFileExists', function() { - it('should use correct options', function(done) { - this.request.callsArgWith(1, null, { statusCode: 200 }) + describe('when the file exists', function() { + let exists - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - err => { - const opts = this.request.args[0][0] - assert.deepEqual(opts.aws, { - key: this.settings.filestore.s3.key, - secret: this.settings.filestore.s3.secret, - bucket: this.bucketName - }) - opts.method.should.equal('head') - opts.timeout.should.equal(30 * 1000) - opts.uri.should.equal( - `https://${this.bucketName}.s3.amazonaws.com/${this.key}` - ) - return done() - } - ) + beforeEach(async function() { + exists = await S3PersistorManager.promises.checkIfFileExists( + bucket, + key + ) + }) + + it('should get the object header', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('should return that the file exists', function() { + expect(exists).to.equal(true) + }) }) - it('should return true for a 200', function(done) { - this.request.callsArgWith(1, null, { statusCode: 200 }) + describe('when the file does not exist', function() { + let exists - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - exists.should.equal(true) - return done() - } - ) + beforeEach(async function() { + S3Client.headObject = sinon + .stub() + .returns({ promise: sinon.stub().rejects(S3NotFoundError) }) + exists = await S3PersistorManager.promises.checkIfFileExists( + bucket, + key + ) + }) + + it('should get the object header', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('should return that the file does not exist', function() { + expect(exists).to.equal(false) + }) }) - it('should return false for a non 200', function(done) { - this.request.callsArgWith(1, null, { statusCode: 404 }) + describe('when there is an error', function() { + let error - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - exists.should.equal(false) - return done() + beforeEach(async function() { + S3Client.headObject = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.checkIfFileExists(bucket, key) + } catch (err) { + error = err } - ) - }) + }) - return it('should return the error', function(done) { - this.request.callsArgWith(1, this.error, {}) + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - err => { - err.should.equal(this.error) - return done() - } - ) + it('should wrap the upstream ReadError', function() { + expect(error.cause).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should eventually wrap the error', function() { + expect(error.cause.cause).to.equal(genericError) + }) }) }) - - return describe('directorySize', () => - it('should sum directory files size', function(done) { - const data = { Contents: [{ Size: 1024 }, { Size: 2048 }] } - this.knoxClient.list.callsArgWith(1, null, data) - return this.S3PersistorManager.directorySize( - this.bucketName, - this.key, - (err, totalSize) => { - totalSize.should.equal(3072) - return done() - } - ) - })) })