mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #73 from overleaf/spd-decaf-cleanup-10
Cleanup and refactor S3PersistorManager to use aws-sdk only
This commit is contained in:
Simon Detheridge
GitHub
commit
384896d70c
9 changed files with 906 additions and 1589 deletions
|
|
@ -1,197 +0,0 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
// This module is not used in production, which currently uses
|
||||
// S3PersistorManager. The intention is to migrate S3PersistorManager to use the
|
||||
// latest aws-sdk and delete this module so that PersistorManager would load the
|
||||
// same backend for both the 's3' and 'aws-sdk' options.
|
||||
|
||||
const logger = require('logger-sharelatex')
|
||||
const aws = require('aws-sdk')
|
||||
const _ = require('underscore')
|
||||
const fs = require('fs')
|
||||
const Errors = require('./Errors')
|
||||
|
||||
const s3 = new aws.S3()
|
||||
|
||||
module.exports = {
|
||||
sendFile(bucketName, key, fsPath, callback) {
|
||||
logger.log({ bucketName, key }, 'send file data to s3')
|
||||
const stream = fs.createReadStream(fsPath)
|
||||
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
|
||||
err,
|
||||
data
|
||||
) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, Bucket: bucketName, Key: key },
|
||||
'error sending file data to s3'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
sendStream(bucketName, key, stream, callback) {
|
||||
logger.log({ bucketName, key }, 'send file stream to s3')
|
||||
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
|
||||
err,
|
||||
data
|
||||
) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, Bucket: bucketName, Key: key },
|
||||
'error sending file stream to s3'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
getFileStream(bucketName, key, opts, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(err, res) {}
|
||||
}
|
||||
logger.log({ bucketName, key }, 'get file stream from s3')
|
||||
callback = _.once(callback)
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
Key: key
|
||||
}
|
||||
if (opts.start != null && opts.end != null) {
|
||||
params.Range = `bytes=${opts.start}-${opts.end}`
|
||||
}
|
||||
const request = s3.getObject(params)
|
||||
const stream = request.createReadStream()
|
||||
stream.on('readable', () => callback(null, stream))
|
||||
return stream.on('error', function(err) {
|
||||
logger.err({ err, bucketName, key }, 'error getting file stream from s3')
|
||||
if (err.code === 'NoSuchKey') {
|
||||
return callback(
|
||||
new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
copyFile(bucketName, sourceKey, destKey, callback) {
|
||||
logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
|
||||
const source = bucketName + '/' + sourceKey
|
||||
return s3.copyObject(
|
||||
{ Bucket: bucketName, Key: destKey, CopySource: source },
|
||||
function(err) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, sourceKey, destKey },
|
||||
'something went wrong copying file in s3'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
deleteFile(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'delete file in s3')
|
||||
return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong deleting file in s3'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
deleteDirectory(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'delete directory in s3')
|
||||
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
|
||||
err,
|
||||
data
|
||||
) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong listing prefix in s3'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
if (data.Contents.length === 0) {
|
||||
logger.log({ bucketName, key }, 'the directory is empty')
|
||||
return callback()
|
||||
}
|
||||
const keys = _.map(data.Contents, entry => ({
|
||||
Key: entry.Key
|
||||
}))
|
||||
return s3.deleteObjects(
|
||||
{
|
||||
Bucket: bucketName,
|
||||
Delete: {
|
||||
Objects: keys,
|
||||
Quiet: true
|
||||
}
|
||||
},
|
||||
function(err) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key: keys },
|
||||
'something went wrong deleting directory in s3'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
checkIfFileExists(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'check file existence in s3')
|
||||
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
|
||||
if (err != null) {
|
||||
if (err.code === 'NotFound') {
|
||||
return callback(null, false)
|
||||
}
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong checking head in s3'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
return callback(null, data.ETag != null)
|
||||
})
|
||||
},
|
||||
|
||||
directorySize(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'get project size in s3')
|
||||
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
|
||||
err,
|
||||
data
|
||||
) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong listing prefix in s3'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
if (data.Contents.length === 0) {
|
||||
logger.log({ bucketName, key }, 'the directory is empty')
|
||||
return callback()
|
||||
}
|
||||
let totalSize = 0
|
||||
_.each(data.Contents, entry => (totalSize += entry.Size))
|
||||
return callback(null, totalSize)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -17,7 +17,7 @@ module.exports = {
|
|||
directorySize
|
||||
}
|
||||
|
||||
function getFile(req, res) {
|
||||
function getFile(req, res, next) {
|
||||
const { key, bucket } = req
|
||||
const { format, style } = req.query
|
||||
const options = {
|
||||
|
|
@ -61,7 +61,9 @@ function getFile(req, res) {
|
|||
}
|
||||
|
||||
logger.log({ key, bucket, format, style }, 'sending file to response')
|
||||
pipeline(fileStream, res)
|
||||
|
||||
// pass 'next' as a callback to 'pipeline' to receive any errors
|
||||
pipeline(fileStream, res, next)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ if (!settings.filestore.backend) {
|
|||
|
||||
switch (settings.filestore.backend) {
|
||||
case 'aws-sdk':
|
||||
module.exports = require('./AWSSDKPersistorManager')
|
||||
break
|
||||
case 's3':
|
||||
module.exports = require('./S3PersistorManager')
|
||||
break
|
||||
|
|
|
|||
|
|
@ -1,376 +1,258 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
new-cap,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
standard/no-callback-literal,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
// This module is the one which is used in production. It needs to be migrated
|
||||
// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
|
||||
// details. The knox library is unmaintained and has bugs.
|
||||
|
||||
const http = require('http')
|
||||
http.globalAgent.maxSockets = 300
|
||||
const https = require('https')
|
||||
http.globalAgent.maxSockets = 300
|
||||
https.globalAgent.maxSockets = 300
|
||||
|
||||
const settings = require('settings-sharelatex')
|
||||
const request = require('request')
|
||||
const logger = require('logger-sharelatex')
|
||||
const metrics = require('metrics-sharelatex')
|
||||
|
||||
const meter = require('stream-meter')
|
||||
const fs = require('fs')
|
||||
const knox = require('knox')
|
||||
const path = require('path')
|
||||
const LocalFileWriter = require('./LocalFileWriter')
|
||||
const Errors = require('./Errors')
|
||||
const _ = require('underscore')
|
||||
const awsS3 = require('aws-sdk/clients/s3')
|
||||
const URL = require('url')
|
||||
const S3 = require('aws-sdk/clients/s3')
|
||||
const { URL } = require('url')
|
||||
const { callbackify } = require('util')
|
||||
const { WriteError, ReadError, NotFoundError } = require('./Errors')
|
||||
|
||||
const thirtySeconds = 30 * 1000
|
||||
|
||||
const buildDefaultOptions = function(bucketName, method, key) {
|
||||
let endpoint
|
||||
if (settings.filestore.s3.endpoint) {
|
||||
endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`
|
||||
} else {
|
||||
endpoint = `https://${bucketName}.s3.amazonaws.com`
|
||||
}
|
||||
return {
|
||||
aws: {
|
||||
key: settings.filestore.s3.key,
|
||||
secret: settings.filestore.s3.secret,
|
||||
bucket: bucketName
|
||||
},
|
||||
method,
|
||||
timeout: thirtySeconds,
|
||||
uri: `${endpoint}/${key}`
|
||||
module.exports = {
|
||||
sendFile: callbackify(sendFile),
|
||||
sendStream: callbackify(sendStream),
|
||||
getFileStream: callbackify(getFileStream),
|
||||
deleteDirectory: callbackify(deleteDirectory),
|
||||
getFileSize: callbackify(getFileSize),
|
||||
deleteFile: callbackify(deleteFile),
|
||||
copyFile: callbackify(copyFile),
|
||||
checkIfFileExists: callbackify(checkIfFileExists),
|
||||
directorySize: callbackify(directorySize),
|
||||
promises: {
|
||||
sendFile,
|
||||
sendStream,
|
||||
getFileStream,
|
||||
deleteDirectory,
|
||||
getFileSize,
|
||||
deleteFile,
|
||||
copyFile,
|
||||
checkIfFileExists,
|
||||
directorySize
|
||||
}
|
||||
}
|
||||
|
||||
const getS3Options = function(credentials) {
|
||||
const _client = new S3(_defaultOptions())
|
||||
|
||||
async function sendFile(bucketName, key, fsPath) {
|
||||
let readStream
|
||||
try {
|
||||
readStream = fs.createReadStream(fsPath)
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'error reading file from disk',
|
||||
{ bucketName, key, fsPath },
|
||||
ReadError
|
||||
)
|
||||
}
|
||||
return sendStream(bucketName, key, readStream)
|
||||
}
|
||||
|
||||
async function sendStream(bucketName, key, readStream) {
|
||||
try {
|
||||
const meteredStream = meter()
|
||||
meteredStream.on('finish', () => {
|
||||
metrics.count('s3.egress', meteredStream.bytes)
|
||||
})
|
||||
|
||||
const response = await _client
|
||||
.upload({
|
||||
Bucket: bucketName,
|
||||
Key: key,
|
||||
Body: readStream.pipe(meteredStream)
|
||||
})
|
||||
.promise()
|
||||
|
||||
logger.log({ response, bucketName, key }, 'data uploaded to s3')
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'upload to S3 failed',
|
||||
{ bucketName, key },
|
||||
WriteError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function getFileStream(bucketName, key, opts) {
|
||||
opts = opts || {}
|
||||
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
Key: key
|
||||
}
|
||||
if (opts.start != null && opts.end != null) {
|
||||
params.Range = `bytes=${opts.start}-${opts.end}`
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const stream = _client.getObject(params).createReadStream()
|
||||
|
||||
const meteredStream = meter()
|
||||
meteredStream.on('finish', () => {
|
||||
metrics.count('s3.ingress', meteredStream.bytes)
|
||||
})
|
||||
|
||||
const onStreamReady = function() {
|
||||
stream.removeListener('readable', onStreamReady)
|
||||
resolve(stream.pipe(meteredStream))
|
||||
}
|
||||
stream.on('readable', onStreamReady)
|
||||
stream.on('error', err => {
|
||||
reject(_wrapError(err, 'error reading from S3', params, ReadError))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function deleteDirectory(bucketName, key) {
|
||||
logger.log({ key, bucketName }, 'deleting directory')
|
||||
let response
|
||||
|
||||
try {
|
||||
response = await _client
|
||||
.listObjects({ Bucket: bucketName, Prefix: key })
|
||||
.promise()
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'failed to list objects in S3',
|
||||
{ bucketName, key },
|
||||
ReadError
|
||||
)
|
||||
}
|
||||
|
||||
const objects = response.Contents.map(item => ({ Key: item.Key }))
|
||||
if (objects.length) {
|
||||
try {
|
||||
await _client
|
||||
.deleteObjects({
|
||||
Bucket: bucketName,
|
||||
Delete: {
|
||||
Objects: objects,
|
||||
Quiet: true
|
||||
}
|
||||
})
|
||||
.promise()
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'failed to delete objects in S3',
|
||||
{ bucketName, key },
|
||||
WriteError
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getFileSize(bucketName, key) {
|
||||
try {
|
||||
const response = await _client
|
||||
.headObject({ Bucket: bucketName, Key: key })
|
||||
.promise()
|
||||
return response.ContentLength
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'error getting size of s3 object',
|
||||
{ bucketName, key },
|
||||
ReadError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFile(bucketName, key) {
|
||||
try {
|
||||
await _client.deleteObject({ Bucket: bucketName, Key: key }).promise()
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'failed to delete file in S3',
|
||||
{ bucketName, key },
|
||||
WriteError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function copyFile(bucketName, sourceKey, destKey) {
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
Key: destKey,
|
||||
CopySource: `${bucketName}/${sourceKey}`
|
||||
}
|
||||
try {
|
||||
await _client.copyObject(params).promise()
|
||||
} catch (err) {
|
||||
throw _wrapError(err, 'failed to copy file in S3', params, WriteError)
|
||||
}
|
||||
}
|
||||
|
||||
async function checkIfFileExists(bucketName, key) {
|
||||
try {
|
||||
await getFileSize(bucketName, key)
|
||||
return true
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
return false
|
||||
}
|
||||
throw _wrapError(
|
||||
err,
|
||||
'error checking whether S3 object exists',
|
||||
{ bucketName, key },
|
||||
ReadError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function directorySize(bucketName, key) {
|
||||
try {
|
||||
const response = await _client
|
||||
.listObjects({ Bucket: bucketName, Prefix: key })
|
||||
.promise()
|
||||
|
||||
return response.Contents.reduce((acc, item) => item.Size + acc, 0)
|
||||
} catch (err) {
|
||||
throw _wrapError(
|
||||
err,
|
||||
'error getting directory size in S3',
|
||||
{ bucketName, key },
|
||||
ReadError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function _wrapError(error, message, params, ErrorType) {
|
||||
if (['NoSuchKey', 'NotFound', 'ENOENT'].includes(error.code)) {
|
||||
return new NotFoundError({
|
||||
message: 'no such file',
|
||||
info: params
|
||||
}).withCause(error)
|
||||
} else {
|
||||
return new ErrorType({
|
||||
message: message,
|
||||
info: params
|
||||
}).withCause(error)
|
||||
}
|
||||
}
|
||||
|
||||
function _defaultOptions() {
|
||||
const options = {
|
||||
credentials: {
|
||||
accessKeyId: credentials.auth_key,
|
||||
secretAccessKey: credentials.auth_secret
|
||||
accessKeyId: settings.filestore.s3.key,
|
||||
secretAccessKey: settings.filestore.s3.secret
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.filestore.s3.endpoint) {
|
||||
const endpoint = URL.parse(settings.filestore.s3.endpoint)
|
||||
const endpoint = new URL(settings.filestore.s3.endpoint)
|
||||
options.endpoint = settings.filestore.s3.endpoint
|
||||
options.sslEnabled = endpoint.protocol === 'https'
|
||||
}
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
const defaultS3Client = new awsS3(
|
||||
getS3Options({
|
||||
auth_key: settings.filestore.s3.key,
|
||||
auth_secret: settings.filestore.s3.secret
|
||||
})
|
||||
)
|
||||
|
||||
const getS3Client = function(credentials) {
|
||||
if (credentials != null) {
|
||||
return new awsS3(getS3Options(credentials))
|
||||
} else {
|
||||
return defaultS3Client
|
||||
}
|
||||
}
|
||||
|
||||
const getKnoxClient = bucketName => {
|
||||
const options = {
|
||||
key: settings.filestore.s3.key,
|
||||
secret: settings.filestore.s3.secret,
|
||||
bucket: bucketName
|
||||
}
|
||||
if (settings.filestore.s3.endpoint) {
|
||||
const endpoint = URL.parse(settings.filestore.s3.endpoint)
|
||||
options.endpoint = endpoint.hostname
|
||||
options.port = endpoint.port
|
||||
}
|
||||
return knox.createClient(options)
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sendFile(bucketName, key, fsPath, callback) {
|
||||
const s3Client = getKnoxClient(bucketName)
|
||||
let uploaded = 0
|
||||
const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) {
|
||||
metrics.count('s3.egress', uploaded)
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key, fsPath },
|
||||
'something went wrong uploading file to s3'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
if (res == null) {
|
||||
logger.err(
|
||||
{ err, res, bucketName, key, fsPath },
|
||||
'no response from s3 put file'
|
||||
)
|
||||
return callback('no response from put file')
|
||||
}
|
||||
if (res.statusCode !== 200) {
|
||||
logger.err(
|
||||
{ bucketName, key, fsPath },
|
||||
'non 200 response from s3 putting file'
|
||||
)
|
||||
return callback('non 200 response from s3 on put file')
|
||||
}
|
||||
logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3')
|
||||
return callback(err)
|
||||
})
|
||||
putEventEmiter.on('error', function(err) {
|
||||
logger.err(
|
||||
{ err, bucketName, key, fsPath },
|
||||
'error emmited on put of file'
|
||||
)
|
||||
return callback(err)
|
||||
})
|
||||
return putEventEmiter.on(
|
||||
'progress',
|
||||
progress => (uploaded = progress.written)
|
||||
)
|
||||
},
|
||||
|
||||
sendStream(bucketName, key, readStream, callback) {
|
||||
logger.log({ bucketName, key }, 'sending file to s3')
|
||||
readStream.on('error', err =>
|
||||
logger.err({ bucketName, key }, 'error on stream to send to s3')
|
||||
)
|
||||
return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ bucketName, key, fsPath, err },
|
||||
'something went wrong writing stream to disk'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
return this.sendFile(bucketName, key, fsPath, (
|
||||
err // delete the temporary file created above and return the original error
|
||||
) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
|
||||
})
|
||||
},
|
||||
|
||||
// opts may be {start: Number, end: Number}
|
||||
getFileStream(bucketName, key, opts, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(err, res) {}
|
||||
}
|
||||
opts = opts || {}
|
||||
callback = _.once(callback)
|
||||
logger.log({ bucketName, key }, 'getting file from s3')
|
||||
|
||||
const s3 = getS3Client(opts.credentials)
|
||||
const s3Params = {
|
||||
Bucket: bucketName,
|
||||
Key: key
|
||||
}
|
||||
if (opts.start != null && opts.end != null) {
|
||||
s3Params.Range = `bytes=${opts.start}-${opts.end}`
|
||||
}
|
||||
const s3Request = s3.getObject(s3Params)
|
||||
|
||||
s3Request.on(
|
||||
'httpHeaders',
|
||||
(statusCode, headers, response, statusMessage) => {
|
||||
if ([403, 404].includes(statusCode)) {
|
||||
// S3 returns a 403 instead of a 404 when the user doesn't have
|
||||
// permission to list the bucket contents.
|
||||
logger.log({ bucketName, key }, 'file not found in s3')
|
||||
return callback(
|
||||
new Errors.NotFoundError(
|
||||
`File not found in S3: ${bucketName}:${key}`
|
||||
),
|
||||
null
|
||||
)
|
||||
}
|
||||
if (![200, 206].includes(statusCode)) {
|
||||
logger.log(
|
||||
{ bucketName, key },
|
||||
`error getting file from s3: ${statusCode}`
|
||||
)
|
||||
return callback(
|
||||
new Error(
|
||||
`Got non-200 response from S3: ${statusCode} ${statusMessage}`
|
||||
),
|
||||
null
|
||||
)
|
||||
}
|
||||
const stream = response.httpResponse.createUnbufferedStream()
|
||||
stream.on('data', data => metrics.count('s3.ingress', data.byteLength))
|
||||
|
||||
return callback(null, stream)
|
||||
}
|
||||
)
|
||||
|
||||
s3Request.on('error', err => {
|
||||
logger.err({ err, bucketName, key }, 'error getting file stream from s3')
|
||||
return callback(err)
|
||||
})
|
||||
|
||||
return s3Request.send()
|
||||
},
|
||||
|
||||
getFileSize(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'getting file size from S3')
|
||||
const s3 = getS3Client()
|
||||
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
|
||||
if (err != null) {
|
||||
if ([403, 404].includes(err.statusCode)) {
|
||||
// S3 returns a 403 instead of a 404 when the user doesn't have
|
||||
// permission to list the bucket contents.
|
||||
logger.log(
|
||||
{
|
||||
bucketName,
|
||||
key
|
||||
},
|
||||
'file not found in s3'
|
||||
)
|
||||
callback(
|
||||
new Errors.NotFoundError(
|
||||
`File not found in S3: ${bucketName}:${key}`
|
||||
)
|
||||
)
|
||||
} else {
|
||||
logger.err(
|
||||
{
|
||||
bucketName,
|
||||
key,
|
||||
err
|
||||
},
|
||||
'error performing S3 HeadObject'
|
||||
)
|
||||
callback(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
return callback(null, data.ContentLength)
|
||||
})
|
||||
},
|
||||
|
||||
copyFile(bucketName, sourceKey, destKey, callback) {
|
||||
logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
|
||||
const source = bucketName + '/' + sourceKey
|
||||
// use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
|
||||
const s3 = getS3Client()
|
||||
return s3.copyObject(
|
||||
{ Bucket: bucketName, Key: destKey, CopySource: source },
|
||||
function(err) {
|
||||
if (err != null) {
|
||||
if (err.code === 'NoSuchKey') {
|
||||
logger.err(
|
||||
{ bucketName, sourceKey },
|
||||
'original file not found in s3 when copying'
|
||||
)
|
||||
return callback(
|
||||
new Errors.NotFoundError(
|
||||
'original file not found in S3 when copying'
|
||||
)
|
||||
)
|
||||
} else {
|
||||
logger.err(
|
||||
{ err, bucketName, sourceKey, destKey },
|
||||
'something went wrong copying file in aws'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
deleteFile(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'delete file in s3')
|
||||
const options = buildDefaultOptions(bucketName, 'delete', key)
|
||||
return request(options, function(err, res) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, res, bucketName, key },
|
||||
'something went wrong deleting file in aws'
|
||||
)
|
||||
}
|
||||
return callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
deleteDirectory(bucketName, key, _callback) {
|
||||
// deleteMultiple can call the callback multiple times so protect against this.
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []))
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
logger.log({ key, bucketName }, 'deleting directory')
|
||||
const s3Client = getKnoxClient(bucketName)
|
||||
return s3Client.list({ prefix: key }, function(err, data) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong listing prefix in aws'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
const keys = _.map(data.Contents, entry => entry.Key)
|
||||
return s3Client.deleteMultiple(keys, callback)
|
||||
})
|
||||
},
|
||||
|
||||
checkIfFileExists(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'checking if file exists in s3')
|
||||
const options = buildDefaultOptions(bucketName, 'head', key)
|
||||
return request(options, function(err, res) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, res, bucketName, key },
|
||||
'something went wrong checking file in aws'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
if (res == null) {
|
||||
logger.err(
|
||||
{ err, res, bucketName, key },
|
||||
'no response object returned when checking if file exists'
|
||||
)
|
||||
err = new Error(`no response from s3 ${bucketName} ${key}`)
|
||||
return callback(err)
|
||||
}
|
||||
const exists = res.statusCode === 200
|
||||
logger.log({ bucketName, key, exists }, 'checked if file exsists in s3')
|
||||
return callback(err, exists)
|
||||
})
|
||||
},
|
||||
|
||||
directorySize(bucketName, key, callback) {
|
||||
logger.log({ bucketName, key }, 'get project size in s3')
|
||||
const s3Client = getKnoxClient(bucketName)
|
||||
return s3Client.list({ prefix: key }, function(err, data) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{ err, bucketName, key },
|
||||
'something went wrong listing prefix in aws'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
let totalSize = 0
|
||||
_.each(data.Contents, entry => (totalSize += entry.Size))
|
||||
logger.log({ totalSize }, 'total size')
|
||||
return callback(null, totalSize)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
34
services/filestore/npm-shrinkwrap.json
generated
34
services/filestore/npm-shrinkwrap.json
generated
|
|
@ -5018,6 +5018,38 @@
|
|||
"resolved": "https://registry.npmjs.org/stream-counter/-/stream-counter-1.0.0.tgz",
|
||||
"integrity": "sha1-kc8lac5NxQYf6816yyY5SloRR1E="
|
||||
},
|
||||
"stream-meter": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz",
|
||||
"integrity": "sha1-Uq+Vql6nYKJJFxZwTb/5D3Ov3R0=",
|
||||
"requires": {
|
||||
"readable-stream": "^2.1.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"readable-stream": {
|
||||
"version": "2.3.6",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
|
||||
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"stream-shift": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
|
||||
|
|
@ -5531,7 +5563,7 @@
|
|||
"xml2js": {
|
||||
"version": "0.4.19",
|
||||
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
|
||||
"integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=",
|
||||
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
|
||||
"requires": {
|
||||
"sax": ">=0.6.0",
|
||||
"xmlbuilder": "~9.0.1"
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@
|
|||
"settings-sharelatex": "^1.1.0",
|
||||
"stream-browserify": "^2.0.1",
|
||||
"stream-buffers": "~0.2.5",
|
||||
"stream-meter": "^1.0.4",
|
||||
"underscore": "~1.5.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -1,509 +0,0 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-dupe-keys,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
|
||||
const should = chai.should()
|
||||
const { expect } = chai
|
||||
|
||||
const modulePath = '../../../app/js/AWSSDKPersistorManager.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
describe('AWSSDKPersistorManager', function() {
|
||||
beforeEach(function() {
|
||||
this.settings = {
|
||||
filestore: {
|
||||
backend: 'aws-sdk'
|
||||
}
|
||||
}
|
||||
this.s3 = {
|
||||
upload: sinon.stub(),
|
||||
getObject: sinon.stub(),
|
||||
copyObject: sinon.stub(),
|
||||
deleteObject: sinon.stub(),
|
||||
listObjects: sinon.stub(),
|
||||
deleteObjects: sinon.stub(),
|
||||
headObject: sinon.stub()
|
||||
}
|
||||
this.awssdk = { S3: sinon.stub().returns(this.s3) }
|
||||
|
||||
this.requires = {
|
||||
'aws-sdk': this.awssdk,
|
||||
'settings-sharelatex': this.settings,
|
||||
'logger-sharelatex': {
|
||||
log() {},
|
||||
err() {}
|
||||
},
|
||||
fs: (this.fs = { createReadStream: sinon.stub() }),
|
||||
'./Errors': (this.Errors = { NotFoundError: sinon.stub() })
|
||||
}
|
||||
this.key = 'my/key'
|
||||
this.bucketName = 'my-bucket'
|
||||
this.error = 'my error'
|
||||
return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, {
|
||||
requires: this.requires
|
||||
}))
|
||||
})
|
||||
|
||||
describe('sendFile', function() {
|
||||
beforeEach(function() {
|
||||
this.stream = {}
|
||||
this.fsPath = '/usr/local/some/file'
|
||||
return this.fs.createReadStream.returns(this.stream)
|
||||
})
|
||||
|
||||
it('should put the file with s3.upload', function(done) {
|
||||
this.s3.upload.callsArgWith(1)
|
||||
return this.AWSSDKPersistorManager.sendFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.fsPath,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.upload.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.upload.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key,
|
||||
Body: this.stream
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.upload', function(done) {
|
||||
this.s3.upload.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.sendFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.fsPath,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function() {
|
||||
beforeEach(function() {
|
||||
return (this.stream = {})
|
||||
})
|
||||
|
||||
it('should put the file with s3.upload', function(done) {
|
||||
this.s3.upload.callsArgWith(1)
|
||||
return this.AWSSDKPersistorManager.sendStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.stream,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.upload.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.upload.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key,
|
||||
Body: this.stream
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.upload', function(done) {
|
||||
this.s3.upload.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.sendStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.stream,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFileStream', function() {
|
||||
beforeEach(function() {
|
||||
this.opts = {}
|
||||
this.stream = {}
|
||||
this.read_stream = { on: (this.read_stream_on = sinon.stub()) }
|
||||
this.object = { createReadStream: sinon.stub().returns(this.read_stream) }
|
||||
return this.s3.getObject.returns(this.object)
|
||||
})
|
||||
|
||||
it('should return a stream from s3.getObject', function(done) {
|
||||
this.read_stream_on.withArgs('readable').callsArgWith(1)
|
||||
|
||||
return this.AWSSDKPersistorManager.getFileStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.opts,
|
||||
(err, stream) => {
|
||||
expect(this.read_stream_on.calledTwice)
|
||||
expect(err).to.not.be.ok
|
||||
expect(stream, 'returned the stream').to.equal(this.read_stream)
|
||||
expect(
|
||||
this.s3.getObject.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('with start and end options', function() {
|
||||
beforeEach(function() {
|
||||
return (this.opts = {
|
||||
start: 0,
|
||||
end: 8
|
||||
})
|
||||
})
|
||||
return it('should pass headers to the s3.GetObject', function(done) {
|
||||
this.read_stream_on.withArgs('readable').callsArgWith(1)
|
||||
this.AWSSDKPersistorManager.getFileStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.opts,
|
||||
(err, stream) => {
|
||||
return expect(
|
||||
this.s3.getObject.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key,
|
||||
Range: 'bytes=0-8'
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
}
|
||||
)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return describe('error conditions', function() {
|
||||
describe("when the file doesn't exist", function() {
|
||||
beforeEach(function() {
|
||||
this.error = new Error()
|
||||
return (this.error.code = 'NoSuchKey')
|
||||
})
|
||||
return it('should produce a NotFoundError', function(done) {
|
||||
this.read_stream_on.withArgs('error').callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.getFileStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.opts,
|
||||
(err, stream) => {
|
||||
expect(stream).to.not.be.ok
|
||||
expect(err).to.be.ok
|
||||
expect(
|
||||
err instanceof this.Errors.NotFoundError,
|
||||
'error is a correct instance'
|
||||
).to.equal(true)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
return describe('when there is some other error', function() {
|
||||
beforeEach(function() {
|
||||
return (this.error = new Error())
|
||||
})
|
||||
return it('should dispatch the error from s3 object stream', function(done) {
|
||||
this.read_stream_on.withArgs('error').callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.getFileStream(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.opts,
|
||||
(err, stream) => {
|
||||
expect(stream).to.not.be.ok
|
||||
expect(err).to.be.ok
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyFile', function() {
|
||||
beforeEach(function() {
|
||||
this.destKey = 'some/key'
|
||||
return (this.stream = {})
|
||||
})
|
||||
|
||||
it('should copy the file with s3.copyObject', function(done) {
|
||||
this.s3.copyObject.callsArgWith(1)
|
||||
return this.AWSSDKPersistorManager.copyFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.destKey,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.copyObject.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.destKey,
|
||||
CopySource: this.bucketName + '/' + this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.copyObject', function(done) {
|
||||
this.s3.copyObject.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.copyFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
this.destKey,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteFile', function() {
|
||||
it('should delete the file with s3.deleteObject', function(done) {
|
||||
this.s3.deleteObject.callsArgWith(1)
|
||||
return this.AWSSDKPersistorManager.deleteFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.deleteObject.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.deleteObject', function(done) {
|
||||
this.s3.deleteObject.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.deleteFile(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteDirectory', function() {
|
||||
it('should list the directory content using s3.listObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, null, { Contents: [] })
|
||||
return this.AWSSDKPersistorManager.deleteDirectory(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.listObjects.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should dispatch the error from s3.listObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.deleteDirectory(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return describe('with directory content', function() {
|
||||
beforeEach(function() {
|
||||
return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }])
|
||||
})
|
||||
|
||||
it('should forward the file keys to s3.deleteObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList })
|
||||
this.s3.deleteObjects.callsArgWith(1)
|
||||
return this.AWSSDKPersistorManager.deleteDirectory(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be
|
||||
.true
|
||||
expect(
|
||||
this.s3.deleteObjects.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Delete: {
|
||||
Quiet: true,
|
||||
Objects: this.fileList
|
||||
}
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.deleteObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList })
|
||||
this.s3.deleteObjects.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.deleteDirectory(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkIfFileExists', function() {
|
||||
it('should check for the file with s3.headObject', function(done) {
|
||||
this.s3.headObject.callsArgWith(1, null, {})
|
||||
return this.AWSSDKPersistorManager.checkIfFileExists(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
(err, exists) => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.headObject.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.headObject.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should return false on an inexistant file', function(done) {
|
||||
this.s3.headObject.callsArgWith(1, null, {})
|
||||
return this.AWSSDKPersistorManager.checkIfFileExists(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
(err, exists) => {
|
||||
expect(exists).to.be.false
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should return true on an existing file', function(done) {
|
||||
this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' })
|
||||
return this.AWSSDKPersistorManager.checkIfFileExists(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
(err, exists) => {
|
||||
expect(exists).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should dispatch the error from s3.headObject', function(done) {
|
||||
this.s3.headObject.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.checkIfFileExists(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
(err, exists) => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
return describe('directorySize', function() {
|
||||
it('should list the directory content using s3.listObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, null, { Contents: [] })
|
||||
return this.AWSSDKPersistorManager.directorySize(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.not.be.ok
|
||||
expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true
|
||||
expect(
|
||||
this.s3.listObjects.calledWith({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.key
|
||||
}),
|
||||
'called with correct arguments'
|
||||
).to.be.true
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should dispatch the error from s3.listObjects', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, this.error)
|
||||
return this.AWSSDKPersistorManager.directorySize(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
err => {
|
||||
expect(err).to.equal(this.error)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should sum directory files sizes', function(done) {
|
||||
this.s3.listObjects.callsArgWith(1, null, {
|
||||
Contents: [{ Size: 1024 }, { Size: 2048 }]
|
||||
})
|
||||
return this.AWSSDKPersistorManager.directorySize(
|
||||
this.bucketName,
|
||||
this.key,
|
||||
(err, size) => {
|
||||
expect(size).to.equal(3072)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -43,6 +43,14 @@ describe('PersistorManager', function() {
|
|||
expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager')
|
||||
})
|
||||
|
||||
it("should implement the S3 wrapped method when 'aws-sdk' is configured", function() {
|
||||
settings.filestore.backend = 'aws-sdk'
|
||||
PersistorManager = SandboxedModule.require(modulePath, { requires })
|
||||
|
||||
expect(PersistorManager).to.respondTo('wrappedMethod')
|
||||
expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager')
|
||||
})
|
||||
|
||||
it('should implement the FS wrapped method when FS is configured', function() {
|
||||
settings.filestore.backend = 'fs'
|
||||
PersistorManager = SandboxedModule.require(modulePath, { requires })
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue