2020-07-02 09:19:45 -04:00
const sinon = require ( 'sinon' )
const chai = require ( 'chai' )
const { expect } = chai
const SandboxedModule = require ( 'sandboxed-module' )
const Errors = require ( '../../src/Errors' )
2024-11-08 05:21:56 -05:00
const { EventEmitter } = require ( 'node:events' )
2020-07-02 09:19:45 -04:00
2022-11-10 07:06:08 -05:00
const MODULE _PATH = '../../src/S3Persistor.js'
2020-07-02 09:19:45 -04:00
describe ( 'S3PersistorTests' , function ( ) {
const defaultS3Key = 'frog'
const defaultS3Secret = 'prince'
const defaultS3Credentials = {
credentials : {
accessKeyId : defaultS3Key ,
2021-12-16 04:04:32 -05:00
secretAccessKey : defaultS3Secret ,
} ,
2020-07-02 09:19:45 -04:00
}
const filename = '/wombat/potato.tex'
const bucket = 'womBucket'
const key = 'monKey'
const destKey = 'donKey'
const objectSize = 5555
const genericError = new Error ( 'guru meditation error' )
const files = [
{ Key : 'llama' , Size : 11 } ,
2021-12-16 04:04:32 -05:00
{ Key : 'hippo' , Size : 22 } ,
2020-07-02 09:19:45 -04:00
]
const filesSize = 33
const md5 = 'ffffffff00000000ffffffff00000000'
2020-07-08 16:56:23 -04:00
const redirectUrl = 'https://wombat.potato/giraffe'
2020-07-02 09:19:45 -04:00
2020-07-02 09:24:01 -04:00
let Logger ,
2020-07-02 09:19:45 -04:00
Transform ,
2023-05-23 03:07:06 -04:00
PassThrough ,
2020-07-02 09:19:45 -04:00
S3 ,
Fs ,
ReadStream ,
Stream ,
2022-11-10 07:06:08 -05:00
StreamPromises ,
2023-05-23 03:07:06 -04:00
S3GetObjectRequest ,
2020-07-02 09:19:45 -04:00
S3Persistor ,
S3Client ,
S3NotFoundError ,
S3AccessDeniedError ,
FileNotFoundError ,
EmptyPromise ,
settings ,
Hash ,
crypto
beforeEach ( function ( ) {
settings = {
secret : defaultS3Secret ,
key : defaultS3Key ,
2021-12-16 04:04:32 -05:00
partSize : 100 * 1024 * 1024 ,
2020-07-02 09:19:45 -04:00
}
Transform = class {
once ( ) { }
}
2023-05-23 03:07:06 -04:00
PassThrough = class { }
2020-07-02 09:19:45 -04:00
Stream = {
2022-05-16 10:25:49 -04:00
Transform ,
2023-05-23 03:07:06 -04:00
PassThrough ,
pipeline : sinon . stub ( ) . yields ( ) ,
2020-07-02 09:19:45 -04:00
}
2022-11-10 07:06:08 -05:00
StreamPromises = {
pipeline : sinon . stub ( ) . resolves ( ) ,
}
2020-07-02 09:19:45 -04:00
EmptyPromise = {
2021-12-16 04:04:32 -05:00
promise : sinon . stub ( ) . resolves ( ) ,
2020-07-02 09:19:45 -04:00
}
2023-05-23 03:07:06 -04:00
ReadStream = new EventEmitter ( )
class FakeS3GetObjectRequest extends EventEmitter {
constructor ( ) {
super ( )
this . statusCode = 200
this . err = null
this . aborted = false
}
abort ( ) {
this . aborted = true
}
createReadStream ( ) {
setTimeout ( ( ) => {
2024-11-08 03:32:58 -05:00
if ( this . notFoundSSEC ) {
// special case for AWS S3: 404 NoSuchKey wrapped in a 400. A single request received a single response, and multiple httpHeaders events are triggered. Don't ask.
this . emit ( 'httpHeaders' , 400 )
this . emit ( 'httpHeaders' , 404 )
ReadStream . emit ( 'error' , S3NotFoundError )
return
}
2023-05-23 03:07:06 -04:00
if ( this . err ) return ReadStream . emit ( 'error' , this . err )
this . emit ( 'httpHeaders' , this . statusCode )
2024-11-08 03:31:44 -05:00
if ( this . statusCode === 403 ) {
ReadStream . emit ( 'error' , S3AccessDeniedError )
}
if ( this . statusCode === 404 ) {
ReadStream . emit ( 'error' , S3NotFoundError )
}
2023-05-23 03:07:06 -04:00
} )
return ReadStream
}
2020-07-02 09:19:45 -04:00
}
2023-05-23 03:07:06 -04:00
S3GetObjectRequest = new FakeS3GetObjectRequest ( )
2020-07-02 09:19:45 -04:00
FileNotFoundError = new Error ( 'File not found' )
FileNotFoundError . code = 'ENOENT'
Fs = {
2021-12-16 04:04:32 -05:00
createReadStream : sinon . stub ( ) . returns ( ReadStream ) ,
2020-07-02 09:19:45 -04:00
}
S3NotFoundError = new Error ( 'not found' )
S3NotFoundError . code = 'NoSuchKey'
S3AccessDeniedError = new Error ( 'access denied' )
S3AccessDeniedError . code = 'AccessDenied'
S3Client = {
2023-05-23 03:07:06 -04:00
getObject : sinon . stub ( ) . returns ( S3GetObjectRequest ) ,
2020-07-02 09:19:45 -04:00
headObject : sinon . stub ( ) . returns ( {
promise : sinon . stub ( ) . resolves ( {
ContentLength : objectSize ,
2021-12-16 04:04:32 -05:00
ETag : md5 ,
} ) ,
2020-07-02 09:19:45 -04:00
} ) ,
listObjectsV2 : sinon . stub ( ) . returns ( {
promise : sinon . stub ( ) . resolves ( {
2021-12-16 04:04:32 -05:00
Contents : files ,
} ) ,
2020-07-02 09:19:45 -04:00
} ) ,
upload : sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . resolves ( { ETag : ` " ${ md5 } " ` } ) } ) ,
copyObject : sinon . stub ( ) . returns ( EmptyPromise ) ,
deleteObject : sinon . stub ( ) . returns ( EmptyPromise ) ,
2020-07-08 16:56:23 -04:00
deleteObjects : sinon . stub ( ) . returns ( EmptyPromise ) ,
2021-12-16 04:04:32 -05:00
getSignedUrlPromise : sinon . stub ( ) . resolves ( redirectUrl ) ,
2020-07-02 09:19:45 -04:00
}
S3 = sinon . stub ( ) . returns ( S3Client )
Hash = {
end : sinon . stub ( ) ,
read : sinon . stub ( ) . returns ( md5 ) ,
2021-12-16 04:04:32 -05:00
setEncoding : sinon . stub ( ) ,
2020-07-02 09:19:45 -04:00
}
crypto = {
2021-12-16 04:04:32 -05:00
createHash : sinon . stub ( ) . returns ( Hash ) ,
2020-07-02 09:19:45 -04:00
}
Logger = {
2021-12-16 04:04:32 -05:00
warn : sinon . stub ( ) ,
2020-07-02 09:19:45 -04:00
}
2022-11-10 07:06:08 -05:00
S3Persistor = new ( SandboxedModule . require ( MODULE _PATH , {
2020-07-02 09:19:45 -04:00
requires : {
'aws-sdk/clients/s3' : S3 ,
2022-05-16 10:25:37 -04:00
'@overleaf/logger' : Logger ,
2020-07-02 09:19:45 -04:00
'./Errors' : Errors ,
fs : Fs ,
stream : Stream ,
2022-11-10 07:06:08 -05:00
'stream/promises' : StreamPromises ,
2021-12-16 04:04:32 -05:00
crypto ,
2020-07-02 09:19:45 -04:00
} ,
2021-12-16 04:04:32 -05:00
globals : { console , Buffer } ,
2024-11-08 03:31:26 -05:00
} ) . S3Persistor ) ( settings )
2020-07-02 09:19:45 -04:00
} )
describe ( 'getObjectStream' , function ( ) {
describe ( 'when called with valid parameters' , function ( ) {
let stream
beforeEach ( async function ( ) {
stream = await S3Persistor . getObjectStream ( bucket , key )
} )
2023-05-23 03:07:06 -04:00
it ( 'returns a PassThrough stream' , function ( ) {
expect ( stream ) . to . be . instanceOf ( PassThrough )
2020-07-02 09:19:45 -04:00
} )
it ( 'sets the AWS client up with credentials from settings' , function ( ) {
expect ( S3 ) . to . have . been . calledWith ( defaultS3Credentials )
} )
it ( 'fetches the right key from the right bucket' , function ( ) {
expect ( S3Client . getObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'pipes the stream through the meter' , async function ( ) {
2023-05-23 03:07:06 -04:00
expect ( Stream . pipeline ) . to . have . been . calledWith (
ReadStream ,
sinon . match . instanceOf ( Transform ) ,
sinon . match . instanceOf ( PassThrough )
2020-07-02 09:19:45 -04:00
)
} )
2023-05-23 03:07:06 -04:00
it ( 'does not abort the request' , function ( ) {
expect ( S3GetObjectRequest . aborted ) . to . equal ( false )
} )
2020-07-02 09:19:45 -04:00
} )
describe ( 'when called with a byte range' , function ( ) {
let stream
beforeEach ( async function ( ) {
stream = await S3Persistor . getObjectStream ( bucket , key , {
start : 5 ,
2021-12-16 04:04:32 -05:00
end : 10 ,
2020-07-02 09:19:45 -04:00
} )
} )
2023-05-23 03:07:06 -04:00
it ( 'returns a PassThrough stream' , function ( ) {
expect ( stream ) . to . be . instanceOf ( Stream . PassThrough )
2020-07-02 09:19:45 -04:00
} )
it ( 'passes the byte range on to S3' , function ( ) {
expect ( S3Client . getObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : key ,
2021-12-16 04:04:32 -05:00
Range : 'bytes=5-10' ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
2023-05-23 03:07:06 -04:00
describe ( 'when streaming fails' , function ( ) {
let stream
beforeEach ( async function ( ) {
Stream . pipeline . yields ( new Error ( ) )
stream = await S3Persistor . getObjectStream ( bucket , key )
} )
it ( 'returns a PassThrough stream' , function ( ) {
expect ( stream ) . to . be . instanceOf ( Stream . PassThrough )
} )
it ( 'aborts the request' , function ( ) {
expect ( S3GetObjectRequest . aborted ) . to . equal ( true )
} )
} )
2020-07-02 09:19:45 -04:00
describe ( 'when there are alternative credentials' , function ( ) {
let stream
const alternativeSecret = 'giraffe'
const alternativeKey = 'hippo'
const alternativeS3Credentials = {
credentials : {
accessKeyId : alternativeKey ,
2021-12-16 04:04:32 -05:00
secretAccessKey : alternativeSecret ,
} ,
2020-07-02 09:19:45 -04:00
}
beforeEach ( async function ( ) {
settings . bucketCreds = { }
settings . bucketCreds [ bucket ] = {
auth _key : alternativeKey ,
2021-12-16 04:04:32 -05:00
auth _secret : alternativeSecret ,
2020-07-02 09:19:45 -04:00
}
stream = await S3Persistor . getObjectStream ( bucket , key )
} )
2023-05-23 03:07:06 -04:00
it ( 'returns a PassThrough stream' , function ( ) {
expect ( stream ) . to . be . instanceOf ( Stream . PassThrough )
2020-07-02 09:19:45 -04:00
} )
it ( 'sets the AWS client up with the alternative credentials' , function ( ) {
expect ( S3 ) . to . have . been . calledWith ( alternativeS3Credentials )
} )
it ( 'fetches the right key from the right bucket' , function ( ) {
expect ( S3Client . getObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'uses the default credentials for an unknown bucket' , async function ( ) {
stream = await S3Persistor . getObjectStream ( 'anotherBucket' , key )
expect ( S3 ) . to . have . been . calledTwice
expect ( S3 . firstCall ) . to . have . been . calledWith ( alternativeS3Credentials )
expect ( S3 . secondCall ) . to . have . been . calledWith ( defaultS3Credentials )
} )
2024-02-21 06:51:22 -05:00
} )
2020-07-02 09:19:45 -04:00
2024-02-21 06:51:22 -05:00
describe ( 'without hard-coded credentials' , function ( ) {
it ( 'uses the default provider chain' , async function ( ) {
2020-07-02 09:19:45 -04:00
delete settings . key
delete settings . secret
2024-02-21 06:51:22 -05:00
await S3Persistor . getObjectStream ( bucket , key )
expect ( S3 ) . to . have . been . calledOnce
expect ( S3 . args [ 0 ] . credentials ) . to . not . exist
2020-07-02 09:19:45 -04:00
} )
} )
2020-07-09 14:01:13 -04:00
describe ( 'when given S3 options' , function ( ) {
const httpOptions = { timeout : 2000 }
const maxRetries = 2
beforeEach ( async function ( ) {
settings . httpOptions = httpOptions
settings . maxRetries = maxRetries
await S3Persistor . getObjectStream ( bucket , key )
} )
it ( 'configures the S3 client appropriately' , function ( ) {
expect ( S3 ) . to . have . been . calledWithMatch ( { httpOptions , maxRetries } )
} )
} )
2020-07-02 09:19:45 -04:00
describe ( "when the file doesn't exist" , function ( ) {
let error , stream
beforeEach ( async function ( ) {
2023-05-23 03:07:06 -04:00
S3GetObjectRequest . statusCode = 404
2020-07-02 09:19:45 -04:00
try {
stream = await S3Persistor . getObjectStream ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'does not return a stream' , function ( ) {
expect ( stream ) . not . to . exist
} )
2024-11-08 03:32:58 -05:00
it ( 'throws a NotFoundError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . NotFoundError )
} )
it ( 'wraps the error' , function ( ) {
expect ( error . cause ) . to . exist
} )
it ( 'stores the bucket and key in the error' , function ( ) {
expect ( error . info ) . to . include ( { bucketName : bucket , key } )
} )
} )
describe ( "when the file doesn't exist -- SSEC" , function ( ) {
let error , stream
beforeEach ( async function ( ) {
S3GetObjectRequest . notFoundSSEC = 404
try {
stream = await S3Persistor . getObjectStream ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'does not return a stream' , function ( ) {
expect ( stream ) . not . to . exist
} )
2020-07-02 09:19:45 -04:00
it ( 'throws a NotFoundError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . NotFoundError )
} )
it ( 'wraps the error' , function ( ) {
expect ( error . cause ) . to . exist
} )
it ( 'stores the bucket and key in the error' , function ( ) {
2022-05-16 10:25:49 -04:00
expect ( error . info ) . to . include ( { bucketName : bucket , key } )
2020-07-02 09:19:45 -04:00
} )
} )
describe ( 'when access to the file is denied' , function ( ) {
let error , stream
beforeEach ( async function ( ) {
2023-05-23 03:07:06 -04:00
S3GetObjectRequest . statusCode = 403
2020-07-02 09:19:45 -04:00
try {
stream = await S3Persistor . getObjectStream ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'does not return a stream' , function ( ) {
expect ( stream ) . not . to . exist
} )
it ( 'throws a NotFoundError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . NotFoundError )
} )
it ( 'wraps the error' , function ( ) {
2024-11-08 03:31:44 -05:00
expect ( error . cause ) . to . equal ( S3AccessDeniedError )
2020-07-02 09:19:45 -04:00
} )
it ( 'stores the bucket and key in the error' , function ( ) {
2022-05-16 10:25:49 -04:00
expect ( error . info ) . to . include ( { bucketName : bucket , key } )
2020-07-02 09:19:45 -04:00
} )
} )
2023-05-23 03:07:06 -04:00
describe ( 'when S3 encounters an unknown error' , function ( ) {
2020-07-02 09:19:45 -04:00
let error , stream
beforeEach ( async function ( ) {
2023-05-23 03:07:06 -04:00
S3GetObjectRequest . err = genericError
2020-07-02 09:19:45 -04:00
try {
stream = await S3Persistor . getObjectStream ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'does not return a stream' , function ( ) {
expect ( stream ) . not . to . exist
} )
it ( 'throws a ReadError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'wraps the error' , function ( ) {
expect ( error . cause ) . to . exist
} )
it ( 'stores the bucket and key in the error' , function ( ) {
2022-05-16 10:25:49 -04:00
expect ( error . info ) . to . include ( { bucketName : bucket , key } )
2020-07-02 09:19:45 -04:00
} )
} )
} )
2020-07-08 16:56:23 -04:00
describe ( 'getRedirectUrl' , function ( ) {
let signedUrl
beforeEach ( async function ( ) {
signedUrl = await S3Persistor . getRedirectUrl ( bucket , key )
} )
it ( 'should request a signed URL' , function ( ) {
expect ( S3Client . getSignedUrlPromise ) . to . have . been . called
} )
it ( 'should return the url' , function ( ) {
expect ( signedUrl ) . to . equal ( redirectUrl )
} )
} )
2020-07-02 09:19:45 -04:00
describe ( 'getObjectSize' , function ( ) {
describe ( 'when called with valid parameters' , function ( ) {
let size
beforeEach ( async function ( ) {
size = await S3Persistor . getObjectSize ( bucket , key )
} )
it ( 'should return the object size' , function ( ) {
expect ( size ) . to . equal ( objectSize )
} )
it ( 'should pass the bucket and key to S3' , function ( ) {
expect ( S3Client . headObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
describe ( 'when the object is not found' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . headObject = sinon . stub ( ) . returns ( {
2021-12-16 04:04:32 -05:00
promise : sinon . stub ( ) . rejects ( S3NotFoundError ) ,
2020-07-02 09:19:45 -04:00
} )
try {
await S3Persistor . getObjectSize ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should return a NotFoundError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . NotFoundError )
} )
it ( 'should wrap the error' , function ( ) {
expect ( error . cause ) . to . equal ( S3NotFoundError )
} )
} )
describe ( 'when S3 returns an error' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . headObject = sinon . stub ( ) . returns ( {
2021-12-16 04:04:32 -05:00
promise : sinon . stub ( ) . rejects ( genericError ) ,
2020-07-02 09:19:45 -04:00
} )
try {
await S3Persistor . getObjectSize ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should return a ReadError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'should wrap the error' , function ( ) {
expect ( error . cause ) . to . equal ( genericError )
} )
} )
} )
describe ( 'sendStream' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
beforeEach ( async function ( ) {
return S3Persistor . sendStream ( bucket , key , ReadStream )
} )
it ( 'should upload the stream' , function ( ) {
expect ( S3Client . upload ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : key ,
2021-12-16 04:04:32 -05:00
Body : sinon . match . instanceOf ( Stream . Transform ) ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should upload files in a single part' , function ( ) {
expect ( S3Client . upload ) . to . have . been . calledWith ( sinon . match . any , {
2021-12-16 04:04:32 -05:00
partSize : 100 * 1024 * 1024 ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should meter the stream' , function ( ) {
2023-05-23 03:07:06 -04:00
expect ( Stream . pipeline ) . to . have . been . calledWith (
ReadStream ,
2020-07-02 09:19:45 -04:00
sinon . match . instanceOf ( Stream . Transform )
)
} )
} )
2020-07-07 10:17:14 -04:00
describe ( 'when a hash is supplied' , function ( ) {
2020-07-02 09:19:45 -04:00
beforeEach ( async function ( ) {
2020-07-07 10:17:14 -04:00
return S3Persistor . sendStream ( bucket , key , ReadStream , {
2021-12-16 04:04:32 -05:00
sourceMd5 : 'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb' ,
2020-07-07 10:17:14 -04:00
} )
2020-07-02 09:19:45 -04:00
} )
it ( 'sends the hash in base64' , function ( ) {
expect ( S3Client . upload ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : key ,
Body : sinon . match . instanceOf ( Transform ) ,
2021-12-16 04:04:32 -05:00
ContentMD5 : 'qqqqqru7u7uqqqqqu7u7uw==' ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
2020-07-07 10:17:14 -04:00
describe ( 'when metadata is supplied' , function ( ) {
const contentType = 'text/csv'
const contentEncoding = 'gzip'
beforeEach ( async function ( ) {
return S3Persistor . sendStream ( bucket , key , ReadStream , {
contentType ,
2021-12-16 04:04:32 -05:00
contentEncoding ,
2020-07-07 10:17:14 -04:00
} )
} )
it ( 'sends the metadata to S3' , function ( ) {
expect ( S3Client . upload ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : key ,
Body : sinon . match . instanceOf ( Transform ) ,
ContentType : contentType ,
2021-12-16 04:04:32 -05:00
ContentEncoding : contentEncoding ,
2020-07-07 10:17:14 -04:00
} )
} )
} )
2020-07-02 09:19:45 -04:00
describe ( 'when the upload fails' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . upload = sinon . stub ( ) . returns ( {
2021-12-16 04:04:32 -05:00
promise : sinon . stub ( ) . rejects ( genericError ) ,
2020-07-02 09:19:45 -04:00
} )
try {
await S3Persistor . sendStream ( bucket , key , ReadStream )
} catch ( err ) {
error = err
}
} )
it ( 'throws a WriteError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . WriteError )
} )
} )
} )
describe ( 'sendFile' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
beforeEach ( async function ( ) {
return S3Persistor . sendFile ( bucket , key , filename )
} )
it ( 'should create a read stream for the file' , function ( ) {
expect ( Fs . createReadStream ) . to . have . been . calledWith ( filename )
} )
it ( 'should upload the stream' , function ( ) {
expect ( S3Client . upload ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : key ,
2021-12-16 04:04:32 -05:00
Body : sinon . match . instanceOf ( Transform ) ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
} )
describe ( 'getObjectMd5Hash' , function ( ) {
describe ( 'when the etag is a valid md5 hash' , function ( ) {
let hash
beforeEach ( async function ( ) {
hash = await S3Persistor . getObjectMd5Hash ( bucket , key )
} )
it ( 'should return the object hash' , function ( ) {
expect ( hash ) . to . equal ( md5 )
} )
it ( 'should get the hash from the object metadata' , function ( ) {
expect ( S3Client . headObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should not download the object' , function ( ) {
expect ( S3Client . getObject ) . not . to . have . been . called
} )
} )
describe ( "when the etag isn't a valid md5 hash" , function ( ) {
let hash
beforeEach ( async function ( ) {
S3Client . headObject = sinon . stub ( ) . returns ( {
promise : sinon . stub ( ) . resolves ( {
ETag : 'somethingthatisntanmd5' ,
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
} ) ,
2020-07-02 09:19:45 -04:00
} )
hash = await S3Persistor . getObjectMd5Hash ( bucket , key )
} )
it ( 'should re-fetch the file to verify it' , function ( ) {
expect ( S3Client . getObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should calculate the md5 hash from the file' , function ( ) {
expect ( Hash . read ) . to . have . been . called
} )
it ( 'should return the md5 hash' , function ( ) {
expect ( hash ) . to . equal ( md5 )
} )
} )
} )
describe ( 'copyObject' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
beforeEach ( async function ( ) {
return S3Persistor . copyObject ( bucket , key , destKey )
} )
it ( 'should copy the object' , function ( ) {
expect ( S3Client . copyObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
Key : destKey ,
2021-12-16 04:04:32 -05:00
CopySource : ` ${ bucket } / ${ key } ` ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
describe ( 'when the file does not exist' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . copyObject = sinon . stub ( ) . returns ( {
2021-12-16 04:04:32 -05:00
promise : sinon . stub ( ) . rejects ( S3NotFoundError ) ,
2020-07-02 09:19:45 -04:00
} )
try {
await S3Persistor . copyObject ( bucket , key , destKey )
} catch ( err ) {
error = err
}
} )
it ( 'should throw a NotFoundError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . NotFoundError )
} )
} )
} )
describe ( 'deleteObject' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
beforeEach ( async function ( ) {
return S3Persistor . deleteObject ( bucket , key )
} )
it ( 'should delete the object' , function ( ) {
expect ( S3Client . deleteObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
} )
describe ( 'deleteDirectory' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
beforeEach ( async function ( ) {
return S3Persistor . deleteDirectory ( bucket , key )
} )
it ( 'should list the objects in the directory' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should delete the objects using their keys' , function ( ) {
expect ( S3Client . deleteObjects ) . to . have . been . calledWith ( {
Bucket : bucket ,
Delete : {
Objects : [ { Key : 'llama' } , { Key : 'hippo' } ] ,
2021-12-16 04:04:32 -05:00
Quiet : true ,
} ,
2020-07-02 09:19:45 -04:00
} )
} )
} )
describe ( 'when there are no files' , function ( ) {
beforeEach ( async function ( ) {
S3Client . listObjectsV2 = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . resolves ( { Contents : [ ] } ) } )
return S3Persistor . deleteDirectory ( bucket , key )
} )
it ( 'should list the objects in the directory' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should not try to delete any objects' , function ( ) {
expect ( S3Client . deleteObjects ) . not . to . have . been . called
} )
} )
describe ( 'when there are more files available' , function ( ) {
const continuationToken = 'wombat'
beforeEach ( async function ( ) {
S3Client . listObjectsV2 . onCall ( 0 ) . returns ( {
promise : sinon . stub ( ) . resolves ( {
Contents : files ,
IsTruncated : true ,
2021-12-16 04:04:32 -05:00
NextContinuationToken : continuationToken ,
} ) ,
2020-07-02 09:19:45 -04:00
} )
return S3Persistor . deleteDirectory ( bucket , key )
} )
it ( 'should list the objects a second time, with a continuation token' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . be . calledTwice
expect ( S3Client . listObjectsV2 ) . to . be . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
expect ( S3Client . listObjectsV2 ) . to . be . calledWith ( {
Bucket : bucket ,
Prefix : key ,
2021-12-16 04:04:32 -05:00
ContinuationToken : continuationToken ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should delete both sets of files' , function ( ) {
expect ( S3Client . deleteObjects ) . to . have . been . calledTwice
} )
} )
describe ( 'when there is an error listing the objects' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . listObjectsV2 = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . rejects ( genericError ) } )
try {
await S3Persistor . deleteDirectory ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should generate a ReadError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'should wrap the error' , function ( ) {
expect ( error . cause ) . to . equal ( genericError )
} )
it ( 'should not try to delete any objects' , function ( ) {
expect ( S3Client . deleteObjects ) . not . to . have . been . called
} )
} )
describe ( 'when there is an error deleting the objects' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . deleteObjects = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . rejects ( genericError ) } )
try {
await S3Persistor . deleteDirectory ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should generate a WriteError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . WriteError )
} )
it ( 'should wrap the error' , function ( ) {
expect ( error . cause ) . to . equal ( genericError )
} )
} )
} )
describe ( 'directorySize' , function ( ) {
describe ( 'with valid parameters' , function ( ) {
let size
beforeEach ( async function ( ) {
size = await S3Persistor . directorySize ( bucket , key )
} )
it ( 'should list the objects in the directory' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should return the directory size' , function ( ) {
expect ( size ) . to . equal ( filesSize )
} )
} )
describe ( 'when there are no files' , function ( ) {
let size
beforeEach ( async function ( ) {
S3Client . listObjectsV2 = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . resolves ( { Contents : [ ] } ) } )
size = await S3Persistor . directorySize ( bucket , key )
} )
it ( 'should list the objects in the directory' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should return zero' , function ( ) {
expect ( size ) . to . equal ( 0 )
} )
} )
describe ( 'when there are more files available' , function ( ) {
const continuationToken = 'wombat'
let size
beforeEach ( async function ( ) {
S3Client . listObjectsV2 . onCall ( 0 ) . returns ( {
promise : sinon . stub ( ) . resolves ( {
Contents : files ,
IsTruncated : true ,
2021-12-16 04:04:32 -05:00
NextContinuationToken : continuationToken ,
} ) ,
2020-07-02 09:19:45 -04:00
} )
size = await S3Persistor . directorySize ( bucket , key )
} )
it ( 'should list the objects a second time, with a continuation token' , function ( ) {
expect ( S3Client . listObjectsV2 ) . to . be . calledTwice
expect ( S3Client . listObjectsV2 ) . to . be . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Prefix : key ,
2020-07-02 09:19:45 -04:00
} )
expect ( S3Client . listObjectsV2 ) . to . be . calledWith ( {
Bucket : bucket ,
Prefix : key ,
2021-12-16 04:04:32 -05:00
ContinuationToken : continuationToken ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should return the size of both sets of files' , function ( ) {
expect ( size ) . to . equal ( filesSize * 2 )
} )
} )
describe ( 'when there is an error listing the objects' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . listObjectsV2 = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . rejects ( genericError ) } )
try {
await S3Persistor . directorySize ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should generate a ReadError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'should wrap the error' , function ( ) {
expect ( error . cause ) . to . equal ( genericError )
} )
} )
} )
describe ( 'checkIfObjectExists' , function ( ) {
describe ( 'when the file exists' , function ( ) {
let exists
beforeEach ( async function ( ) {
exists = await S3Persistor . checkIfObjectExists ( bucket , key )
} )
it ( 'should get the object header' , function ( ) {
expect ( S3Client . headObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should return that the file exists' , function ( ) {
expect ( exists ) . to . equal ( true )
} )
} )
describe ( 'when the file does not exist' , function ( ) {
let exists
beforeEach ( async function ( ) {
S3Client . headObject = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . rejects ( S3NotFoundError ) } )
exists = await S3Persistor . checkIfObjectExists ( bucket , key )
} )
it ( 'should get the object header' , function ( ) {
expect ( S3Client . headObject ) . to . have . been . calledWith ( {
Bucket : bucket ,
2021-12-16 04:04:32 -05:00
Key : key ,
2020-07-02 09:19:45 -04:00
} )
} )
it ( 'should return that the file does not exist' , function ( ) {
expect ( exists ) . to . equal ( false )
} )
} )
describe ( 'when there is an error' , function ( ) {
let error
beforeEach ( async function ( ) {
S3Client . headObject = sinon
. stub ( )
. returns ( { promise : sinon . stub ( ) . rejects ( genericError ) } )
try {
await S3Persistor . checkIfObjectExists ( bucket , key )
} catch ( err ) {
error = err
}
} )
it ( 'should generate a ReadError' , function ( ) {
expect ( error ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'should wrap the upstream ReadError' , function ( ) {
expect ( error . cause ) . to . be . an . instanceOf ( Errors . ReadError )
} )
it ( 'should eventually wrap the error' , function ( ) {
expect ( error . cause . cause ) . to . equal ( genericError )
} )
} )
} )
} )