mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
f2521a29b9
The AWS SDK has a retry strategy to deal with rate limiting or transient unavailability of S3. We hope it will reduce our error rates.
342 lines
11 KiB
CoffeeScript
342 lines
11 KiB
CoffeeScript
assert = require("chai").assert
|
|
sinon = require('sinon')
|
|
chai = require('chai')
|
|
should = chai.should()
|
|
expect = chai.expect
|
|
modulePath = "../../../app/js/S3PersistorManager.js"
|
|
SandboxedModule = require('sandboxed-module')
|
|
|
|
describe "S3PersistorManagerTests", ->
|
|
|
|
beforeEach ->
|
|
@settings =
|
|
filestore:
|
|
backend: "s3"
|
|
s3:
|
|
secret: "secret"
|
|
key: "this_key"
|
|
stores:
|
|
user_files:"sl_user_files"
|
|
@knoxClient =
|
|
putFile:sinon.stub()
|
|
copyFile:sinon.stub()
|
|
list: sinon.stub()
|
|
deleteMultiple: sinon.stub()
|
|
get: sinon.stub()
|
|
@knox =
|
|
createClient: sinon.stub().returns(@knoxClient)
|
|
@s3EventHandlers = {}
|
|
@s3Request =
|
|
on: sinon.stub().callsFake (event, callback) =>
|
|
@s3EventHandlers[event] = callback
|
|
send: sinon.stub()
|
|
@s3Response =
|
|
httpResponse:
|
|
getUnbufferedStream: sinon.stub()
|
|
@s3Client =
|
|
copyObject: sinon.stub()
|
|
headObject: sinon.stub()
|
|
getObject: sinon.stub().returns(@s3Request)
|
|
@awsS3 = sinon.stub().returns(@s3Client)
|
|
@LocalFileWriter =
|
|
writeStream: sinon.stub()
|
|
deleteFile: sinon.stub()
|
|
@request = sinon.stub()
|
|
@requires =
|
|
"knox": @knox
|
|
"aws-sdk/clients/s3": @awsS3
|
|
"settings-sharelatex": @settings
|
|
"./LocalFileWriter":@LocalFileWriter
|
|
"logger-sharelatex":
|
|
log:->
|
|
err:->
|
|
"request": @request
|
|
"./Errors": @Errors =
|
|
NotFoundError: sinon.stub()
|
|
@key = "my/key"
|
|
@bucketName = "my-bucket"
|
|
@error = "my errror"
|
|
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
|
|
|
|
describe "getFileStream", ->
|
|
describe "success", ->
|
|
beforeEach () ->
|
|
@expectedStream = { expectedStream: true }
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK")
|
|
@s3Response.httpResponse.getUnbufferedStream.returns(@expectedStream)
|
|
|
|
it "returns a stream", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
it "sets the AWS client up with credentials from settings", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@awsS3.lastCall.args).to.deep.equal([{
|
|
credentials:
|
|
accessKeyId: @settings.filestore.s3.key
|
|
secretAccessKey: @settings.filestore.s3.secret
|
|
}])
|
|
done()
|
|
|
|
it "fetches the right key from the right bucket", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
|
|
Bucket: @bucketName,
|
|
Key: @key
|
|
}])
|
|
done()
|
|
|
|
it "accepts alternative credentials", (done) ->
|
|
accessKeyId = "that_key"
|
|
secret = "that_secret"
|
|
opts = {
|
|
credentials:
|
|
auth_key: accessKeyId
|
|
auth_secret: secret
|
|
}
|
|
@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@awsS3.lastCall.args).to.deep.equal([{
|
|
credentials:
|
|
accessKeyId: accessKeyId
|
|
secretAccessKey: secret
|
|
}])
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
it "accepts byte range", (done) ->
|
|
start = 0
|
|
end = 8
|
|
opts = { start: start, end: end }
|
|
@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
|
|
Bucket: @bucketName
|
|
Key: @key
|
|
Range: "bytes=#{start}-#{end}"
|
|
}])
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
describe "errors", ->
|
|
describe "when the file doesn't exist", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found")
|
|
|
|
it "returns a NotFoundError that indicates the bucket and key", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(@Errors.NotFoundError)
|
|
errMsg = @Errors.NotFoundError.lastCall.args[0]
|
|
expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*"))
|
|
expect(errMsg).to.match(new RegExp(".*#{@key}.*"))
|
|
done()
|
|
|
|
describe "when S3 encounters an unkown error", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error")
|
|
|
|
it "returns an error", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(Error)
|
|
done()
|
|
|
|
describe "when the S3 request errors out before receiving HTTP headers", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.error(new Error("connection failed"))
|
|
|
|
it "returns an error", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(Error)
|
|
done()
|
|
|
|
describe "getFileSize", ->
|
|
it "should obtain the file size from S3", (done) ->
|
|
expectedFileSize = 123
|
|
@s3Client.headObject.yields(new Error(
|
|
"s3Client.headObject got unexpected arguments"
|
|
))
|
|
@s3Client.headObject.withArgs({
|
|
Bucket: @bucketName
|
|
Key: @key
|
|
}).yields(null, { ContentLength: expectedFileSize })
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
if err?
|
|
return done(err)
|
|
expect(fileSize).to.equal(expectedFileSize)
|
|
done()
|
|
|
|
[403, 404].forEach (statusCode) ->
|
|
it "should throw NotFoundError when S3 responds with #{statusCode}", (done) ->
|
|
error = new Error()
|
|
error.statusCode = statusCode
|
|
@s3Client.headObject.yields(error)
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
expect(err).to.be.an.instanceof(@Errors.NotFoundError)
|
|
done()
|
|
|
|
it "should rethrow any other error", (done) ->
|
|
error = new Error()
|
|
@s3Client.headObject.yields(error)
|
|
@s3Client.headObject.yields(error)
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
expect(err).to.equal(error)
|
|
done()
|
|
|
|
describe "sendFile", ->
|
|
|
|
beforeEach ->
|
|
@knoxClient.putFile.returns on:->
|
|
|
|
it "should put file with knox", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@knoxClient.putFile.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
|
|
@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
|
|
err.should.equal @error
|
|
done()
|
|
|
|
it "should delete the file and pass the error with it", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@knoxClient.putFile.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
|
|
@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "sendStream", ->
|
|
beforeEach ->
|
|
@fsPath = "to/some/where"
|
|
@origin =
|
|
on:->
|
|
@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
|
|
|
|
it "should send stream to LocalFileWriter", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
|
|
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
|
|
done()
|
|
|
|
it "should return the error from LocalFileWriter", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
it "should send the file to the filestore", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
|
|
@S3PersistorManager.sendFile.called.should.equal true
|
|
done()
|
|
|
|
describe "copyFile", ->
|
|
beforeEach ->
|
|
@sourceKey = "my/key"
|
|
@destKey = "my/dest/key"
|
|
|
|
it "should use AWS SDK to copy file", (done)->
|
|
@s3Client.copyObject.callsArgWith(1, @error)
|
|
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
|
|
err.should.equal @error
|
|
@s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true
|
|
done()
|
|
|
|
it "should return a NotFoundError object if the original file does not exist", (done)->
|
|
NoSuchKeyError = {code: "NoSuchKey"}
|
|
@s3Client.copyObject.callsArgWith(1, NoSuchKeyError)
|
|
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
|
|
expect(err instanceof @Errors.NotFoundError).to.equal true
|
|
done()
|
|
|
|
describe "deleteDirectory", ->
|
|
|
|
it "should list the contents passing them onto multi delete", (done)->
|
|
data =
|
|
Contents: [{Key:"1234"}, {Key: "456"}]
|
|
@knoxClient.list.callsArgWith(1, null, data)
|
|
@knoxClient.deleteMultiple.callsArgWith(1)
|
|
@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
|
|
@knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
|
|
done()
|
|
|
|
describe "deleteFile", ->
|
|
|
|
it "should use correct options", (done)->
|
|
@request.callsArgWith(1)
|
|
|
|
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
|
|
opts = @request.args[0][0]
|
|
assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
|
|
opts.method.should.equal "delete"
|
|
opts.timeout.should.equal (30*1000)
|
|
opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
|
|
done()
|
|
|
|
it "should return the error", (done)->
|
|
@request.callsArgWith(1, @error)
|
|
|
|
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "checkIfFileExists", ->
|
|
|
|
it "should use correct options", (done)->
|
|
@request.callsArgWith(1, null, statusCode:200)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
|
|
opts = @request.args[0][0]
|
|
assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
|
|
opts.method.should.equal "head"
|
|
opts.timeout.should.equal (30*1000)
|
|
opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
|
|
done()
|
|
|
|
it "should return true for a 200", (done)->
|
|
@request.callsArgWith(1, null, statusCode:200)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
|
|
exists.should.equal true
|
|
done()
|
|
|
|
it "should return false for a non 200", (done)->
|
|
@request.callsArgWith(1, null, statusCode:404)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
|
|
exists.should.equal false
|
|
done()
|
|
|
|
it "should return the error", (done)->
|
|
@request.callsArgWith(1, @error, {})
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "directorySize", ->
|
|
|
|
it "should sum directory files size", (done) ->
|
|
data =
|
|
Contents: [ {Size: 1024}, {Size: 2048} ]
|
|
@knoxClient.list.callsArgWith(1, null, data)
|
|
@S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=>
|
|
totalSize.should.equal 3072
|
|
done()
|