mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-23 02:55:13 -04:00
7d900b57bf
In 49a21155f642670dfea264ac73fb60241f37cb87, I managed to incorrectly write the `createUnbufferedStream()` function from the AWS SDK as `getUnbufferedStream()` and to consistently use that naming in the unit tests. This commit fixes that. I have tested again on S3.
342 lines
11 KiB
CoffeeScript
342 lines
11 KiB
CoffeeScript
assert = require("chai").assert
|
|
sinon = require('sinon')
|
|
chai = require('chai')
|
|
should = chai.should()
|
|
expect = chai.expect
|
|
modulePath = "../../../app/js/S3PersistorManager.js"
|
|
SandboxedModule = require('sandboxed-module')
|
|
|
|
describe "S3PersistorManagerTests", ->
|
|
|
|
beforeEach ->
|
|
@settings =
|
|
filestore:
|
|
backend: "s3"
|
|
s3:
|
|
secret: "secret"
|
|
key: "this_key"
|
|
stores:
|
|
user_files:"sl_user_files"
|
|
@knoxClient =
|
|
putFile:sinon.stub()
|
|
copyFile:sinon.stub()
|
|
list: sinon.stub()
|
|
deleteMultiple: sinon.stub()
|
|
get: sinon.stub()
|
|
@knox =
|
|
createClient: sinon.stub().returns(@knoxClient)
|
|
@s3EventHandlers = {}
|
|
@s3Request =
|
|
on: sinon.stub().callsFake (event, callback) =>
|
|
@s3EventHandlers[event] = callback
|
|
send: sinon.stub()
|
|
@s3Response =
|
|
httpResponse:
|
|
createUnbufferedStream: sinon.stub()
|
|
@s3Client =
|
|
copyObject: sinon.stub()
|
|
headObject: sinon.stub()
|
|
getObject: sinon.stub().returns(@s3Request)
|
|
@awsS3 = sinon.stub().returns(@s3Client)
|
|
@LocalFileWriter =
|
|
writeStream: sinon.stub()
|
|
deleteFile: sinon.stub()
|
|
@request = sinon.stub()
|
|
@requires =
|
|
"knox": @knox
|
|
"aws-sdk/clients/s3": @awsS3
|
|
"settings-sharelatex": @settings
|
|
"./LocalFileWriter":@LocalFileWriter
|
|
"logger-sharelatex":
|
|
log:->
|
|
err:->
|
|
"request": @request
|
|
"./Errors": @Errors =
|
|
NotFoundError: sinon.stub()
|
|
@key = "my/key"
|
|
@bucketName = "my-bucket"
|
|
@error = "my errror"
|
|
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
|
|
|
|
describe "getFileStream", ->
|
|
describe "success", ->
|
|
beforeEach () ->
|
|
@expectedStream = { expectedStream: true }
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK")
|
|
@s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream)
|
|
|
|
it "returns a stream", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
it "sets the AWS client up with credentials from settings", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@awsS3.lastCall.args).to.deep.equal([{
|
|
credentials:
|
|
accessKeyId: @settings.filestore.s3.key
|
|
secretAccessKey: @settings.filestore.s3.secret
|
|
}])
|
|
done()
|
|
|
|
it "fetches the right key from the right bucket", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
|
|
Bucket: @bucketName,
|
|
Key: @key
|
|
}])
|
|
done()
|
|
|
|
it "accepts alternative credentials", (done) ->
|
|
accessKeyId = "that_key"
|
|
secret = "that_secret"
|
|
opts = {
|
|
credentials:
|
|
auth_key: accessKeyId
|
|
auth_secret: secret
|
|
}
|
|
@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@awsS3.lastCall.args).to.deep.equal([{
|
|
credentials:
|
|
accessKeyId: accessKeyId
|
|
secretAccessKey: secret
|
|
}])
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
it "accepts byte range", (done) ->
|
|
start = 0
|
|
end = 8
|
|
opts = { start: start, end: end }
|
|
@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
|
|
if err?
|
|
return done(err)
|
|
expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
|
|
Bucket: @bucketName
|
|
Key: @key
|
|
Range: "bytes=#{start}-#{end}"
|
|
}])
|
|
expect(stream).to.equal(@expectedStream)
|
|
done()
|
|
|
|
describe "errors", ->
|
|
describe "when the file doesn't exist", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found")
|
|
|
|
it "returns a NotFoundError that indicates the bucket and key", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(@Errors.NotFoundError)
|
|
errMsg = @Errors.NotFoundError.lastCall.args[0]
|
|
expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*"))
|
|
expect(errMsg).to.match(new RegExp(".*#{@key}.*"))
|
|
done()
|
|
|
|
describe "when S3 encounters an unkown error", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error")
|
|
|
|
it "returns an error", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(Error)
|
|
done()
|
|
|
|
describe "when the S3 request errors out before receiving HTTP headers", ->
|
|
beforeEach ->
|
|
@s3Request.send.callsFake () =>
|
|
@s3EventHandlers.error(new Error("connection failed"))
|
|
|
|
it "returns an error", (done) ->
|
|
@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
|
|
expect(err).to.be.instanceof(Error)
|
|
done()
|
|
|
|
describe "getFileSize", ->
|
|
it "should obtain the file size from S3", (done) ->
|
|
expectedFileSize = 123
|
|
@s3Client.headObject.yields(new Error(
|
|
"s3Client.headObject got unexpected arguments"
|
|
))
|
|
@s3Client.headObject.withArgs({
|
|
Bucket: @bucketName
|
|
Key: @key
|
|
}).yields(null, { ContentLength: expectedFileSize })
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
if err?
|
|
return done(err)
|
|
expect(fileSize).to.equal(expectedFileSize)
|
|
done()
|
|
|
|
[403, 404].forEach (statusCode) ->
|
|
it "should throw NotFoundError when S3 responds with #{statusCode}", (done) ->
|
|
error = new Error()
|
|
error.statusCode = statusCode
|
|
@s3Client.headObject.yields(error)
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
expect(err).to.be.an.instanceof(@Errors.NotFoundError)
|
|
done()
|
|
|
|
it "should rethrow any other error", (done) ->
|
|
error = new Error()
|
|
@s3Client.headObject.yields(error)
|
|
@s3Client.headObject.yields(error)
|
|
|
|
@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
|
|
expect(err).to.equal(error)
|
|
done()
|
|
|
|
describe "sendFile", ->
|
|
|
|
beforeEach ->
|
|
@knoxClient.putFile.returns on:->
|
|
|
|
it "should put file with knox", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@knoxClient.putFile.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
|
|
@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
|
|
err.should.equal @error
|
|
done()
|
|
|
|
it "should delete the file and pass the error with it", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@knoxClient.putFile.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
|
|
@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "sendStream", ->
|
|
beforeEach ->
|
|
@fsPath = "to/some/where"
|
|
@origin =
|
|
on:->
|
|
@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
|
|
|
|
it "should send stream to LocalFileWriter", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
|
|
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
|
|
done()
|
|
|
|
it "should return the error from LocalFileWriter", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2, @error)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
it "should send the file to the filestore", (done)->
|
|
@LocalFileWriter.deleteFile.callsArgWith(1)
|
|
@LocalFileWriter.writeStream.callsArgWith(2)
|
|
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
|
|
@S3PersistorManager.sendFile.called.should.equal true
|
|
done()
|
|
|
|
describe "copyFile", ->
|
|
beforeEach ->
|
|
@sourceKey = "my/key"
|
|
@destKey = "my/dest/key"
|
|
|
|
it "should use AWS SDK to copy file", (done)->
|
|
@s3Client.copyObject.callsArgWith(1, @error)
|
|
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
|
|
err.should.equal @error
|
|
@s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true
|
|
done()
|
|
|
|
it "should return a NotFoundError object if the original file does not exist", (done)->
|
|
NoSuchKeyError = {code: "NoSuchKey"}
|
|
@s3Client.copyObject.callsArgWith(1, NoSuchKeyError)
|
|
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
|
|
expect(err instanceof @Errors.NotFoundError).to.equal true
|
|
done()
|
|
|
|
describe "deleteDirectory", ->
|
|
|
|
it "should list the contents passing them onto multi delete", (done)->
|
|
data =
|
|
Contents: [{Key:"1234"}, {Key: "456"}]
|
|
@knoxClient.list.callsArgWith(1, null, data)
|
|
@knoxClient.deleteMultiple.callsArgWith(1)
|
|
@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
|
|
@knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
|
|
done()
|
|
|
|
describe "deleteFile", ->
|
|
|
|
it "should use correct options", (done)->
|
|
@request.callsArgWith(1)
|
|
|
|
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
|
|
opts = @request.args[0][0]
|
|
assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
|
|
opts.method.should.equal "delete"
|
|
opts.timeout.should.equal (30*1000)
|
|
opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
|
|
done()
|
|
|
|
it "should return the error", (done)->
|
|
@request.callsArgWith(1, @error)
|
|
|
|
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "checkIfFileExists", ->
|
|
|
|
it "should use correct options", (done)->
|
|
@request.callsArgWith(1, null, statusCode:200)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
|
|
opts = @request.args[0][0]
|
|
assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
|
|
opts.method.should.equal "head"
|
|
opts.timeout.should.equal (30*1000)
|
|
opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
|
|
done()
|
|
|
|
it "should return true for a 200", (done)->
|
|
@request.callsArgWith(1, null, statusCode:200)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
|
|
exists.should.equal true
|
|
done()
|
|
|
|
it "should return false for a non 200", (done)->
|
|
@request.callsArgWith(1, null, statusCode:404)
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
|
|
exists.should.equal false
|
|
done()
|
|
|
|
it "should return the error", (done)->
|
|
@request.callsArgWith(1, @error, {})
|
|
|
|
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
|
|
err.should.equal @error
|
|
done()
|
|
|
|
describe "directorySize", ->
|
|
|
|
it "should sum directory files size", (done) ->
|
|
data =
|
|
Contents: [ {Size: 1024}, {Size: 2048} ]
|
|
@knoxClient.list.callsArgWith(1, null, data)
|
|
@S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=>
|
|
totalSize.should.equal 3072
|
|
done()
|