overleaf/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee

assert = require("chai").assert
sinon = require('sinon')
chai = require('chai')
should = chai.should()
expect = chai.expect
modulePath = "../../../app/js/S3PersistorManager.js"
SandboxedModule = require('sandboxed-module')

describe "S3PersistorManagerTests", ->

	beforeEach ->
		@settings =
			filestore:
				backend: "s3"
				s3:
					secret: "secret"
					key: "this_key"
				stores:
					user_files:"sl_user_files"
		@knoxClient =
			putFile:sinon.stub()
			copyFile:sinon.stub()
			list: sinon.stub()
			deleteMultiple: sinon.stub()
			get: sinon.stub()
		@knox =
			createClient: sinon.stub().returns(@knoxClient)
		@s3EventHandlers = {}
		@s3Request =
			on: sinon.stub().callsFake (event, callback) =>
				@s3EventHandlers[event] = callback
			send: sinon.stub()
		@s3Response =
			httpResponse:
				createUnbufferedStream: sinon.stub()
		@s3Client =
			copyObject: sinon.stub()
			headObject: sinon.stub()
			getObject: sinon.stub().returns(@s3Request)
		@awsS3 = sinon.stub().returns(@s3Client)
		@LocalFileWriter =
			writeStream: sinon.stub()
			deleteFile: sinon.stub()
		@request = sinon.stub()
		@requires =
			"knox": @knox
			"aws-sdk/clients/s3": @awsS3
			"settings-sharelatex": @settings
			"./LocalFileWriter":@LocalFileWriter
			"logger-sharelatex":
				log:->
				err:->
			"request": @request
			"./Errors": @Errors =
				NotFoundError: sinon.stub()
		@key = "my/key"
		@bucketName = "my-bucket"
		@error = "my errror"
		@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires

	describe "getFileStream", ->
		describe "success", ->
			beforeEach () ->
				@expectedStream = { expectedStream: true }
				@expectedStream.on = sinon.stub()
				@s3Request.send.callsFake () =>
					@s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK")
				@s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream)

			it "returns a stream", (done) ->
				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
					if err?
						return done(err)
					expect(stream).to.equal(@expectedStream)
					done()

			it "sets the AWS client up with credentials from settings", (done) ->
				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
					if err?
						return done(err)
					expect(@awsS3.lastCall.args).to.deep.equal([{
						credentials:
							accessKeyId: @settings.filestore.s3.key
							secretAccessKey: @settings.filestore.s3.secret
					}])
					done()

			it "fetches the right key from the right bucket", (done) ->
				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
					if err?
						return done(err)
					expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
						Bucket: @bucketName,
						Key: @key
					}])
					done()

			it "accepts alternative credentials", (done) ->
				accessKeyId = "that_key"
				secret = "that_secret"
				opts = {
					credentials:
						auth_key: accessKeyId
						auth_secret: secret
				}
				@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
					if err?
						return done(err)
					expect(@awsS3.lastCall.args).to.deep.equal([{
						credentials:
							accessKeyId: accessKeyId
							secretAccessKey: secret
					}])
					expect(stream).to.equal(@expectedStream)
					done()

			it "accepts byte range", (done) ->
				start = 0
				end = 8
				opts = { start: start, end: end }
				@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
					if err?
						return done(err)
					expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
						Bucket: @bucketName
						Key: @key
						Range: "bytes=#{start}-#{end}"
					}])
					expect(stream).to.equal(@expectedStream)
					done()

		describe "errors", ->
			describe "when the file doesn't exist", ->
				beforeEach ->
					@s3Request.send.callsFake () =>
						@s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found")

				it "returns a NotFoundError that indicates the bucket and key", (done) ->
					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
						expect(err).to.be.instanceof(@Errors.NotFoundError)
						errMsg = @Errors.NotFoundError.lastCall.args[0]
						expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*"))
						expect(errMsg).to.match(new RegExp(".*#{@key}.*"))
						done()

			describe "when S3 encounters an unkown error", ->
				beforeEach ->
					@s3Request.send.callsFake () =>
						@s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error")

				it "returns an error", (done) ->
					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
						expect(err).to.be.instanceof(Error)
						done()

			describe "when the S3 request errors out before receiving HTTP headers", ->
				beforeEach ->
					@s3Request.send.callsFake () =>
						@s3EventHandlers.error(new Error("connection failed"))

				it "returns an error", (done) ->
					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
						expect(err).to.be.instanceof(Error)
						done()

	describe "getFileSize", ->
		it "should obtain the file size from S3", (done) ->
			expectedFileSize = 123
			@s3Client.headObject.yields(new Error(
				"s3Client.headObject got unexpected arguments"
			))
			@s3Client.headObject.withArgs({
				Bucket: @bucketName
				Key: @key
			}).yields(null, { ContentLength: expectedFileSize })

			@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
				if err?
					return done(err)
				expect(fileSize).to.equal(expectedFileSize)
				done()

		[403, 404].forEach (statusCode) ->
			it "should throw NotFoundError when S3 responds with #{statusCode}", (done) ->
				error = new Error()
				error.statusCode = statusCode
				@s3Client.headObject.yields(error)

				@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
					expect(err).to.be.an.instanceof(@Errors.NotFoundError)
					done()

		it "should rethrow any other error", (done) ->
			error = new Error()
			@s3Client.headObject.yields(error)
			@s3Client.headObject.yields(error)

			@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
				expect(err).to.equal(error)
				done()

	describe "sendFile", ->

		beforeEach ->
			@knoxClient.putFile.returns on:->

		it "should put file with knox", (done)->
			@LocalFileWriter.deleteFile.callsArgWith(1)
			@knoxClient.putFile.callsArgWith(2, @error)
			@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
				@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
				err.should.equal @error
				done()

		it "should delete the file and pass the error with it", (done)->
			@LocalFileWriter.deleteFile.callsArgWith(1)
			@knoxClient.putFile.callsArgWith(2, @error)
			@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
				@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
				err.should.equal @error
				done()

	describe "sendStream", ->
		beforeEach ->
			@fsPath = "to/some/where"
			@origin =
				on:->
			@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)

		it "should send stream to LocalFileWriter", (done)->
			@LocalFileWriter.deleteFile.callsArgWith(1)
			@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
			@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
				@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
				done()

		it "should return the error from LocalFileWriter", (done)->
			@LocalFileWriter.deleteFile.callsArgWith(1)
			@LocalFileWriter.writeStream.callsArgWith(2, @error)
			@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
				err.should.equal @error
				done()

		it "should send the file to the filestore", (done)->
			@LocalFileWriter.deleteFile.callsArgWith(1)
			@LocalFileWriter.writeStream.callsArgWith(2)
			@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
				@S3PersistorManager.sendFile.called.should.equal true
				done()

	describe "copyFile", ->
		beforeEach ->
			@sourceKey = "my/key"
			@destKey = "my/dest/key"

		it "should use AWS SDK to copy file", (done)->
			@s3Client.copyObject.callsArgWith(1, @error)
			@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
				err.should.equal @error
				@s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true
				done()

		it "should return a NotFoundError object if the original file does not exist", (done)->
			NoSuchKeyError = {code: "NoSuchKey"}
			@s3Client.copyObject.callsArgWith(1, NoSuchKeyError)
			@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
				expect(err instanceof @Errors.NotFoundError).to.equal true
				done()

	describe "deleteDirectory", ->

		it "should list the contents passing them onto multi delete", (done)->
			data =
				Contents: [{Key:"1234"}, {Key: "456"}]
			@knoxClient.list.callsArgWith(1, null, data)
			@knoxClient.deleteMultiple.callsArgWith(1)
			@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
				@knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
				done()

	describe "deleteFile", ->

		it "should use correct options", (done)->
			@request.callsArgWith(1)

			@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
				opts = @request.args[0][0]
				assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
				opts.method.should.equal "delete"
				opts.timeout.should.equal (30*1000)
				opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
				done()

		it "should return the error", (done)->
			@request.callsArgWith(1, @error)

			@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
				err.should.equal @error
				done()

	describe "checkIfFileExists", ->

		it "should use correct options", (done)->
			@request.callsArgWith(1,  null, statusCode:200)

			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
				opts = @request.args[0][0]
				assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
				opts.method.should.equal "head"
				opts.timeout.should.equal (30*1000)
				opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
				done()

		it "should return true for a 200", (done)->
			@request.callsArgWith(1, null, statusCode:200)

			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
				exists.should.equal true
				done()

		it "should return false for a non 200", (done)->
			@request.callsArgWith(1, null, statusCode:404)

			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
				exists.should.equal false
				done()

		it "should return the error", (done)->
			@request.callsArgWith(1, @error, {})

			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
				err.should.equal @error
				done()

	describe "directorySize", ->

		it "should sum directory files size", (done) ->
			data =
				Contents: [ {Size: 1024}, {Size: 2048} ]
			@knoxClient.list.callsArgWith(1, null, data)
			@S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=>
				totalSize.should.equal 3072
				done()