overleaf/services/filestore/test/unit/js/S3PersistorManagerTests.js

/* eslint-disable
    handle-callback-err,
    no-return-assign,
    no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
 * decaffeinate suggestions:
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
const { assert } = require('chai')
const sinon = require('sinon')
const chai = require('chai')
const should = chai.should()
const { expect } = chai
const modulePath = '../../../app/js/S3PersistorManager.js'
const SandboxedModule = require('sandboxed-module')

describe('S3PersistorManagerTests', function() {
  beforeEach(function() {
    this.settings = {
      filestore: {
        backend: 's3',
        s3: {
          secret: 'secret',
          key: 'this_key'
        },
        stores: {
          user_files: 'sl_user_files'
        }
      }
    }
    this.knoxClient = {
      putFile: sinon.stub(),
      copyFile: sinon.stub(),
      list: sinon.stub(),
      deleteMultiple: sinon.stub(),
      get: sinon.stub()
    }
    this.knox = { createClient: sinon.stub().returns(this.knoxClient) }
    this.s3EventHandlers = {}
    this.s3Request = {
      on: sinon.stub().callsFake((event, callback) => {
        return (this.s3EventHandlers[event] = callback)
      }),
      send: sinon.stub()
    }
    this.s3Response = {
      httpResponse: {
        createUnbufferedStream: sinon.stub()
      }
    }
    this.s3Client = {
      copyObject: sinon.stub(),
      headObject: sinon.stub(),
      getObject: sinon.stub().returns(this.s3Request)
    }
    this.awsS3 = sinon.stub().returns(this.s3Client)
    this.LocalFileWriter = {
      writeStream: sinon.stub(),
      deleteFile: sinon.stub()
    }
    this.request = sinon.stub()
    this.requires = {
      knox: this.knox,
      'aws-sdk/clients/s3': this.awsS3,
      'settings-sharelatex': this.settings,
      './LocalFileWriter': this.LocalFileWriter,
      'logger-sharelatex': {
        log() {},
        err() {}
      },
      request: this.request,
      './Errors': (this.Errors = { NotFoundError: sinon.stub() })
    }
    this.key = 'my/key'
    this.bucketName = 'my-bucket'
    this.error = 'my errror'
    return (this.S3PersistorManager = SandboxedModule.require(modulePath, {
      requires: this.requires
    }))
  })

  describe('getFileStream', function() {
    describe('success', function() {
      beforeEach(function() {
        this.expectedStream = { expectedStream: true }
        this.expectedStream.on = sinon.stub()
        this.s3Request.send.callsFake(() => {
          return this.s3EventHandlers.httpHeaders(
            200,
            {},
            this.s3Response,
            'OK'
          )
        })
        return this.s3Response.httpResponse.createUnbufferedStream.returns(
          this.expectedStream
        )
      })

      it('returns a stream', function(done) {
        return this.S3PersistorManager.getFileStream(
          this.bucketName,
          this.key,
          {},
          (err, stream) => {
            if (err != null) {
              return done(err)
            }
            expect(stream).to.equal(this.expectedStream)
            return done()
          }
        )
      })

      it('sets the AWS client up with credentials from settings', function(done) {
        return this.S3PersistorManager.getFileStream(
          this.bucketName,
          this.key,
          {},
          (err, stream) => {
            if (err != null) {
              return done(err)
            }
            expect(this.awsS3.lastCall.args).to.deep.equal([
              {
                credentials: {
                  accessKeyId: this.settings.filestore.s3.key,
                  secretAccessKey: this.settings.filestore.s3.secret
                }
              }
            ])
            return done()
          }
        )
      })

      it('fetches the right key from the right bucket', function(done) {
        return this.S3PersistorManager.getFileStream(
          this.bucketName,
          this.key,
          {},
          (err, stream) => {
            if (err != null) {
              return done(err)
            }
            expect(this.s3Client.getObject.lastCall.args).to.deep.equal([
              {
                Bucket: this.bucketName,
                Key: this.key
              }
            ])
            return done()
          }
        )
      })

      it('accepts alternative credentials', function(done) {
        const accessKeyId = 'that_key'
        const secret = 'that_secret'
        const opts = {
          credentials: {
            auth_key: accessKeyId,
            auth_secret: secret
          }
        }
        return this.S3PersistorManager.getFileStream(
          this.bucketName,
          this.key,
          opts,
          (err, stream) => {
            if (err != null) {
              return done(err)
            }
            expect(this.awsS3.lastCall.args).to.deep.equal([
              {
                credentials: {
                  accessKeyId,
                  secretAccessKey: secret
                }
              }
            ])
            expect(stream).to.equal(this.expectedStream)
            return done()
          }
        )
      })

      return it('accepts byte range', function(done) {
        const start = 0
        const end = 8
        const opts = { start, end }
        return this.S3PersistorManager.getFileStream(
          this.bucketName,
          this.key,
          opts,
          (err, stream) => {
            if (err != null) {
              return done(err)
            }
            expect(this.s3Client.getObject.lastCall.args).to.deep.equal([
              {
                Bucket: this.bucketName,
                Key: this.key,
                Range: `bytes=${start}-${end}`
              }
            ])
            expect(stream).to.equal(this.expectedStream)
            return done()
          }
        )
      })
    })

    return describe('errors', function() {
      describe("when the file doesn't exist", function() {
        beforeEach(function() {
          return this.s3Request.send.callsFake(() => {
            return this.s3EventHandlers.httpHeaders(
              404,
              {},
              this.s3Response,
              'Not found'
            )
          })
        })

        return it('returns a NotFoundError that indicates the bucket and key', function(done) {
          return this.S3PersistorManager.getFileStream(
            this.bucketName,
            this.key,
            {},
            (err, stream) => {
              expect(err).to.be.instanceof(this.Errors.NotFoundError)
              const errMsg = this.Errors.NotFoundError.lastCall.args[0]
              expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`))
              expect(errMsg).to.match(new RegExp(`.*${this.key}.*`))
              return done()
            }
          )
        })
      })

      describe('when S3 encounters an unkown error', function() {
        beforeEach(function() {
          return this.s3Request.send.callsFake(() => {
            return this.s3EventHandlers.httpHeaders(
              500,
              {},
              this.s3Response,
              'Internal server error'
            )
          })
        })

        return it('returns an error', function(done) {
          return this.S3PersistorManager.getFileStream(
            this.bucketName,
            this.key,
            {},
            (err, stream) => {
              expect(err).to.be.instanceof(Error)
              return done()
            }
          )
        })
      })

      return describe('when the S3 request errors out before receiving HTTP headers', function() {
        beforeEach(function() {
          return this.s3Request.send.callsFake(() => {
            return this.s3EventHandlers.error(new Error('connection failed'))
          })
        })

        return it('returns an error', function(done) {
          return this.S3PersistorManager.getFileStream(
            this.bucketName,
            this.key,
            {},
            (err, stream) => {
              expect(err).to.be.instanceof(Error)
              return done()
            }
          )
        })
      })
    })
  })

  describe('getFileSize', function() {
    it('should obtain the file size from S3', function(done) {
      const expectedFileSize = 123
      this.s3Client.headObject.yields(
        new Error('s3Client.headObject got unexpected arguments')
      )
      this.s3Client.headObject
        .withArgs({
          Bucket: this.bucketName,
          Key: this.key
        })
        .yields(null, { ContentLength: expectedFileSize })

      return this.S3PersistorManager.getFileSize(
        this.bucketName,
        this.key,
        (err, fileSize) => {
          if (err != null) {
            return done(err)
          }
          expect(fileSize).to.equal(expectedFileSize)
          return done()
        }
      )
    })
    ;[403, 404].forEach(statusCode =>
      it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) {
        const error = new Error()
        error.statusCode = statusCode
        this.s3Client.headObject.yields(error)

        return this.S3PersistorManager.getFileSize(
          this.bucketName,
          this.key,
          (err, fileSize) => {
            expect(err).to.be.an.instanceof(this.Errors.NotFoundError)
            return done()
          }
        )
      })
    )

    return it('should rethrow any other error', function(done) {
      const error = new Error()
      this.s3Client.headObject.yields(error)
      this.s3Client.headObject.yields(error)

      return this.S3PersistorManager.getFileSize(
        this.bucketName,
        this.key,
        (err, fileSize) => {
          expect(err).to.equal(error)
          return done()
        }
      )
    })
  })

  describe('sendFile', function() {
    beforeEach(function() {
      return this.knoxClient.putFile.returns({ on() {} })
    })

    it('should put file with knox', function(done) {
      this.LocalFileWriter.deleteFile.callsArgWith(1)
      this.knoxClient.putFile.callsArgWith(2, this.error)
      return this.S3PersistorManager.sendFile(
        this.bucketName,
        this.key,
        this.fsPath,
        err => {
          this.knoxClient.putFile
            .calledWith(this.fsPath, this.key)
            .should.equal(true)
          err.should.equal(this.error)
          return done()
        }
      )
    })

    return it('should delete the file and pass the error with it', function(done) {
      this.LocalFileWriter.deleteFile.callsArgWith(1)
      this.knoxClient.putFile.callsArgWith(2, this.error)
      return this.S3PersistorManager.sendFile(
        this.bucketName,
        this.key,
        this.fsPath,
        err => {
          this.knoxClient.putFile
            .calledWith(this.fsPath, this.key)
            .should.equal(true)
          err.should.equal(this.error)
          return done()
        }
      )
    })
  })

  describe('sendStream', function() {
    beforeEach(function() {
      this.fsPath = 'to/some/where'
      this.origin = { on() {} }
      return (this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3))
    })

    it('should send stream to LocalFileWriter', function(done) {
      this.LocalFileWriter.deleteFile.callsArgWith(1)
      this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath)
      return this.S3PersistorManager.sendStream(
        this.bucketName,
        this.key,
        this.origin,
        () => {
          this.LocalFileWriter.writeStream
            .calledWith(this.origin)
            .should.equal(true)
          return done()
        }
      )
    })

    it('should return the error from LocalFileWriter', function(done) {
      this.LocalFileWriter.deleteFile.callsArgWith(1)
      this.LocalFileWriter.writeStream.callsArgWith(2, this.error)
      return this.S3PersistorManager.sendStream(
        this.bucketName,
        this.key,
        this.origin,
        err => {
          err.should.equal(this.error)
          return done()
        }
      )
    })

    return it('should send the file to the filestore', function(done) {
      this.LocalFileWriter.deleteFile.callsArgWith(1)
      this.LocalFileWriter.writeStream.callsArgWith(2)
      return this.S3PersistorManager.sendStream(
        this.bucketName,
        this.key,
        this.origin,
        err => {
          this.S3PersistorManager.sendFile.called.should.equal(true)
          return done()
        }
      )
    })
  })

  describe('copyFile', function() {
    beforeEach(function() {
      this.sourceKey = 'my/key'
      return (this.destKey = 'my/dest/key')
    })

    it('should use AWS SDK to copy file', function(done) {
      this.s3Client.copyObject.callsArgWith(1, this.error)
      return this.S3PersistorManager.copyFile(
        this.bucketName,
        this.sourceKey,
        this.destKey,
        err => {
          err.should.equal(this.error)
          this.s3Client.copyObject
            .calledWith({
              Bucket: this.bucketName,
              Key: this.destKey,
              CopySource: this.bucketName + '/' + this.key
            })
            .should.equal(true)
          return done()
        }
      )
    })

    return it('should return a NotFoundError object if the original file does not exist', function(done) {
      const NoSuchKeyError = { code: 'NoSuchKey' }
      this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError)
      return this.S3PersistorManager.copyFile(
        this.bucketName,
        this.sourceKey,
        this.destKey,
        err => {
          expect(err instanceof this.Errors.NotFoundError).to.equal(true)
          return done()
        }
      )
    })
  })

  describe('deleteDirectory', () =>
    it('should list the contents passing them onto multi delete', function(done) {
      const data = { Contents: [{ Key: '1234' }, { Key: '456' }] }
      this.knoxClient.list.callsArgWith(1, null, data)
      this.knoxClient.deleteMultiple.callsArgWith(1)
      return this.S3PersistorManager.deleteDirectory(
        this.bucketName,
        this.key,
        err => {
          this.knoxClient.deleteMultiple
            .calledWith(['1234', '456'])
            .should.equal(true)
          return done()
        }
      )
    }))

  describe('deleteFile', function() {
    it('should use correct options', function(done) {
      this.request.callsArgWith(1)

      return this.S3PersistorManager.deleteFile(
        this.bucketName,
        this.key,
        err => {
          const opts = this.request.args[0][0]
          assert.deepEqual(opts.aws, {
            key: this.settings.filestore.s3.key,
            secret: this.settings.filestore.s3.secret,
            bucket: this.bucketName
          })
          opts.method.should.equal('delete')
          opts.timeout.should.equal(30 * 1000)
          opts.uri.should.equal(
            `https://${this.bucketName}.s3.amazonaws.com/${this.key}`
          )
          return done()
        }
      )
    })

    return it('should return the error', function(done) {
      this.request.callsArgWith(1, this.error)

      return this.S3PersistorManager.deleteFile(
        this.bucketName,
        this.key,
        err => {
          err.should.equal(this.error)
          return done()
        }
      )
    })
  })

  describe('checkIfFileExists', function() {
    it('should use correct options', function(done) {
      this.request.callsArgWith(1, null, { statusCode: 200 })

      return this.S3PersistorManager.checkIfFileExists(
        this.bucketName,
        this.key,
        err => {
          const opts = this.request.args[0][0]
          assert.deepEqual(opts.aws, {
            key: this.settings.filestore.s3.key,
            secret: this.settings.filestore.s3.secret,
            bucket: this.bucketName
          })
          opts.method.should.equal('head')
          opts.timeout.should.equal(30 * 1000)
          opts.uri.should.equal(
            `https://${this.bucketName}.s3.amazonaws.com/${this.key}`
          )
          return done()
        }
      )
    })

    it('should return true for a 200', function(done) {
      this.request.callsArgWith(1, null, { statusCode: 200 })

      return this.S3PersistorManager.checkIfFileExists(
        this.bucketName,
        this.key,
        (err, exists) => {
          exists.should.equal(true)
          return done()
        }
      )
    })

    it('should return false for a non 200', function(done) {
      this.request.callsArgWith(1, null, { statusCode: 404 })

      return this.S3PersistorManager.checkIfFileExists(
        this.bucketName,
        this.key,
        (err, exists) => {
          exists.should.equal(false)
          return done()
        }
      )
    })

    return it('should return the error', function(done) {
      this.request.callsArgWith(1, this.error, {})

      return this.S3PersistorManager.checkIfFileExists(
        this.bucketName,
        this.key,
        err => {
          err.should.equal(this.error)
          return done()
        }
      )
    })
  })

  return describe('directorySize', () =>
    it('should sum directory files size', function(done) {
      const data = { Contents: [{ Size: 1024 }, { Size: 2048 }] }
      this.knoxClient.list.callsArgWith(1, null, data)
      return this.S3PersistorManager.directorySize(
        this.bucketName,
        this.key,
        (err, totalSize) => {
          totalSize.should.equal(3072)
          return done()
        }
      )
    }))
})