overleaf/services/filestore/test/unit/js/GcsPersistorTests.js

const sinon = require('sinon')
const chai = require('chai')
const { expect } = chai
const modulePath = '../../../app/js/GcsPersistor.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb')
const asyncPool = require('tiny-async-pool')

const Errors = require('../../../app/js/Errors')

describe('GcsPersistorTests', function() {
  const filename = '/wombat/potato.tex'
  const bucket = 'womBucket'
  const key = 'monKey'
  const destKey = 'donKey'
  const objectSize = 5555
  const genericError = new Error('guru meditation error')
  const filesSize = 33
  const md5 = 'ffffffff00000000ffffffff00000000'
  const WriteStream = 'writeStream'

  let Metrics,
    Logger,
    Storage,
    Fs,
    GcsNotFoundError,
    Meter,
    MeteredStream,
    ReadStream,
    Stream,
    GcsBucket,
    GcsFile,
    GcsPersistor,
    FileNotFoundError,
    Hash,
    settings,
    crypto,
    files

  beforeEach(function() {
    settings = {
      filestore: {
        backend: 'gcs',
        stores: {
          user_files: 'user_files'
        },
        gcs: {
          directoryKeyRegex: /^[0-9a-fA-F]{24}\/[0-9a-fA-F]{24}/
        }
      }
    }

    files = [
      {
        metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' },
        delete: sinon.stub()
      },
      {
        metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' },
        delete: sinon.stub()
      }
    ]

    ReadStream = {
      pipe: sinon.stub().returns('readStream'),
      on: sinon
        .stub()
        .withArgs('end')
        .yields(),
      removeListener: sinon.stub()
    }

    Stream = {
      pipeline: sinon.stub().yields()
    }

    Metrics = {
      count: sinon.stub()
    }

    GcsFile = {
      delete: sinon.stub().resolves(),
      createReadStream: sinon.stub().returns(ReadStream),
      getMetadata: sinon.stub().resolves([files[0].metadata]),
      createWriteStream: sinon.stub().returns(WriteStream),
      copy: sinon.stub().resolves(),
      exists: sinon.stub().resolves([true])
    }

    GcsBucket = {
      file: sinon.stub().returns(GcsFile),
      getFiles: sinon.stub().resolves([files])
    }

    Storage = class {
      constructor() {
        this.interceptors = []
      }
    }
    Storage.prototype.bucket = sinon.stub().returns(GcsBucket)

    GcsNotFoundError = new Error('File not found')
    GcsNotFoundError.code = 404

    Fs = {
      createReadStream: sinon.stub().returns(ReadStream)
    }

    FileNotFoundError = new Error('File not found')
    FileNotFoundError.code = 'ENOENT'

    MeteredStream = {
      type: 'metered',
      on: sinon.stub(),
      bytes: objectSize
    }
    MeteredStream.on.withArgs('finish').yields()
    MeteredStream.on.withArgs('readable').yields()
    Meter = sinon.stub().returns(MeteredStream)

    Hash = {
      end: sinon.stub(),
      read: sinon.stub().returns(md5),
      setEncoding: sinon.stub()
    }
    crypto = {
      createHash: sinon.stub().returns(Hash)
    }

    Logger = {
      warn: sinon.stub()
    }

    GcsPersistor = SandboxedModule.require(modulePath, {
      requires: {
        '@google-cloud/storage': { Storage },
        'settings-sharelatex': settings,
        'logger-sharelatex': Logger,
        'tiny-async-pool': asyncPool,
        './Errors': Errors,
        fs: Fs,
        'stream-meter': Meter,
        stream: Stream,
        'metrics-sharelatex': Metrics,
        crypto
      },
      globals: { console, Buffer }
    })
  })

  describe('getFileStream', function() {
    describe('when called with valid parameters', function() {
      let stream

      beforeEach(async function() {
        stream = await GcsPersistor.promises.getFileStream(bucket, key)
      })

      it('returns a metered stream', function() {
        expect(stream).to.equal(MeteredStream)
      })

      it('fetches the right key from the right bucket', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.createReadStream).to.have.been.called
      })

      it('pipes the stream through the meter', function() {
        expect(Stream.pipeline).to.have.been.calledWith(
          ReadStream,
          MeteredStream
        )
      })

      it('records an ingress metric', function() {
        expect(Metrics.count).to.have.been.calledWith('gcs.ingress', objectSize)
      })
    })

    describe('when called with a byte range', function() {
      let stream

      beforeEach(async function() {
        stream = await GcsPersistor.promises.getFileStream(bucket, key, {
          start: 5,
          end: 10
        })
      })

      it('returns a metered stream', function() {
        expect(stream).to.equal(MeteredStream)
      })

      it('passes the byte range on to GCS', function() {
        expect(GcsFile.createReadStream).to.have.been.calledWith({
          start: 5,
          end: 11 // we increment the end because Google's 'end' is exclusive
        })
      })
    })

    describe("when the file doesn't exist", function() {
      let error, stream

      beforeEach(async function() {
        ReadStream.on = sinon.stub()
        ReadStream.on.withArgs('error').yields(GcsNotFoundError)
        try {
          stream = await GcsPersistor.promises.getFileStream(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('does not return a stream', function() {
        expect(stream).not.to.exist
      })

      it('throws a NotFoundError', function() {
        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
      })

      it('wraps the error', function() {
        expect(error.cause).to.exist
      })

      it('stores the bucket and key in the error', function() {
        expect(error.info).to.include({ bucketName: bucket, key: key })
      })
    })

    describe('when Gcs encounters an unkown error', function() {
      let error, stream

      beforeEach(async function() {
        ReadStream.on = sinon.stub()
        ReadStream.on.withArgs('error').yields(genericError)
        try {
          stream = await GcsPersistor.promises.getFileStream(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('does not return a stream', function() {
        expect(stream).not.to.exist
      })

      it('throws a ReadError', function() {
        expect(error).to.be.an.instanceOf(Errors.ReadError)
      })

      it('wraps the error', function() {
        expect(error.cause).to.exist
      })

      it('stores the bucket and key in the error', function() {
        expect(error.info).to.include({ bucketName: bucket, key: key })
      })
    })
  })

  describe('getFileSize', function() {
    describe('when called with valid parameters', function() {
      let size

      beforeEach(async function() {
        size = await GcsPersistor.promises.getFileSize(bucket, key)
      })

      it('should return the object size', function() {
        expect(size).to.equal(files[0].metadata.size)
      })

      it('should pass the bucket and key to GCS', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.getMetadata).to.have.been.called
      })
    })

    describe('when the object is not found', function() {
      let error

      beforeEach(async function() {
        GcsFile.getMetadata = sinon.stub().rejects(GcsNotFoundError)
        try {
          await GcsPersistor.promises.getFileSize(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('should return a NotFoundError', function() {
        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
      })

      it('should wrap the error', function() {
        expect(error.cause).to.equal(GcsNotFoundError)
      })
    })

    describe('when GCS returns an error', function() {
      let error

      beforeEach(async function() {
        GcsFile.getMetadata = sinon.stub().rejects(genericError)
        try {
          await GcsPersistor.promises.getFileSize(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('should return a ReadError', function() {
        expect(error).to.be.an.instanceOf(Errors.ReadError)
      })

      it('should wrap the error', function() {
        expect(error.cause).to.equal(genericError)
      })
    })
  })

  describe('sendStream', function() {
    describe('with valid parameters', function() {
      beforeEach(async function() {
        return GcsPersistor.promises.sendStream(bucket, key, ReadStream)
      })

      it('should upload the stream', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.createWriteStream).to.have.been.called
      })

      it('should not try to create a resumable upload', function() {
        expect(GcsFile.createWriteStream).to.have.been.calledWith({
          resumable: false
        })
      })

      it('should meter the stream', function() {
        expect(Stream.pipeline).to.have.been.calledWith(
          ReadStream,
          MeteredStream
        )
      })

      it('should pipe the metered stream to GCS', function() {
        expect(Stream.pipeline).to.have.been.calledWith(
          MeteredStream,
          WriteStream
        )
      })

      it('should record an egress metric', function() {
        expect(Metrics.count).to.have.been.calledWith('gcs.egress', objectSize)
      })

      it('calculates the md5 hash of the file', function() {
        expect(Stream.pipeline).to.have.been.calledWith(ReadStream, Hash)
      })
    })

    describe('when a hash is supplied', function() {
      beforeEach(async function() {
        return GcsPersistor.promises.sendStream(
          bucket,
          key,
          ReadStream,
          'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb'
        )
      })

      it('should not calculate the md5 hash of the file', function() {
        expect(Stream.pipeline).not.to.have.been.calledWith(
          sinon.match.any,
          Hash
        )
      })

      it('sends the hash in base64', function() {
        expect(GcsFile.createWriteStream).to.have.been.calledWith({
          validation: 'md5',
          metadata: {
            md5Hash: 'qqqqqru7u7uqqqqqu7u7uw=='
          },
          resumable: false
        })
      })

      it('does not fetch the md5 hash of the uploaded file', function() {
        expect(GcsFile.getMetadata).not.to.have.been.called
      })
    })

    describe('when the upload fails', function() {
      let error
      beforeEach(async function() {
        Stream.pipeline
          .withArgs(MeteredStream, WriteStream, sinon.match.any)
          .yields(genericError)
        try {
          await GcsPersistor.promises.sendStream(bucket, key, ReadStream)
        } catch (err) {
          error = err
        }
      })

      it('throws a WriteError', function() {
        expect(error).to.be.an.instanceOf(Errors.WriteError)
      })

      it('wraps the error', function() {
        expect(error.cause).to.equal(genericError)
      })
    })
  })

  describe('sendFile', function() {
    describe('with valid parameters', function() {
      beforeEach(async function() {
        return GcsPersistor.promises.sendFile(bucket, key, filename)
      })

      it('should create a read stream for the file', function() {
        expect(Fs.createReadStream).to.have.been.calledWith(filename)
      })

      it('should create a write stream', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.createWriteStream).to.have.been.called
      })

      it('should upload the stream via the meter', function() {
        expect(Stream.pipeline).to.have.been.calledWith(
          ReadStream,
          MeteredStream
        )
        expect(Stream.pipeline).to.have.been.calledWith(
          MeteredStream,
          WriteStream
        )
      })
    })
  })

  describe('copyFile', function() {
    const destinationFile = 'destFile'

    beforeEach(function() {
      GcsBucket.file.withArgs(destKey).returns(destinationFile)
    })

    describe('with valid parameters', function() {
      beforeEach(async function() {
        return GcsPersistor.promises.copyFile(bucket, key, destKey)
      })

      it('should copy the object', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.copy).to.have.been.calledWith(destinationFile)
      })
    })

    describe('when the file does not exist', function() {
      let error

      beforeEach(async function() {
        GcsFile.copy = sinon.stub().rejects(GcsNotFoundError)
        try {
          await GcsPersistor.promises.copyFile(bucket, key, destKey)
        } catch (err) {
          error = err
        }
      })

      it('should throw a NotFoundError', function() {
        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
      })
    })
  })

  describe('deleteFile', function() {
    describe('with valid parameters', function() {
      beforeEach(async function() {
        return GcsPersistor.promises.deleteFile(bucket, key)
      })

      it('should delete the object', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.delete).to.have.been.called
      })
    })

    describe('when the file does not exist', function() {
      let error

      beforeEach(async function() {
        GcsFile.delete = sinon.stub().rejects(GcsNotFoundError)
        try {
          await GcsPersistor.promises.deleteFile(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('should not throw an error', function() {
        expect(error).not.to.exist
      })
    })
  })

  describe('deleteDirectory', function() {
    const directoryName = `${ObjectId()}/${ObjectId()}`
    describe('with valid parameters', function() {
      beforeEach(async function() {
        console.log(key)
        return GcsPersistor.promises.deleteDirectory(bucket, directoryName)
      })

      it('should list the objects in the directory', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.getFiles).to.have.been.calledWith({
          directory: directoryName
        })
      })

      it('should delete the files', function() {
        expect(GcsFile.delete).to.have.been.calledTwice
      })
    })

    describe('when there is an error listing the objects', function() {
      let error

      beforeEach(async function() {
        GcsBucket.getFiles = sinon.stub().rejects(genericError)
        try {
          await GcsPersistor.promises.deleteDirectory(bucket, directoryName)
        } catch (err) {
          error = err
        }
      })

      it('should generate a WriteError', function() {
        expect(error).to.be.an.instanceOf(Errors.WriteError)
      })

      it('should wrap the error', function() {
        expect(error.cause).to.equal(genericError)
      })
    })
  })

  describe('directorySize', function() {
    describe('with valid parameters', function() {
      let size

      beforeEach(async function() {
        size = await GcsPersistor.promises.directorySize(bucket, key)
      })

      it('should list the objects in the directory', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
      })

      it('should return the directory size', function() {
        expect(size).to.equal(filesSize)
      })
    })

    describe('when there are no files', function() {
      let size

      beforeEach(async function() {
        GcsBucket.getFiles.resolves([[]])
        size = await GcsPersistor.promises.directorySize(bucket, key)
      })

      it('should list the objects in the directory', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
      })

      it('should return zero', function() {
        expect(size).to.equal(0)
      })
    })

    describe('when there is an error listing the objects', function() {
      let error

      beforeEach(async function() {
        GcsBucket.getFiles.rejects(genericError)
        try {
          await GcsPersistor.promises.directorySize(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('should generate a ReadError', function() {
        expect(error).to.be.an.instanceOf(Errors.ReadError)
      })

      it('should wrap the error', function() {
        expect(error.cause).to.equal(genericError)
      })
    })
  })

  describe('checkIfFileExists', function() {
    describe('when the file exists', function() {
      let exists

      beforeEach(async function() {
        exists = await GcsPersistor.promises.checkIfFileExists(bucket, key)
      })

      it('should ask the file if it exists', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.exists).to.have.been.called
      })

      it('should return that the file exists', function() {
        expect(exists).to.equal(true)
      })
    })

    describe('when the file does not exist', function() {
      let exists

      beforeEach(async function() {
        GcsFile.exists = sinon.stub().resolves([false])
        exists = await GcsPersistor.promises.checkIfFileExists(bucket, key)
      })

      it('should get the object header', function() {
        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
        expect(GcsBucket.file).to.have.been.calledWith(key)
        expect(GcsFile.exists).to.have.been.called
      })

      it('should return that the file does not exist', function() {
        expect(exists).to.equal(false)
      })
    })

    describe('when there is an error', function() {
      let error

      beforeEach(async function() {
        GcsFile.exists = sinon.stub().rejects(genericError)
        try {
          await GcsPersistor.promises.checkIfFileExists(bucket, key)
        } catch (err) {
          error = err
        }
      })

      it('should generate a ReadError', function() {
        expect(error).to.be.an.instanceOf(Errors.ReadError)
      })

      it('should wrap the error', function() {
        expect(error.cause).to.equal(genericError)
      })
    })
  })
})