Merge pull request #86 from overleaf/spd-gcs-persistor

Add GCS Persistor

services/filestore/app.js

@@ -61,6 +61,11 @@ app.delete(
   keyBuilder.userFileKeyMiddleware,
   fileController.deleteFile
 )
+app.delete(
+  '/project/:project_id',
+  keyBuilder.userProjectKeyMiddleware,
+  fileController.deleteProject
+)
 
 app.head(
   '/template/:template_id/v/:version/:format',

services/filestore/app/js/Errors.js

@@ -25,6 +25,7 @@ class ConversionsDisabledError extends BackwardCompatibleError {}
 class ConversionError extends BackwardCompatibleError {}
 class SettingsError extends BackwardCompatibleError {}
 class TimeoutError extends BackwardCompatibleError {}
+class InvalidParametersError extends BackwardCompatibleError {}
 
 class FailedCommandError extends OError {
   constructor(command, code, stdout, stderr) {
@@ -50,5 +51,6 @@ module.exports = {
   ConversionError,
   HealthCheckError,
   SettingsError,
-  TimeoutError
+  TimeoutError,
+  InvalidParametersError
 }

services/filestore/app/js/FSPersistor.js

@@ -1,7 +1,6 @@
 const fs = require('fs')
 const glob = require('glob')
 const path = require('path')
-const rimraf = require('rimraf')
 const Stream = require('stream')
 const { promisify, callbackify } = require('util')
 
@@ -14,7 +13,6 @@ const fsUnlink = promisify(fs.unlink)
 const fsOpen = promisify(fs.open)
 const fsStat = promisify(fs.stat)
 const fsGlob = promisify(glob)
-const rmrf = promisify(rimraf)
 
 const filterName = key => key.replace(/\//g, '_')
 
@@ -146,7 +144,9 @@ async function deleteDirectory(location, name) {
   const filteredName = filterName(name.replace(/\/$/, ''))
 
   try {
-    await rmrf(`${location}/${filteredName}`)
+    await Promise.all(
+      (await fsGlob(`${location}/${filteredName}*`)).map(file => fsUnlink(file))
+    )
   } catch (err) {
     throw PersistorHelper.wrapError(
       err,

services/filestore/app/js/FileController.js

@@ -13,6 +13,7 @@ module.exports = {
   insertFile,
   copyFile,
   deleteFile,
+  deleteProject,
   directorySize
 }
 
@@ -158,6 +159,25 @@ function deleteFile(req, res, next) {
   })
 }
 
+function deleteProject(req, res, next) {
+  metrics.inc('deleteProject')
+  const { key, bucket } = req
+
+  req.requestLogger.setMessage('deleting project')
+  req.requestLogger.addFields({ key, bucket })
+
+  FileHandler.deleteProject(bucket, key, function(err) {
+    if (err) {
+      if (err instanceof Errors.InvalidParametersError) {
+        return res.sendStatus(400)
+      }
+      next(err)
+    } else {
+      res.sendStatus(204)
+    }
+  })
+}
+
 function directorySize(req, res, next) {
   metrics.inc('projectSize')
   const { project_id: projectId, bucket } = req

services/filestore/app/js/FileHandler.js

@@ -5,11 +5,12 @@ const LocalFileWriter = require('./LocalFileWriter')
 const FileConverter = require('./FileConverter')
 const KeyBuilder = require('./KeyBuilder')
 const ImageOptimiser = require('./ImageOptimiser')
-const { ConversionError } = require('./Errors')
+const { ConversionError, InvalidParametersError } = require('./Errors')
 
 module.exports = {
   insertFile: callbackify(insertFile),
   deleteFile: callbackify(deleteFile),
+  deleteProject: callbackify(deleteProject),
   getFile: callbackify(getFile),
   getFileSize: callbackify(getFileSize),
   getDirectorySize: callbackify(getDirectorySize),
@@ -17,6 +18,7 @@ module.exports = {
     getFile,
     insertFile,
     deleteFile,
+    deleteProject,
     getFileSize,
     getDirectorySize
   }
@@ -24,18 +26,40 @@ module.exports = {
 
 async function insertFile(bucket, key, stream) {
   const convertedKey = KeyBuilder.getConvertedFolderKey(key)
+  if (!convertedKey.match(/^[0-9a-f]{24}\/[0-9a-f]{24}/i)) {
+    throw new InvalidParametersError({
+      message: 'key does not match validation regex',
+      info: { bucket, key, convertedKey }
+    })
+  }
   await PersistorManager.promises.deleteDirectory(bucket, convertedKey)
   await PersistorManager.promises.sendStream(bucket, key, stream)
 }
 
 async function deleteFile(bucket, key) {
   const convertedKey = KeyBuilder.getConvertedFolderKey(key)
+  if (!convertedKey.match(/^[0-9a-f]{24}\/[0-9a-f]{24}/i)) {
+    throw new InvalidParametersError({
+      message: 'key does not match validation regex',
+      info: { bucket, key, convertedKey }
+    })
+  }
   await Promise.all([
     PersistorManager.promises.deleteFile(bucket, key),
     PersistorManager.promises.deleteDirectory(bucket, convertedKey)
   ])
 }
 
+async function deleteProject(bucket, key) {
+  if (!key.match(/^[0-9a-f]{24}\//i)) {
+    throw new InvalidParametersError({
+      message: 'key does not match validation regex',
+      info: { bucket, key }
+    })
+  }
+  await PersistorManager.promises.deleteDirectory(bucket, key)
+}
+
 async function getFile(bucket, key, opts) {
   opts = opts || {}
   if (!opts.format && !opts.style) {

services/filestore/app/js/GcsPersistor.js

@@ -0,0 +1,288 @@
+const settings = require('settings-sharelatex')
+const fs = require('fs')
+const { promisify } = require('util')
+const Stream = require('stream')
+const { Storage } = require('@google-cloud/storage')
+const { callbackify } = require('util')
+const { WriteError, ReadError, NotFoundError } = require('./Errors')
+const asyncPool = require('tiny-async-pool')
+const PersistorHelper = require('./PersistorHelper')
+
+const pipeline = promisify(Stream.pipeline)
+
+// endpoint settings will be null by default except for tests
+// that's OK - GCS uses the locally-configured service account by default
+const storage = new Storage(settings.filestore.gcs.endpoint)
+// workaround for broken uploads with custom endpoints:
+// https://github.com/googleapis/nodejs-storage/issues/898
+if (
+  settings.filestore.gcs.endpoint &&
+  settings.filestore.gcs.endpoint.apiEndpoint
+) {
+  storage.interceptors.push({
+    request: function(reqOpts) {
+      const url = new URL(reqOpts.uri)
+      url.host = settings.filestore.gcs.endpoint.apiEndpoint
+      if (settings.filestore.gcs.endpoint.apiScheme) {
+        url.protocol = settings.filestore.gcs.endpoint.apiScheme
+      }
+      reqOpts.uri = url.toString()
+      return reqOpts
+    }
+  })
+}
+
+const GcsPersistor = {
+  sendFile: callbackify(sendFile),
+  sendStream: callbackify(sendStream),
+  getFileStream: callbackify(getFileStream),
+  getFileMd5Hash: callbackify(getFileMd5Hash),
+  deleteDirectory: callbackify(deleteDirectory),
+  getFileSize: callbackify(getFileSize),
+  deleteFile: callbackify(deleteFile),
+  copyFile: callbackify(copyFile),
+  checkIfFileExists: callbackify(checkIfFileExists),
+  directorySize: callbackify(directorySize),
+  promises: {
+    sendFile,
+    sendStream,
+    getFileStream,
+    getFileMd5Hash,
+    deleteDirectory,
+    getFileSize,
+    deleteFile,
+    copyFile,
+    checkIfFileExists,
+    directorySize
+  }
+}
+
+module.exports = GcsPersistor
+
+async function sendFile(bucketName, key, fsPath) {
+  return sendStream(bucketName, key, fs.createReadStream(fsPath))
+}
+
+async function sendStream(bucketName, key, readStream, sourceMd5) {
+  try {
+    let hashPromise
+
+    // if there is no supplied md5 hash, we calculate the hash as the data passes through
+    if (!sourceMd5) {
+      hashPromise = PersistorHelper.calculateStreamMd5(readStream)
+    }
+
+    const meteredStream = PersistorHelper.getMeteredStream(
+      readStream,
+      'gcs.egress' // egress from us to gcs
+    )
+
+    const writeOptions = {
+      // disabling of resumable uploads is recommended by Google:
+      resumable: false
+    }
+
+    if (sourceMd5) {
+      writeOptions.validation = 'md5'
+      writeOptions.metadata = {
+        md5Hash: PersistorHelper.hexToBase64(sourceMd5)
+      }
+    }
+
+    const uploadStream = storage
+      .bucket(bucketName)
+      .file(key)
+      .createWriteStream(writeOptions)
+
+    await pipeline(meteredStream, uploadStream)
+
+    // if we didn't have an md5 hash, we should compare our computed one with Google's
+    // as we couldn't tell GCS about it beforehand
+    if (hashPromise) {
+      sourceMd5 = await hashPromise
+      // throws on mismatch
+      await PersistorHelper.verifyMd5(GcsPersistor, bucketName, key, sourceMd5)
+    }
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'upload to GCS failed',
+      { bucketName, key },
+      WriteError
+    )
+  }
+}
+
+async function getFileStream(bucketName, key, _opts = {}) {
+  const opts = Object.assign({}, _opts)
+  if (opts.end) {
+    // S3 (and http range headers) treat 'end' as inclusive, so increase this by 1
+    opts.end++
+  }
+  const stream = storage
+    .bucket(bucketName)
+    .file(key)
+    .createReadStream(opts)
+
+  const meteredStream = PersistorHelper.getMeteredStream(
+    stream,
+    'gcs.ingress' // ingress to us from gcs
+  )
+
+  try {
+    await PersistorHelper.waitForStreamReady(stream)
+    return meteredStream
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'error reading file from GCS',
+      { bucketName, key, opts },
+      ReadError
+    )
+  }
+}
+
+async function getFileSize(bucketName, key) {
+  try {
+    const [metadata] = await storage
+      .bucket(bucketName)
+      .file(key)
+      .getMetadata()
+    return metadata.size
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'error getting size of GCS object',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+async function getFileMd5Hash(bucketName, key) {
+  try {
+    const [metadata] = await storage
+      .bucket(bucketName)
+      .file(key)
+      .getMetadata()
+    return PersistorHelper.base64ToHex(metadata.md5Hash)
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'error getting hash of GCS object',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+async function deleteFile(bucketName, key) {
+  try {
+    const file = storage.bucket(bucketName).file(key)
+
+    if (settings.filestore.gcs.deletedBucketSuffix) {
+      await file.copy(
+        storage
+          .bucket(`${bucketName}${settings.filestore.gcs.deletedBucketSuffix}`)
+          .file(`${key}-${new Date().toISOString()}`)
+      )
+    }
+    if (settings.filestore.gcs.unlockBeforeDelete) {
+      await file.setMetadata({ eventBasedHold: false })
+    }
+    await file.delete()
+  } catch (err) {
+    const error = PersistorHelper.wrapError(
+      err,
+      'error deleting GCS object',
+      { bucketName, key },
+      WriteError
+    )
+    if (!(error instanceof NotFoundError)) {
+      throw error
+    }
+  }
+}
+
+async function deleteDirectory(bucketName, key) {
+  try {
+    const [files] = await storage
+      .bucket(bucketName)
+      .getFiles({ directory: key })
+
+    await asyncPool(
+      settings.filestore.gcs.deleteConcurrency,
+      files,
+      async file => {
+        await deleteFile(bucketName, file.name)
+      }
+    )
+  } catch (err) {
+    const error = PersistorHelper.wrapError(
+      err,
+      'failed to delete directory in GCS',
+      { bucketName, key },
+      WriteError
+    )
+    if (error instanceof NotFoundError) {
+      return
+    }
+    throw error
+  }
+}
+
+async function directorySize(bucketName, key) {
+  let files
+
+  try {
+    const [response] = await storage
+      .bucket(bucketName)
+      .getFiles({ directory: key })
+    files = response
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'failed to list objects in GCS',
+      { bucketName, key },
+      ReadError
+    )
+  }
+
+  return files.reduce((acc, file) => Number(file.metadata.size) + acc, 0)
+}
+
+async function checkIfFileExists(bucketName, key) {
+  try {
+    const [response] = await storage
+      .bucket(bucketName)
+      .file(key)
+      .exists()
+    return response
+  } catch (err) {
+    throw PersistorHelper.wrapError(
+      err,
+      'error checking if file exists in GCS',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+async function copyFile(bucketName, sourceKey, destKey) {
+  try {
+    const src = storage.bucket(bucketName).file(sourceKey)
+    const dest = storage.bucket(bucketName).file(destKey)
+    await src.copy(dest)
+  } catch (err) {
+    // fake-gcs-server has a bug that returns an invalid response when the file does not exist
+    if (err.message === 'Cannot parse response as JSON: not found\n') {
+      err.code = 404
+    }
+    throw PersistorHelper.wrapError(
+      err,
+      'failed to copy file in GCS',
+      { bucketName, sourceKey, destKey },
+      WriteError
+    )
+  }
+}

services/filestore/app/js/KeyBuilder.js

@@ -4,6 +4,7 @@ module.exports = {
   getConvertedFolderKey,
   addCachingToKey,
   userFileKeyMiddleware,
+  userProjectKeyMiddleware,
   publicFileKeyMiddleware,
   publicProjectKeyMiddleware,
   bucketFileKeyMiddleware,
@@ -37,6 +38,13 @@ function userFileKeyMiddleware(req, res, next) {
   next()
 }
 
+function userProjectKeyMiddleware(req, res, next) {
+  const { project_id: projectId } = req.params
+  req.key = `${projectId}/`
+  req.bucket = settings.filestore.stores.user_files
+  next()
+}
+
 function publicFileKeyMiddleware(req, res, next) {
   if (settings.filestore.stores.public_files == null) {
     return res.status(501).send('public files not available')

services/filestore/app/js/PersistorHelper.js

@@ -1,4 +1,5 @@
 const crypto = require('crypto')
+const metrics = require('metrics-sharelatex')
 const meter = require('stream-meter')
 const Stream = require('stream')
 const logger = require('logger-sharelatex')
@@ -12,7 +13,9 @@ module.exports = {
   verifyMd5,
   getMeteredStream,
   waitForStreamReady,
-  wrapError
+  wrapError,
+  hexToBase64,
+  base64ToHex
 }
 
 // returns a promise which resolves with the md5 hash of the stream
@@ -52,16 +55,16 @@ async function verifyMd5(persistor, bucket, key, sourceMd5, destMd5 = null) {
 
 // returns the next stream in the pipeline, and calls the callback with the byte count
 // when the stream finishes or receives an error
-function getMeteredStream(stream, callback) {
+function getMeteredStream(stream, metricName) {
   const meteredStream = meter()
 
   pipeline(stream, meteredStream)
     .then(() => {
-      callback(null, meteredStream.bytes)
+      metrics.count(metricName, meteredStream.bytes)
     })
-    .catch(err => {
+    .catch(() => {
       // on error, just send how many bytes we received before the stream stopped
-      callback(err, meteredStream.bytes)
+      metrics.count(metricName, meteredStream.bytes)
     })
 
   return meteredStream
@@ -90,7 +93,8 @@ function wrapError(error, message, params, ErrorType) {
     error instanceof NotFoundError ||
     ['NoSuchKey', 'NotFound', 404, 'AccessDenied', 'ENOENT'].includes(
       error.code
-    )
+    ) ||
+    (error.response && error.response.statusCode === 404)
   ) {
     return new NotFoundError({
       message: 'no such file',
@@ -103,3 +107,11 @@ function wrapError(error, message, params, ErrorType) {
     }).withCause(error)
   }
 }
+
+function base64ToHex(base64) {
+  return Buffer.from(base64, 'base64').toString('hex')
+}
+
+function hexToBase64(hex) {
+  return Buffer.from(hex, 'hex').toString('base64')
+}

services/filestore/app/js/PersistorManager.js

@@ -19,6 +19,8 @@ function getPersistor(backend) {
       return require('./S3Persistor')
     case 'fs':
       return require('./FSPersistor')
+    case 'gcs':
+      return require('./GcsPersistor')
     default:
       throw new Error(`unknown filestore backend: ${backend}`)
   }

services/filestore/app/js/S3Persistor.js

@@ -4,7 +4,6 @@ http.globalAgent.maxSockets = 300
 https.globalAgent.maxSockets = 300
 
 const settings = require('settings-sharelatex')
-const metrics = require('metrics-sharelatex')
 
 const PersistorHelper = require('./PersistorHelper')
 
@@ -46,23 +45,8 @@ const S3Persistor = {
 
 module.exports = S3Persistor
 
-function hexToBase64(hex) {
-  return Buffer.from(hex, 'hex').toString('base64')
-}
-
 async function sendFile(bucketName, key, fsPath) {
-  let readStream
+  return sendStream(bucketName, key, fs.createReadStream(fsPath))
-  try {
-    readStream = fs.createReadStream(fsPath)
-  } catch (err) {
-    throw PersistorHelper.wrapError(
-      err,
-      'error reading file from disk',
-      { bucketName, key, fsPath },
-      ReadError
-    )
-  }
-  return sendStream(bucketName, key, readStream)
 }
 
 async function sendStream(bucketName, key, readStream, sourceMd5) {
@@ -72,17 +56,14 @@ async function sendStream(bucketName, key, readStream, sourceMd5) {
     let b64Hash
 
     if (sourceMd5) {
-      b64Hash = hexToBase64(sourceMd5)
+      b64Hash = PersistorHelper.hexToBase64(sourceMd5)
     } else {
       hashPromise = PersistorHelper.calculateStreamMd5(readStream)
     }
 
     const meteredStream = PersistorHelper.getMeteredStream(
       readStream,
-      (_, byteCount) => {
+      's3.egress' // egress from us to s3
-        // ignore the error parameter and just log the byte count
-        metrics.count('s3.egress', byteCount)
-      }
     )
 
     // if we have an md5 hash, pass this to S3 to verify the upload
@@ -149,10 +130,7 @@ async function getFileStream(bucketName, key, opts) {
 
   const meteredStream = PersistorHelper.getMeteredStream(
     stream,
-    (_, byteCount) => {
+    's3.ingress' // ingress to us from s3
-      // ignore the error parameter and just log the byte count
-      metrics.count('s3.ingress', byteCount)
-    }
   )
 
   try {

services/filestore/config/settings.defaults.coffee

@@ -31,8 +31,19 @@ settings =
 		# Choices are
 		# s3 - Amazon S3
 		# fs - local filesystem
+		# gcs - Google Cloud Storage
 		backend: process.env['BACKEND']
 
+		gcs:
+			endpoint:
+				if process.env['GCS_API_ENDPOINT']
+					apiEndpoint: process.env['GCS_API_ENDPOINT']
+					apiScheme: process.env['GCS_API_SCHEME']
+					projectId: process.env['GCS_PROJECT_ID']
+			unlockBeforeDelete: process.env['GCS_UNLOCK_BEFORE_DELETE'] == "true"  # unlock an event-based hold before deleting. default false
+			deletedBucketSuffix: process.env['GCS_DELETED_BUCKET_SUFFIX']          # if present, copy file to another bucket on delete. default null
+			deleteConcurrency: parseInt(process.env['GCS_DELETE_CONCURRENCY']) || 50
+
 		s3:
 			if process.env['AWS_ACCESS_KEY_ID']? or process.env['S3_BUCKET_CREDENTIALS']?
 				key: process.env['AWS_ACCESS_KEY_ID']
@@ -41,6 +52,9 @@ settings =
 				pathStyle: process.env['AWS_S3_PATH_STYLE']
 				partSize: process.env['AWS_S3_PARTSIZE'] or (100 * 1024 * 1024)
 
+		# GCS should be configured by the service account on the kubernetes pod. See GOOGLE_APPLICATION_CREDENTIALS,
+		# which will be picked up automatically.
+
 		stores:
 			user_files: process.env['USER_FILES_BUCKET_NAME']
 			template_files: process.env['TEMPLATE_FILES_BUCKET_NAME']

services/filestore/docker-compose.ci.yml

@@ -22,10 +22,6 @@ services:
       REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
-      AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
-      AWS_SECRET_ACCESS_KEY: fake
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
       ENABLE_CONVERSIONS: "true"
@@ -33,9 +29,21 @@ services:
       AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
       AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
       AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
+      AWS_S3_PATH_STYLE: 'true'
+      GCS_API_ENDPOINT: gcs:9090
+      GCS_API_SCHEME: http
+      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      GCS_PUBLIC_FILES_BUCKET_NAME: fake_publicfiles
+      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
     depends_on:
       s3:
         condition: service_healthy
+      gcs:
+        condition: service_healthy
     user: node
     command: npm run test:acceptance:_run
 
@@ -48,8 +56,13 @@ services:
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
   s3:
-    image: adobe/s3mock
+    build:
+      context: test/acceptance/deps
+      dockerfile: Dockerfile.s3mock
     environment:
-      - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
+      - initialBuckets=fake_user_files,fake_template_files,fake_public_files
-    healthcheck:
+
-      test: ["CMD", "curl", "-f", "http://localhost:9090"]
+  gcs:
+    build:
+      context: test/acceptance/deps
+      dockerfile: Dockerfile.fake-gcs

services/filestore/docker-compose.yml

@@ -31,10 +31,6 @@ services:
       REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
-      AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
-      AWS_SECRET_ACCESS_KEY: fake
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ERROR
       NODE_ENV: test
@@ -43,15 +39,32 @@ services:
       AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
       AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
       AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_PATH_STYLE: 'true'
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
+      GCS_API_ENDPOINT: gcs:9090
+      GCS_API_SCHEME: http
+      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      GCS_PUBLIC_FILES_BUCKET_NAME: fake_publicfiles
+      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
     user: node
     depends_on:
       s3:
         condition: service_healthy
+      gcs:
+        condition: service_healthy
     command: npm run test:acceptance
 
   s3:
-    image: adobe/s3mock
+    build:
+      context: test/acceptance/deps
+      dockerfile: Dockerfile.s3mock
     environment:
-      - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
+      - initialBuckets=fake_user_files,fake_template_files,fake_public_files
-    healthcheck:
+
-      test: ["CMD", "curl", "-f", "http://localhost:9090"]
+  gcs:
+    build:
+      context: test/acceptance/deps
+      dockerfile: Dockerfile.fake-gcs

services/filestore/package-lock.json

@@ -603,6 +603,52 @@
       "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz",
       "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ=="
     },
+    "@google-cloud/storage": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-4.4.0.tgz",
+      "integrity": "sha512-R64ey4dLIG3IgiKw0CL5MdZ4ZtZdGhN75171vjiL+ioZG+hlLFkjsrCTRuIdE35v42nNe5nXmVhBHQQTuPozHA==",
+      "requires": {
+        "@google-cloud/common": "^2.1.1",
+        "@google-cloud/paginator": "^2.0.0",
+        "@google-cloud/promisify": "^1.0.0",
+        "arrify": "^2.0.0",
+        "compressible": "^2.0.12",
+        "concat-stream": "^2.0.0",
+        "date-and-time": "^0.12.0",
+        "duplexify": "^3.5.0",
+        "extend": "^3.0.2",
+        "gaxios": "^2.0.1",
+        "gcs-resumable-upload": "^2.2.4",
+        "hash-stream-validation": "^0.2.2",
+        "mime": "^2.2.0",
+        "mime-types": "^2.0.8",
+        "onetime": "^5.1.0",
+        "p-limit": "^2.2.0",
+        "pumpify": "^2.0.0",
+        "readable-stream": "^3.4.0",
+        "snakeize": "^0.1.0",
+        "stream-events": "^1.0.1",
+        "through2": "^3.0.0",
+        "xdg-basedir": "^4.0.0"
+      },
+      "dependencies": {
+        "mime": {
+          "version": "2.4.4",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
+          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
+        },
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
+      }
+    },
     "@google-cloud/trace-agent": {
       "version": "3.6.1",
       "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz",
@@ -1369,6 +1415,16 @@
       "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz",
       "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g=="
     },
+    "bl": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
+      "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
+      "dev": true,
+      "requires": {
+        "readable-stream": "^2.3.5",
+        "safe-buffer": "^5.1.1"
+      }
+    },
     "body-parser": {
       "version": "1.19.0",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
@@ -1407,6 +1463,12 @@
       "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
       "dev": true
     },
+    "bson": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.3.tgz",
+      "integrity": "sha512-TdiJxMVnodVS7r0BdL42y/pqC9cL2iKynVwA0Ho3qbsQYr428veL3l7BQyuqiw+Q5SqqoT0m4srSY/BlZ9AxXg==",
+      "dev": true
+    },
     "buffer": {
       "version": "4.9.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
@@ -1422,6 +1484,11 @@
       "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
       "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
     },
+    "buffer-from": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+    },
     "builtin-modules": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
@@ -1608,11 +1675,55 @@
       "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==",
       "dev": true
     },
+    "compressible": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
+      "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
+      "requires": {
+        "mime-db": ">= 1.43.0 < 2"
+      }
+    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
     },
+    "concat-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+      "requires": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.0.2",
+        "typedarray": "^0.0.6"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
+      }
+    },
+    "configstore": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
+      "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==",
+      "requires": {
+        "dot-prop": "^5.2.0",
+        "graceful-fs": "^4.1.2",
+        "make-dir": "^3.0.0",
+        "unique-string": "^2.0.0",
+        "write-file-atomic": "^3.0.0",
+        "xdg-basedir": "^4.0.0"
+      }
+    },
     "console-log-level": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz",
@@ -1688,6 +1799,11 @@
         }
       }
     },
+    "crypto-random-string": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz",
+      "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA=="
+    },
     "d64": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/d64/-/d64-1.0.0.tgz",
@@ -1701,6 +1817,11 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "date-and-time": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.12.0.tgz",
+      "integrity": "sha512-n2RJIAp93AucgF/U/Rz5WRS2Hjg5Z+QxscaaMCi6pVZT1JpJKRH+C08vyH/lRR1kxNXnPxgo3lWfd+jCb/UcuQ=="
+    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -1749,6 +1870,12 @@
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
     },
+    "denque": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
+      "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==",
+      "dev": true
+    },
     "depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -2525,6 +2652,19 @@
         "json-bigint": "^0.3.0"
       }
     },
+    "gcs-resumable-upload": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-2.3.2.tgz",
+      "integrity": "sha512-OPS0iAmPCV+r7PziOIhyxmQOzsazFCy76yYDOS/Z80O/7cuny1KMfqDQa2T0jLaL8EreTU7EMZG5pUuqBKgzHA==",
+      "requires": {
+        "abort-controller": "^3.0.0",
+        "configstore": "^5.0.0",
+        "gaxios": "^2.0.0",
+        "google-auth-library": "^5.0.0",
+        "pumpify": "^2.0.0",
+        "stream-events": "^1.0.4"
+      }
+    },
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -2628,8 +2768,7 @@
     "graceful-fs": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
-      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
+      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
-      "dev": true
     },
     "growl": {
       "version": "1.10.5",
@@ -2707,6 +2846,25 @@
       "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
       "dev": true
     },
+    "hash-stream-validation": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.2.tgz",
+      "integrity": "sha512-cMlva5CxWZOrlS/cY0C+9qAzesn5srhFA8IT1VPiHc9bWWBLkJfEUIZr7MWoi89oOOGmpg8ymchaOjiArsGu5A==",
+      "requires": {
+        "through2": "^2.0.0"
+      },
+      "dependencies": {
+        "through2": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
+          "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==",
+          "requires": {
+            "readable-stream": "~2.3.6",
+            "xtend": "~4.0.1"
+          }
+        }
+      }
+    },
     "he": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz",
@@ -2827,8 +2985,7 @@
     "imurmurhash": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="
-      "dev": true
     },
     "indent-string": {
       "version": "4.0.0",
@@ -3330,6 +3487,14 @@
         "statsd-parser": "~0.0.4"
       }
     },
+    "make-dir": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz",
+      "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==",
+      "requires": {
+        "semver": "^6.0.0"
+      }
+    },
     "make-plural": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-4.3.0.tgz",
@@ -3358,6 +3523,13 @@
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
       "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="
     },
+    "memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+      "dev": true,
+      "optional": true
+    },
     "merge-descriptors": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
@@ -3432,8 +3604,7 @@
     "mimic-fn": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
-      "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
+      "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
-      "dev": true
     },
     "minimatch": {
       "version": "3.0.4",
@@ -3511,6 +3682,20 @@
       "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==",
       "optional": true
     },
+    "mongodb": {
+      "version": "3.5.4",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.4.tgz",
+      "integrity": "sha512-xGH41Ig4dkSH5ROGezkgDbsgt/v5zbNUwE3TcFsSbDc6Qn3Qil17dhLsESSDDPTiyFDCPJRpfd4887dtsPgKtA==",
+      "dev": true,
+      "requires": {
+        "bl": "^2.2.0",
+        "bson": "^1.1.1",
+        "denque": "^1.4.1",
+        "require_optional": "^1.0.1",
+        "safe-buffer": "^5.1.2",
+        "saslprep": "^1.0.0"
+      }
+    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -3720,7 +3905,6 @@
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
       "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
-      "dev": true,
       "requires": {
         "mimic-fn": "^2.1.0"
       }
@@ -4798,6 +4982,30 @@
       "integrity": "sha512-AKGr4qvHiryxRb19m3PsLRGuKVAbJLUD7E6eOaHkfKhwc+vSgVOCY5xNvm9EkolBKTOf0GrQAZKLimOCz81Khg==",
       "dev": true
     },
+    "require_optional": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
+      "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
+      "dev": true,
+      "requires": {
+        "resolve-from": "^2.0.0",
+        "semver": "^5.1.0"
+      },
+      "dependencies": {
+        "resolve-from": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
+          "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=",
+          "dev": true
+        },
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "dev": true
+        }
+      }
+    },
     "resolve": {
       "version": "1.15.1",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz",
@@ -4851,11 +5059,6 @@
         }
       }
     },
-    "rimraf": {
-      "version": "2.2.8",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz",
-      "integrity": "sha512-R5KMKHnPAQaZMqLOsyuyUmcIjSeDm+73eoqQpaXA7AZ22BL+6C+1mcUscgOsNd8WVlJuvlgAPsegcx7pjlV0Dg=="
-    },
     "run-async": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.0.tgz",
@@ -4900,6 +5103,16 @@
         "stack-trace": "0.0.9"
       }
     },
+    "saslprep": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
+      "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "sparse-bitfield": "^3.0.3"
+      }
+    },
     "sax": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
@@ -4990,8 +5203,7 @@
     "signal-exit": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-      "integrity": "sha512-meQNNykwecVxdu1RlYMKpQx4+wefIYpmxi6gexo/KAbwquJrBUrBmKYJrE8KFkVQAAVWEnwNdu21PgrD77J3xA==",
+      "integrity": "sha512-meQNNykwecVxdu1RlYMKpQx4+wefIYpmxi6gexo/KAbwquJrBUrBmKYJrE8KFkVQAAVWEnwNdu21PgrD77J3xA=="
-      "dev": true
     },
     "sinon": {
       "version": "7.1.1",
@@ -5055,11 +5267,26 @@
         "to-snake-case": "^1.0.0"
       }
     },
+    "snakeize": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz",
+      "integrity": "sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0="
+    },
     "source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
       "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
     },
+    "sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "memory-pager": "^1.0.2"
+      }
+    },
     "spdx-correct": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz",
@@ -5350,6 +5577,28 @@
         "readable-stream": "2 || 3"
       }
     },
+    "timekeeper": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-2.2.0.tgz",
+      "integrity": "sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==",
+      "dev": true
+    },
+    "tiny-async-pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.1.0.tgz",
+      "integrity": "sha512-jIglyHF/9QdCC3662m/UMVADE6SlocBDpXdFLMZyiAfrw8MSG1pml7lwRtBMT6L/z4dddAxfzw2lpW2Vm42fyQ==",
+      "requires": {
+        "semver": "^5.5.0",
+        "yaassertion": "^1.0.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
     "tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@@ -5455,6 +5704,19 @@
         "mime-types": "~2.1.24"
       }
     },
+    "typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
+    },
+    "typedarray-to-buffer": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
+      "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==",
+      "requires": {
+        "is-typedarray": "^1.0.0"
+      }
+    },
     "typescript": {
       "version": "3.8.2",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.2.tgz",
@@ -5466,6 +5728,14 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz",
       "integrity": "sha512-z4o1fvKUojIWh9XuaVLUDdf86RQiq13AC1dmHbTpoyuu+bquHms76v16CjycCbec87J7z0k//SiQVk0sMdFmpQ=="
     },
+    "unique-string": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz",
+      "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==",
+      "requires": {
+        "crypto-random-string": "^2.0.0"
+      }
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -5686,6 +5956,22 @@
         "mkdirp": "^0.5.1"
       }
     },
+    "write-file-atomic": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz",
+      "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==",
+      "requires": {
+        "imurmurhash": "^0.1.4",
+        "is-typedarray": "^1.0.0",
+        "signal-exit": "^3.0.2",
+        "typedarray-to-buffer": "^3.1.5"
+      }
+    },
+    "xdg-basedir": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz",
+      "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q=="
+    },
     "xml2js": {
       "version": "0.4.19",
       "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
@@ -5700,12 +5986,22 @@
       "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
       "integrity": "sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ=="
     },
+    "xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
+    },
     "y18n": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
       "dev": true
     },
+    "yaassertion": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
+      "integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
+    },
     "yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

services/filestore/package.json

@@ -20,6 +20,7 @@
     "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js"
   },
   "dependencies": {
+    "@google-cloud/storage": "^4.3.0",
     "@overleaf/o-error": "^2.1.0",
     "aws-sdk": "^2.628.0",
     "body-parser": "^1.2.0",
@@ -32,10 +33,10 @@
     "range-parser": "^1.0.2",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.8",
-    "rimraf": "2.2.8",
     "settings-sharelatex": "^1.1.0",
     "stream-buffers": "~0.2.5",
-    "stream-meter": "^1.0.4"
+    "stream-meter": "^1.0.4",
+    "tiny-async-pool": "^1.1.0"
   },
   "devDependencies": {
     "babel-eslint": "^10.0.3",
@@ -55,11 +56,13 @@
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
     "mocha": "5.2.0",
+    "mongodb": "^3.5.4",
     "prettier-eslint": "^9.0.1",
     "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "2.0.3",
     "sinon": "7.1.1",
     "sinon-chai": "^3.3.0",
-    "streamifier": "^0.1.1"
+    "streamifier": "^0.1.1",
+    "timekeeper": "^2.2.0"
   }
 }

services/filestore/test/acceptance/deps/Dockerfile.fake-gcs

@@ -0,0 +1,5 @@
+FROM fsouza/fake-gcs-server
+RUN apk add --update --no-cache curl
+COPY healthcheck.sh /healthcheck.sh
+HEALTHCHECK --interval=1s --timeout=1s --retries=30 CMD /healthcheck.sh http://localhost:9090
+CMD ["--port=9090", "--scheme=http"]

services/filestore/test/acceptance/deps/Dockerfile.s3mock

@@ -0,0 +1,4 @@
+FROM adobe/s3mock
+RUN apk add --update --no-cache curl
+COPY healthcheck.sh /healthcheck.sh
+HEALTHCHECK --interval=1s --timeout=1s --retries=30 CMD /healthcheck.sh http://localhost:9090

services/filestore/test/acceptance/deps/healthcheck.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# health check to allow 404 status code as valid
+STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1)
+# will be 000 on non-http error (e.g. connection failure)
+if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then
+  exit 1
+fi
+exit 0

services/filestore/test/acceptance/js/FilestoreTests.js

@@ -4,6 +4,7 @@ const fs = require('fs')
 const Settings = require('settings-sharelatex')
 const Path = require('path')
 const FilestoreApp = require('./FilestoreApp')
+const TestHelper = require('./TestHelper')
 const rp = require('request-promise-native').defaults({
   resolveWithFullResponse: true
 })
@@ -11,130 +12,32 @@ const S3 = require('aws-sdk/clients/s3')
 const Stream = require('stream')
 const request = require('request')
 const { promisify } = require('util')
+const { Storage } = require('@google-cloud/storage')
 const streamifier = require('streamifier')
 chai.use(require('chai-as-promised'))
+const { ObjectId } = require('mongodb')
+const tk = require('timekeeper')
 
 const fsWriteFile = promisify(fs.writeFile)
 const fsStat = promisify(fs.stat)
 const pipeline = promisify(Stream.pipeline)
 
-async function getMetric(filestoreUrl, metric) {
-  const res = await rp.get(`${filestoreUrl}/metrics`)
-  expect(res.statusCode).to.equal(200)
-  const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
-  const found = metricRegex.exec(res.body)
-  return parseInt(found ? found[1] : 0) || 0
-}
-
 if (!process.env.AWS_ACCESS_KEY_ID) {
   throw new Error('please provide credentials for the AWS S3 test server')
 }
 
-function streamToString(stream) {
-  const chunks = []
-  return new Promise((resolve, reject) => {
-    stream.on('data', chunk => chunks.push(chunk))
-    stream.on('error', reject)
-    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
-    stream.resume()
-  })
-}
-
 // store settings for multiple backends, so that we can test each one.
 // fs will always be available - add others if they are configured
-const BackendSettings = {
+const BackendSettings = require('./TestConfig')
-  FSPersistor: {
-    backend: 'fs',
-    stores: {
-      user_files: Path.resolve(__dirname, '../../../user_files'),
-      public_files: Path.resolve(__dirname, '../../../public_files'),
-      template_files: Path.resolve(__dirname, '../../../template_files')
-    }
-  },
-  S3Persistor: {
-    backend: 's3',
-    s3: {
-      key: process.env.AWS_ACCESS_KEY_ID,
-      secret: process.env.AWS_SECRET_ACCESS_KEY,
-      endpoint: process.env.AWS_S3_ENDPOINT,
-      pathStyle: true,
-      partSize: 100 * 1024 * 1024
-    },
-    stores: {
-      user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
-      template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
-      public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
-    }
-  },
-  FallbackS3ToFSPersistor: {
-    backend: 's3',
-    s3: {
-      key: process.env.AWS_ACCESS_KEY_ID,
-      secret: process.env.AWS_SECRET_ACCESS_KEY,
-      endpoint: process.env.AWS_S3_ENDPOINT,
-      pathStyle: true,
-      partSize: 100 * 1024 * 1024
-    },
-    stores: {
-      user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
-      template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
-      public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
-    },
-    fallback: {
-      backend: 'fs',
-      buckets: {
-        [process.env.AWS_S3_USER_FILES_BUCKET_NAME]: Path.resolve(
-          __dirname,
-          '../../../user_files'
-        ),
-        [process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME]: Path.resolve(
-          __dirname,
-          '../../../public_files'
-        ),
-        [process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME]: Path.resolve(
-          __dirname,
-          '../../../template_files'
-        )
-      }
-    }
-  },
-  FallbackFSToS3Persistor: {
-    backend: 'fs',
-    s3: {
-      key: process.env.AWS_ACCESS_KEY_ID,
-      secret: process.env.AWS_SECRET_ACCESS_KEY,
-      endpoint: process.env.AWS_S3_ENDPOINT,
-      pathStyle: true,
-      partSize: 100 * 1024 * 1024
-    },
-    stores: {
-      user_files: Path.resolve(__dirname, '../../../user_files'),
-      public_files: Path.resolve(__dirname, '../../../public_files'),
-      template_files: Path.resolve(__dirname, '../../../template_files')
-    },
-    fallback: {
-      backend: 's3',
-      buckets: {
-        [Path.resolve(__dirname, '../../../user_files')]: process.env
-          .AWS_S3_USER_FILES_BUCKET_NAME,
-        [Path.resolve(__dirname, '../../../public_files')]: process.env
-          .AWS_S3_PUBLIC_FILES_BUCKET_NAME,
-        [Path.resolve(__dirname, '../../../template_files')]: process.env
-          .AWS_S3_TEMPLATE_FILES_BUCKET_NAME
-      }
-    }
-  }
-}
 
 describe('Filestore', function() {
   this.timeout(1000 * 10)
   const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}`
-  const directoryName = 'directory'
 
   // redefine the test suite for every available backend
   Object.keys(BackendSettings).forEach(backend => {
     describe(backend, function() {
-      let app, previousEgress, previousIngress, projectId
+      let app, previousEgress, previousIngress, metricPrefix, projectId
 
       before(async function() {
         // create the app with the relevant filestore settings
@@ -143,15 +46,38 @@ describe('Filestore', function() {
         await app.runServer()
       })
 
+      if (BackendSettings[backend].gcs) {
+        before(async function() {
+          const storage = new Storage(Settings.filestore.gcs.endpoint)
+          await storage.createBucket(process.env.GCS_USER_FILES_BUCKET_NAME)
+          await storage.createBucket(process.env.GCS_PUBLIC_FILES_BUCKET_NAME)
+          await storage.createBucket(process.env.GCS_TEMPLATE_FILES_BUCKET_NAME)
+          await storage.createBucket(
+            `${process.env.GCS_USER_FILES_BUCKET_NAME}-deleted`
+          )
+          await storage.createBucket(
+            `${process.env.GCS_PUBLIC_FILES_BUCKET_NAME}-deleted`
+          )
+          await storage.createBucket(
+            `${process.env.GCS_TEMPLATE_FILES_BUCKET_NAME}-deleted`
+          )
+        })
+      }
+
       after(async function() {
         return app.stop()
       })
 
       beforeEach(async function() {
-        if (Settings.filestore.backend === 's3') {
+        // retrieve previous metrics from the app
-          previousEgress = await getMetric(filestoreUrl, 's3_egress')
+        if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
+          metricPrefix = Settings.filestore.backend
+          previousEgress = await TestHelper.getMetric(
+            filestoreUrl,
+            `${metricPrefix}_egress`
+          )
         }
-        projectId = `acceptance_tests_${Math.random()}`
+        projectId = ObjectId().toString()
       })
 
       it('should send a 200 for the status endpoint', async function() {
@@ -174,8 +100,8 @@ describe('Filestore', function() {
           '/tmp/filestore_acceptance_tests_file_read.txt'
 
         beforeEach(async function() {
-          fileId = Math.random()
+          fileId = ObjectId().toString()
-          fileUrl = `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileId}`
+          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
           constantFileContent = [
             'hello world',
             `line 2 goes here ${Math.random()}`,
@@ -195,8 +121,11 @@ describe('Filestore', function() {
           // The upload request can bump the ingress metric.
           // The content hash validation might require a full download
           //  in case the ETag field of the upload response is not a md5 sum.
-          if (Settings.filestore.backend === 's3') {
+          if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
-            previousIngress = await getMetric(filestoreUrl, 's3_ingress')
+            previousIngress = await TestHelper.getMetric(
+              filestoreUrl,
+              `${metricPrefix}_ingress`
+            )
           }
         })
 
@@ -252,16 +181,16 @@ describe('Filestore', function() {
         })
 
         it('should be able to copy files', async function() {
-          const newProjectID = `acceptance_tests_copied_project_${Math.random()}`
+          const newProjectID = ObjectId().toString()
-          const newFileId = Math.random()
+          const newFileId = ObjectId().toString()
-          const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}`
+          const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
           const opts = {
             method: 'put',
             uri: newFileUrl,
             json: {
               source: {
                 project_id: projectId,
-                file_id: `${directoryName}/${fileId}`
+                file_id: fileId
               }
             }
           }
@@ -285,15 +214,21 @@ describe('Filestore', function() {
           expect(response.body).to.equal(newContent)
         })
 
-        if (backend === 'S3Persistor') {
+        if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
           it('should record an egress metric for the upload', async function() {
-            const metric = await getMetric(filestoreUrl, 's3_egress')
+            const metric = await TestHelper.getMetric(
+              filestoreUrl,
+              `${metricPrefix}_egress`
+            )
             expect(metric - previousEgress).to.equal(constantFileContent.length)
           })
 
           it('should record an ingress metric when downloading the file', async function() {
             await rp.get(fileUrl)
-            const metric = await getMetric(filestoreUrl, 's3_ingress')
+            const metric = await TestHelper.getMetric(
+              filestoreUrl,
+              `${metricPrefix}_ingress`
+            )
             expect(metric - previousIngress).to.equal(
               constantFileContent.length
             )
@@ -307,15 +242,17 @@ describe('Filestore', function() {
               }
             }
             await rp.get(options)
-            const metric = await getMetric(filestoreUrl, 's3_ingress')
+            const metric = await TestHelper.getMetric(
+              filestoreUrl,
+              `${metricPrefix}_ingress`
+            )
             expect(metric - previousIngress).to.equal(9)
           })
         }
       })
 
       describe('with multiple files', function() {
-        let fileIds, fileUrls
+        let fileIds, fileUrls, projectUrl
-        const directoryName = 'directory'
         const localFileReadPaths = [
           '/tmp/filestore_acceptance_tests_file_read_1.txt',
           '/tmp/filestore_acceptance_tests_file_read_2.txt'
@@ -341,10 +278,11 @@ describe('Filestore', function() {
         })
 
         beforeEach(async function() {
-          fileIds = [Math.random(), Math.random()]
+          projectUrl = `${filestoreUrl}/project/${projectId}`
+          fileIds = [ObjectId().toString(), ObjectId().toString()]
           fileUrls = [
-            `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileIds[0]}`,
+            `${projectUrl}/file/${fileIds[0]}`,
-            `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileIds[1]}`
+            `${projectUrl}/file/${fileIds[1]}`
           ]
 
           const writeStreams = [
@@ -374,14 +312,42 @@ describe('Filestore', function() {
             constantFileContents[0].length + constantFileContents[1].length
           )
         })
+
+        it('should store the files', async function() {
+          for (const index in fileUrls) {
+            await expect(rp.get(fileUrls[index])).to.eventually.have.property(
+              'body',
+              constantFileContents[index]
+            )
+          }
+        })
+
+        it('should be able to delete the project', async function() {
+          await expect(rp.delete(projectUrl)).to.eventually.have.property(
+            'statusCode',
+            204
+          )
+
+          for (const index in fileUrls) {
+            await expect(
+              rp.get(fileUrls[index])
+            ).to.eventually.be.rejected.and.have.property('statusCode', 404)
+          }
+        })
+
+        it('should not delete a partial project id', async function() {
+          await expect(
+            rp.delete(`${filestoreUrl}/project/5`)
+          ).to.eventually.be.rejected.and.have.property('statusCode', 400)
+        })
       })
 
       describe('with a large file', function() {
         let fileId, fileUrl, largeFileContent, error
 
         beforeEach(async function() {
-          fileId = Math.random()
+          fileId = ObjectId().toString()
-          fileUrl = `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileId}`
+          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
 
           largeFileContent = '_wombat_'.repeat(1024 * 1024) // 8 megabytes
           largeFileContent += Math.random()
@@ -414,8 +380,8 @@ describe('Filestore', function() {
 
           beforeEach(async function() {
             constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
-            fileId = Math.random().toString()
+            fileId = ObjectId().toString()
-            bucketName = Math.random().toString()
+            bucketName = ObjectId().toString()
             fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
 
             const s3ClientSettings = {
@@ -450,50 +416,60 @@ describe('Filestore', function() {
         })
       }
 
+      if (backend === 'GcsPersistor') {
+        describe('when deleting a file in GCS', function() {
+          let fileId, fileUrl, content, error, date
+
+          beforeEach(async function() {
+            date = new Date()
+            tk.freeze(date)
+            fileId = ObjectId()
+            fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
+
+            content = '_wombat_' + Math.random()
+
+            const writeStream = request.post(fileUrl)
+            const readStream = streamifier.createReadStream(content)
+            // hack to consume the result to ensure the http request has been fully processed
+            const resultStream = fs.createWriteStream('/dev/null')
+
+            try {
+              await pipeline(readStream, writeStream, resultStream)
+              await rp.delete(fileUrl)
+            } catch (err) {
+              error = err
+            }
+          })
+
+          afterEach(function() {
+            tk.reset()
+          })
+
+          it('should not throw an error', function() {
+            expect(error).not.to.exist
+          })
+
+          it('should copy the file to the deleted-files bucket', async function() {
+            await TestHelper.expectPersistorToHaveFile(
+              app.persistor,
+              `${Settings.filestore.stores.user_files}-deleted`,
+              `${projectId}/${fileId}-${date.toISOString()}`,
+              content
+            )
+          })
+
+          it('should remove the file from the original bucket', async function() {
+            await TestHelper.expectPersistorNotToHaveFile(
+              app.persistor,
+              Settings.filestore.stores.user_files,
+              `${projectId}/${fileId}`
+            )
+          })
+        })
+      }
+
       if (BackendSettings[backend].fallback) {
         describe('with a fallback', function() {
-          async function uploadStringToPersistor(
-            persistor,
-            bucket,
-            key,
-            content
-          ) {
-            const fileStream = streamifier.createReadStream(content)
-            await persistor.promises.sendStream(bucket, key, fileStream)
-          }
-
-          async function getStringFromPersistor(persistor, bucket, key) {
-            const stream = await persistor.promises.getFileStream(
-              bucket,
-              key,
-              {}
-            )
-            return streamToString(stream)
-          }
-
-          async function expectPersistorToHaveFile(
-            persistor,
-            bucket,
-            key,
-            content
-          ) {
-            const foundContent = await getStringFromPersistor(
-              persistor,
-              bucket,
-              key
-            )
-            expect(foundContent).to.equal(content)
-          }
-
-          async function expectPersistorNotToHaveFile(persistor, bucket, key) {
-            await expect(
-              getStringFromPersistor(persistor, bucket, key)
-            ).to.eventually.have.been.rejected.with.property(
-              'name',
-              'NotFoundError'
-            )
-          }
-
           let constantFileContent,
             fileId,
             fileKey,
@@ -503,9 +479,9 @@ describe('Filestore', function() {
 
           beforeEach(function() {
             constantFileContent = `This is yet more file content ${Math.random()}`
-            fileId = Math.random().toString()
+            fileId = ObjectId().toString()
-            fileKey = `${projectId}/${directoryName}/${fileId}`
+            fileKey = `${projectId}/${fileId}`
-            fileUrl = `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileId}`
+            fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
 
             bucket = Settings.filestore.stores.user_files
             fallbackBucket = Settings.filestore.fallback.buckets[bucket]
@@ -513,7 +489,7 @@ describe('Filestore', function() {
 
           describe('with a file in the fallback bucket', function() {
             beforeEach(async function() {
-              await uploadStringToPersistor(
+              await TestHelper.uploadStringToPersistor(
                 app.persistor.fallbackPersistor,
                 fallbackBucket,
                 fileKey,
@@ -522,7 +498,7 @@ describe('Filestore', function() {
             })
 
             it('should not find file in the primary', async function() {
-              await expectPersistorNotToHaveFile(
+              await TestHelper.expectPersistorNotToHaveFile(
                 app.persistor.primaryPersistor,
                 bucket,
                 fileKey
@@ -530,7 +506,7 @@ describe('Filestore', function() {
             })
 
             it('should find the file in the fallback', async function() {
-              await expectPersistorToHaveFile(
+              await TestHelper.expectPersistorToHaveFile(
                 app.persistor.fallbackPersistor,
                 fallbackBucket,
                 fileKey,
@@ -551,7 +527,7 @@ describe('Filestore', function() {
               it('should not copy the file to the primary', async function() {
                 await rp.get(fileUrl)
 
-                await expectPersistorNotToHaveFile(
+                await TestHelper.expectPersistorNotToHaveFile(
                   app.persistor.primaryPersistor,
                   bucket,
                   fileKey
@@ -574,7 +550,7 @@ describe('Filestore', function() {
                 // wait for the file to copy in the background
                 await promisify(setTimeout)(1000)
 
-                await expectPersistorToHaveFile(
+                await TestHelper.expectPersistorToHaveFile(
                   app.persistor.primaryPersistor,
                   bucket,
                   fileKey,
@@ -587,10 +563,10 @@ describe('Filestore', function() {
               let newFileId, newFileUrl, newFileKey, opts
 
               beforeEach(function() {
-                const newProjectID = `acceptance_tests_copied_project_${Math.random()}`
+                const newProjectID = ObjectId().toString()
-                newFileId = Math.random()
+                newFileId = ObjectId().toString()
-                newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}`
+                newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
-                newFileKey = `${newProjectID}/${directoryName}/${newFileId}`
+                newFileKey = `${newProjectID}/${newFileId}`
 
                 opts = {
                   method: 'put',
@@ -598,7 +574,7 @@ describe('Filestore', function() {
                   json: {
                     source: {
                       project_id: projectId,
-                      file_id: `${directoryName}/${fileId}`
+                      file_id: fileId
                     }
                   }
                 }
@@ -613,7 +589,7 @@ describe('Filestore', function() {
                 })
 
                 it('should leave the old file in the old bucket', async function() {
-                  await expectPersistorToHaveFile(
+                  await TestHelper.expectPersistorToHaveFile(
                     app.persistor.fallbackPersistor,
                     fallbackBucket,
                     fileKey,
@@ -622,7 +598,7 @@ describe('Filestore', function() {
                 })
 
                 it('should not create a new file in the old bucket', async function() {
-                  await expectPersistorNotToHaveFile(
+                  await TestHelper.expectPersistorNotToHaveFile(
                     app.persistor.fallbackPersistor,
                     fallbackBucket,
                     newFileKey
@@ -630,7 +606,7 @@ describe('Filestore', function() {
                 })
 
                 it('should create a new file in the new bucket', async function() {
-                  await expectPersistorToHaveFile(
+                  await TestHelper.expectPersistorToHaveFile(
                     app.persistor.primaryPersistor,
                     bucket,
                     newFileKey,
@@ -642,7 +618,7 @@ describe('Filestore', function() {
                   // wait for the file to copy in the background
                   await promisify(setTimeout)(1000)
 
-                  await expectPersistorNotToHaveFile(
+                  await TestHelper.expectPersistorNotToHaveFile(
                     app.persistor.primaryPersistor,
                     bucket,
                     fileKey
@@ -659,7 +635,7 @@ describe('Filestore', function() {
                 })
 
                 it('should leave the old file in the old bucket', async function() {
-                  await expectPersistorToHaveFile(
+                  await TestHelper.expectPersistorToHaveFile(
                     app.persistor.fallbackPersistor,
                     fallbackBucket,
                     fileKey,
@@ -668,7 +644,7 @@ describe('Filestore', function() {
                 })
 
                 it('should not create a new file in the old bucket', async function() {
-                  await expectPersistorNotToHaveFile(
+                  await TestHelper.expectPersistorNotToHaveFile(
                     app.persistor.fallbackPersistor,
                     fallbackBucket,
                     newFileKey
@@ -676,7 +652,7 @@ describe('Filestore', function() {
                 })
 
                 it('should create a new file in the new bucket', async function() {
-                  await expectPersistorToHaveFile(
+                  await TestHelper.expectPersistorToHaveFile(
                     app.persistor.primaryPersistor,
                     bucket,
                     newFileKey,
@@ -688,7 +664,7 @@ describe('Filestore', function() {
                   // wait for the file to copy in the background
                   await promisify(setTimeout)(1000)
 
-                  await expectPersistorToHaveFile(
+                  await TestHelper.expectPersistorToHaveFile(
                     app.persistor.primaryPersistor,
                     bucket,
                     fileKey,
@@ -711,7 +687,7 @@ describe('Filestore', function() {
             })
 
             it('should store the file on the primary', async function() {
-              await expectPersistorToHaveFile(
+              await TestHelper.expectPersistorToHaveFile(
                 app.persistor.primaryPersistor,
                 bucket,
                 fileKey,
@@ -720,10 +696,10 @@ describe('Filestore', function() {
             })
 
             it('should not store the file on the fallback', async function() {
-              await expectPersistorNotToHaveFile(
+              await TestHelper.expectPersistorNotToHaveFile(
                 app.persistor.fallbackPersistor,
                 fallbackBucket,
-                `${projectId}/${directoryName}/${fileId}`
+                `${projectId}/${fileId}`
               )
             })
           })
@@ -731,7 +707,7 @@ describe('Filestore', function() {
           describe('when deleting a file', function() {
             describe('when the file exists on the primary', function() {
               beforeEach(async function() {
-                await uploadStringToPersistor(
+                await TestHelper.uploadStringToPersistor(
                   app.persistor.primaryPersistor,
                   bucket,
                   fileKey,
@@ -750,7 +726,7 @@ describe('Filestore', function() {
 
             describe('when the file exists on the fallback', function() {
               beforeEach(async function() {
-                await uploadStringToPersistor(
+                await TestHelper.uploadStringToPersistor(
                   app.persistor.fallbackPersistor,
                   fallbackBucket,
                   fileKey,
@@ -769,13 +745,13 @@ describe('Filestore', function() {
 
             describe('when the file exists on both the primary and the fallback', function() {
               beforeEach(async function() {
-                await uploadStringToPersistor(
+                await TestHelper.uploadStringToPersistor(
                   app.persistor.primaryPersistor,
                   bucket,
                   fileKey,
                   constantFileContent
                 )
-                await uploadStringToPersistor(
+                await TestHelper.uploadStringToPersistor(
                   app.persistor.fallbackPersistor,
                   fallbackBucket,
                   fileKey,
@@ -812,8 +788,8 @@ describe('Filestore', function() {
         )
 
         beforeEach(async function() {
-          fileId = Math.random()
+          fileId = ObjectId().toString()
-          fileUrl = `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileId}`
+          fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
           const stat = await fsStat(localFileReadPath)
           localFileSize = stat.size
           const writeStream = request.post(fileUrl)
@@ -827,9 +803,12 @@ describe('Filestore', function() {
           expect(response.body.substring(0, 8)).to.equal('%PDF-1.5')
         })
 
-        if (backend === 'S3Persistor') {
+        if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
           it('should record an egress metric for the upload', async function() {
-            const metric = await getMetric(filestoreUrl, 's3_egress')
+            const metric = await TestHelper.getMetric(
+              filestoreUrl,
+              `${metricPrefix}_egress`
+            )
             expect(metric - previousEgress).to.equal(localFileSize)
           })
         }

services/filestore/test/acceptance/js/TestConfig.js

@@ -0,0 +1,113 @@
+const Path = require('path')
+
+// use functions to get a fresh copy, not a reference, each time
+function s3Config() {
+  return {
+    key: process.env.AWS_ACCESS_KEY_ID,
+    secret: process.env.AWS_SECRET_ACCESS_KEY,
+    endpoint: process.env.AWS_S3_ENDPOINT,
+    pathStyle: true,
+    partSize: 100 * 1024 * 1024
+  }
+}
+
+function s3Stores() {
+  return {
+    user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
+    template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
+    public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
+  }
+}
+
+function gcsConfig() {
+  return {
+    endpoint: {
+      apiEndpoint: process.env.GCS_API_ENDPOINT,
+      apiScheme: process.env.GCS_API_SCHEME,
+      projectId: 'fake'
+    },
+    directoryKeyRegex: new RegExp('^[0-9a-fA-F]{24}/[0-9a-fA-F]{24}'),
+    unlockBeforeDelete: false, // fake-gcs does not support this
+    deletedBucketSuffix: '-deleted'
+  }
+}
+
+function gcsStores() {
+  return {
+    user_files: process.env.GCS_USER_FILES_BUCKET_NAME,
+    template_files: process.env.GCS_TEMPLATE_FILES_BUCKET_NAME,
+    public_files: process.env.GCS_PUBLIC_FILES_BUCKET_NAME
+  }
+}
+
+function fsStores() {
+  return {
+    user_files: Path.resolve(__dirname, '../../../user_files'),
+    public_files: Path.resolve(__dirname, '../../../public_files'),
+    template_files: Path.resolve(__dirname, '../../../template_files')
+  }
+}
+
+function fallbackStores(primaryConfig, fallbackConfig) {
+  return {
+    [primaryConfig.user_files]: fallbackConfig.user_files,
+    [primaryConfig.public_files]: fallbackConfig.public_files,
+    [primaryConfig.template_files]: fallbackConfig.template_files
+  }
+}
+
+module.exports = {
+  FSPersistor: {
+    backend: 'fs',
+    stores: fsStores()
+  },
+  S3Persistor: {
+    backend: 's3',
+    s3: s3Config(),
+    stores: s3Stores()
+  },
+  GcsPersistor: {
+    backend: 'gcs',
+    gcs: gcsConfig(),
+    stores: gcsStores()
+  },
+  FallbackS3ToFSPersistor: {
+    backend: 's3',
+    s3: s3Config(),
+    stores: s3Stores(),
+    fallback: {
+      backend: 'fs',
+      buckets: fallbackStores(s3Stores(), fsStores())
+    }
+  },
+  FallbackFSToS3Persistor: {
+    backend: 'fs',
+    s3: s3Config(),
+    stores: fsStores(),
+    fallback: {
+      backend: 's3',
+      buckets: fallbackStores(fsStores(), s3Stores())
+    }
+  },
+  FallbackGcsToS3Persistor: {
+    backend: 'gcs',
+    gcs: gcsConfig(),
+    stores: gcsStores(),
+    s3: s3Config(),
+    fallback: {
+      backend: 's3',
+      buckets: fallbackStores(gcsStores(), s3Stores())
+    }
+  },
+  FallbackS3ToGcsPersistor: {
+    backend: 's3',
+    // can use the same bucket names for gcs and s3 (in tests)
+    stores: s3Stores(),
+    s3: s3Config(),
+    gcs: gcsConfig(),
+    fallback: {
+      backend: 'gcs',
+      buckets: fallbackStores(s3Stores(), gcsStores())
+    }
+  }
+}

services/filestore/test/acceptance/js/TestHelper.js

@@ -0,0 +1,54 @@
+const streamifier = require('streamifier')
+const rp = require('request-promise-native').defaults({
+  resolveWithFullResponse: true
+})
+
+const { expect } = require('chai')
+
+module.exports = {
+  uploadStringToPersistor,
+  getStringFromPersistor,
+  expectPersistorToHaveFile,
+  expectPersistorNotToHaveFile,
+  streamToString,
+  getMetric
+}
+
+async function getMetric(filestoreUrl, metric) {
+  const res = await rp.get(`${filestoreUrl}/metrics`)
+  expect(res.statusCode).to.equal(200)
+  const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
+  const found = metricRegex.exec(res.body)
+  return parseInt(found ? found[1] : 0) || 0
+}
+
+function streamToString(stream) {
+  const chunks = []
+  return new Promise((resolve, reject) => {
+    stream.on('data', chunk => chunks.push(chunk))
+    stream.on('error', reject)
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
+    stream.resume()
+  })
+}
+
+async function uploadStringToPersistor(persistor, bucket, key, content) {
+  const fileStream = streamifier.createReadStream(content)
+  await persistor.promises.sendStream(bucket, key, fileStream)
+}
+
+async function getStringFromPersistor(persistor, bucket, key) {
+  const stream = await persistor.promises.getFileStream(bucket, key, {})
+  return streamToString(stream)
+}
+
+async function expectPersistorToHaveFile(persistor, bucket, key, content) {
+  const foundContent = await getStringFromPersistor(persistor, bucket, key)
+  expect(foundContent).to.equal(content)
+}
+
+async function expectPersistorNotToHaveFile(persistor, bucket, key) {
+  await expect(
+    getStringFromPersistor(persistor, bucket, key)
+  ).to.eventually.have.been.rejected.with.property('name', 'NotFoundError')
+}

services/filestore/test/unit/js/FSPersistorTests.js

@@ -22,15 +22,7 @@ describe('FSPersistorTests', function() {
   const files = ['animals/wombat.tex', 'vegetables/potato.tex']
   const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`]
   const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex']
-  let fs,
+  let fs, stream, LocalFileWriter, FSPersistor, glob, readStream, crypto, Hash
-    rimraf,
-    stream,
-    LocalFileWriter,
-    FSPersistor,
-    glob,
-    readStream,
-    crypto,
-    Hash
 
   beforeEach(function() {
     readStream = {
@@ -46,7 +38,6 @@ describe('FSPersistorTests', function() {
       stat: sinon.stub().yields(null, stat)
     }
     glob = sinon.stub().yields(null, globs)
-    rimraf = sinon.stub().yields()
     stream = { pipeline: sinon.stub().yields() }
     LocalFileWriter = {
       promises: {
@@ -68,12 +59,12 @@ describe('FSPersistorTests', function() {
         './Errors': Errors,
         fs,
         glob,
-        rimraf,
         stream,
         crypto,
         // imported by PersistorHelper but otherwise unused here
         'stream-meter': {},
-        'logger-sharelatex': {}
+        'logger-sharelatex': {},
+        'metrics-sharelatex': {}
       },
       globals: { console }
     })
@@ -270,15 +261,22 @@ describe('FSPersistorTests', function() {
   })
 
   describe('deleteDirectory', function() {
-    it('Should call rmdir(rimraf) with correct options', async function() {
+    it('Should call glob with correct options', async function() {
       await FSPersistor.promises.deleteDirectory(location, files[0])
-      expect(rimraf).to.have.been.calledWith(
+      expect(glob).to.have.been.calledWith(
-        `${location}/${filteredFilenames[0]}`
+        `${location}/${filteredFilenames[0]}*`
       )
     })
 
+    it('Should call unlink on the returned files', async function() {
+      await FSPersistor.promises.deleteDirectory(location, files[0])
+      for (const filename of globs) {
+        expect(fs.unlink).to.have.been.calledWith(filename)
+      }
+    })
+
     it('Should propagate the error', async function() {
-      rimraf.yields(error)
+      glob.yields(error)
       await expect(
         FSPersistor.promises.deleteDirectory(location, files[0])
       ).to.eventually.be.rejected.and.have.property('cause', error)

services/filestore/test/unit/js/FileControllerTests.js

@@ -40,6 +40,7 @@ describe('FileController', function() {
       getFile: sinon.stub().yields(null, fileStream),
       getFileSize: sinon.stub().yields(null, fileSize),
       deleteFile: sinon.stub().yields(),
+      deleteProject: sinon.stub().yields(),
       insertFile: sinon.stub().yields(),
       getDirectorySize: sinon.stub().yields(null, fileSize)
     }
@@ -67,6 +68,7 @@ describe('FileController', function() {
     req = {
       key: key,
       bucket: bucket,
+      project_id: projectId,
       query: {},
       params: {
         project_id: projectId,
@@ -257,6 +259,23 @@ describe('FileController', function() {
     })
   })
 
+  describe('delete project', function() {
+    it('should tell the file handler', function(done) {
+      res.sendStatus = code => {
+        code.should.equal(204)
+        expect(FileHandler.deleteProject).to.have.been.calledWith(bucket, key)
+        done()
+      }
+      FileController.deleteProject(req, res, next)
+    })
+
+    it('should send a 500 if there was an error', function() {
+      FileHandler.deleteProject.yields(error)
+      FileController.deleteProject(req, res, next)
+      expect(next).to.have.been.calledWith(error)
+    })
+  })
+
   describe('directorySize', function() {
     it('should return total directory size bytes', function(done) {
       FileController.directorySize(req, {

services/filestore/test/unit/js/FileHandlerTests.js

@@ -3,6 +3,7 @@ const chai = require('chai')
 const { expect } = chai
 const modulePath = '../../../app/js/FileHandler.js'
 const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongodb')
 
 chai.use(require('sinon-chai'))
 chai.use(require('chai-as-promised'))
@@ -24,8 +25,9 @@ describe('FileHandler', function() {
   }
 
   const bucket = 'my_bucket'
-  const key = 'key/here'
+  const key = `${ObjectId()}/${ObjectId()}`
-  const convertedFolderKey = 'convertedFolder'
+  const convertedFolderKey = `${ObjectId()}/${ObjectId()}`
+  const projectKey = `${ObjectId()}/`
   const sourceStream = 'sourceStream'
   const convertedKey = 'convertedKey'
   const readStream = {
@@ -112,6 +114,14 @@ describe('FileHandler', function() {
         done()
       })
     })
+
+    it('should throw an error when the key is in the wrong format', function(done) {
+      KeyBuilder.getConvertedFolderKey.returns('wombat')
+      FileHandler.insertFile(bucket, key, stream, err => {
+        expect(err).to.exist
+        done()
+      })
+    })
   })
 
   describe('deleteFile', function() {
@@ -135,6 +145,33 @@ describe('FileHandler', function() {
         done()
       })
     })
+
+    it('should throw an error when the key is in the wrong format', function(done) {
+      KeyBuilder.getConvertedFolderKey.returns('wombat')
+      FileHandler.deleteFile(bucket, key, err => {
+        expect(err).to.exist
+        done()
+      })
+    })
+  })
+
+  describe('deleteProject', function() {
+    it('should tell the filestore manager to delete the folder', function(done) {
+      FileHandler.deleteProject(bucket, projectKey, err => {
+        expect(err).not.to.exist
+        expect(
+          PersistorManager.promises.deleteDirectory
+        ).to.have.been.calledWith(bucket, projectKey)
+        done()
+      })
+    })
+
+    it('should throw an error when the key is in the wrong format', function(done) {
+      FileHandler.deleteProject(bucket, 'wombat', err => {
+        expect(err).to.exist
+        done()
+      })
+    })
   })
 
   describe('getFile', function() {

services/filestore/test/unit/js/GcsPersistorTests.js

@@ -0,0 +1,678 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const modulePath = '../../../app/js/GcsPersistor.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongodb')
+const asyncPool = require('tiny-async-pool')
+
+const Errors = require('../../../app/js/Errors')
+
+describe('GcsPersistorTests', function() {
+  const filename = '/wombat/potato.tex'
+  const bucket = 'womBucket'
+  const key = 'monKey'
+  const destKey = 'donKey'
+  const objectSize = 5555
+  const genericError = new Error('guru meditation error')
+  const filesSize = 33
+  const md5 = 'ffffffff00000000ffffffff00000000'
+  const WriteStream = 'writeStream'
+
+  let Metrics,
+    Logger,
+    Storage,
+    Fs,
+    GcsNotFoundError,
+    Meter,
+    MeteredStream,
+    ReadStream,
+    Stream,
+    GcsBucket,
+    GcsFile,
+    GcsPersistor,
+    FileNotFoundError,
+    Hash,
+    settings,
+    crypto,
+    files
+
+  beforeEach(function() {
+    settings = {
+      filestore: {
+        backend: 'gcs',
+        stores: {
+          user_files: 'user_files'
+        },
+        gcs: {
+          directoryKeyRegex: /^[0-9a-fA-F]{24}\/[0-9a-fA-F]{24}/
+        }
+      }
+    }
+
+    files = [
+      {
+        metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' },
+        delete: sinon.stub()
+      },
+      {
+        metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' },
+        delete: sinon.stub()
+      }
+    ]
+
+    ReadStream = {
+      pipe: sinon.stub().returns('readStream'),
+      on: sinon
+        .stub()
+        .withArgs('end')
+        .yields(),
+      removeListener: sinon.stub()
+    }
+
+    Stream = {
+      pipeline: sinon.stub().yields()
+    }
+
+    Metrics = {
+      count: sinon.stub()
+    }
+
+    GcsFile = {
+      delete: sinon.stub().resolves(),
+      createReadStream: sinon.stub().returns(ReadStream),
+      getMetadata: sinon.stub().resolves([files[0].metadata]),
+      createWriteStream: sinon.stub().returns(WriteStream),
+      copy: sinon.stub().resolves(),
+      exists: sinon.stub().resolves([true])
+    }
+
+    GcsBucket = {
+      file: sinon.stub().returns(GcsFile),
+      getFiles: sinon.stub().resolves([files])
+    }
+
+    Storage = class {
+      constructor() {
+        this.interceptors = []
+      }
+    }
+    Storage.prototype.bucket = sinon.stub().returns(GcsBucket)
+
+    GcsNotFoundError = new Error('File not found')
+    GcsNotFoundError.code = 404
+
+    Fs = {
+      createReadStream: sinon.stub().returns(ReadStream)
+    }
+
+    FileNotFoundError = new Error('File not found')
+    FileNotFoundError.code = 'ENOENT'
+
+    MeteredStream = {
+      type: 'metered',
+      on: sinon.stub(),
+      bytes: objectSize
+    }
+    MeteredStream.on.withArgs('finish').yields()
+    MeteredStream.on.withArgs('readable').yields()
+    Meter = sinon.stub().returns(MeteredStream)
+
+    Hash = {
+      end: sinon.stub(),
+      read: sinon.stub().returns(md5),
+      setEncoding: sinon.stub()
+    }
+    crypto = {
+      createHash: sinon.stub().returns(Hash)
+    }
+
+    Logger = {
+      warn: sinon.stub()
+    }
+
+    GcsPersistor = SandboxedModule.require(modulePath, {
+      requires: {
+        '@google-cloud/storage': { Storage },
+        'settings-sharelatex': settings,
+        'logger-sharelatex': Logger,
+        'tiny-async-pool': asyncPool,
+        './Errors': Errors,
+        fs: Fs,
+        'stream-meter': Meter,
+        stream: Stream,
+        'metrics-sharelatex': Metrics,
+        crypto
+      },
+      globals: { console, Buffer }
+    })
+  })
+
+  describe('getFileStream', function() {
+    describe('when called with valid parameters', function() {
+      let stream
+
+      beforeEach(async function() {
+        stream = await GcsPersistor.promises.getFileStream(bucket, key)
+      })
+
+      it('returns a metered stream', function() {
+        expect(stream).to.equal(MeteredStream)
+      })
+
+      it('fetches the right key from the right bucket', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.createReadStream).to.have.been.called
+      })
+
+      it('pipes the stream through the meter', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(
+          ReadStream,
+          MeteredStream
+        )
+      })
+
+      it('records an ingress metric', function() {
+        expect(Metrics.count).to.have.been.calledWith('gcs.ingress', objectSize)
+      })
+    })
+
+    describe('when called with a byte range', function() {
+      let stream
+
+      beforeEach(async function() {
+        stream = await GcsPersistor.promises.getFileStream(bucket, key, {
+          start: 5,
+          end: 10
+        })
+      })
+
+      it('returns a metered stream', function() {
+        expect(stream).to.equal(MeteredStream)
+      })
+
+      it('passes the byte range on to GCS', function() {
+        expect(GcsFile.createReadStream).to.have.been.calledWith({
+          start: 5,
+          end: 11 // we increment the end because Google's 'end' is exclusive
+        })
+      })
+    })
+
+    describe("when the file doesn't exist", function() {
+      let error, stream
+
+      beforeEach(async function() {
+        ReadStream.on = sinon.stub()
+        ReadStream.on.withArgs('error').yields(GcsNotFoundError)
+        try {
+          stream = await GcsPersistor.promises.getFileStream(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('does not return a stream', function() {
+        expect(stream).not.to.exist
+      })
+
+      it('throws a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('wraps the error', function() {
+        expect(error.cause).to.exist
+      })
+
+      it('stores the bucket and key in the error', function() {
+        expect(error.info).to.include({ bucketName: bucket, key: key })
+      })
+    })
+
+    describe('when Gcs encounters an unkown error', function() {
+      let error, stream
+
+      beforeEach(async function() {
+        ReadStream.on = sinon.stub()
+        ReadStream.on.withArgs('error').yields(genericError)
+        try {
+          stream = await GcsPersistor.promises.getFileStream(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('does not return a stream', function() {
+        expect(stream).not.to.exist
+      })
+
+      it('throws a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('wraps the error', function() {
+        expect(error.cause).to.exist
+      })
+
+      it('stores the bucket and key in the error', function() {
+        expect(error.info).to.include({ bucketName: bucket, key: key })
+      })
+    })
+  })
+
+  describe('getFileSize', function() {
+    describe('when called with valid parameters', function() {
+      let size
+
+      beforeEach(async function() {
+        size = await GcsPersistor.promises.getFileSize(bucket, key)
+      })
+
+      it('should return the object size', function() {
+        expect(size).to.equal(files[0].metadata.size)
+      })
+
+      it('should pass the bucket and key to GCS', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.getMetadata).to.have.been.called
+      })
+    })
+
+    describe('when the object is not found', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsFile.getMetadata = sinon.stub().rejects(GcsNotFoundError)
+        try {
+          await GcsPersistor.promises.getFileSize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should return a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(GcsNotFoundError)
+      })
+    })
+
+    describe('when GCS returns an error', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsFile.getMetadata = sinon.stub().rejects(genericError)
+        try {
+          await GcsPersistor.promises.getFileSize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should return a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('sendStream', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return GcsPersistor.promises.sendStream(bucket, key, ReadStream)
+      })
+
+      it('should upload the stream', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.createWriteStream).to.have.been.called
+      })
+
+      it('should not try to create a resumable upload', function() {
+        expect(GcsFile.createWriteStream).to.have.been.calledWith({
+          resumable: false
+        })
+      })
+
+      it('should meter the stream', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(
+          ReadStream,
+          MeteredStream
+        )
+      })
+
+      it('should pipe the metered stream to GCS', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(
+          MeteredStream,
+          WriteStream
+        )
+      })
+
+      it('should record an egress metric', function() {
+        expect(Metrics.count).to.have.been.calledWith('gcs.egress', objectSize)
+      })
+
+      it('calculates the md5 hash of the file', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(ReadStream, Hash)
+      })
+    })
+
+    describe('when a hash is supplied', function() {
+      beforeEach(async function() {
+        return GcsPersistor.promises.sendStream(
+          bucket,
+          key,
+          ReadStream,
+          'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb'
+        )
+      })
+
+      it('should not calculate the md5 hash of the file', function() {
+        expect(Stream.pipeline).not.to.have.been.calledWith(
+          sinon.match.any,
+          Hash
+        )
+      })
+
+      it('sends the hash in base64', function() {
+        expect(GcsFile.createWriteStream).to.have.been.calledWith({
+          validation: 'md5',
+          metadata: {
+            md5Hash: 'qqqqqru7u7uqqqqqu7u7uw=='
+          },
+          resumable: false
+        })
+      })
+
+      it('does not fetch the md5 hash of the uploaded file', function() {
+        expect(GcsFile.getMetadata).not.to.have.been.called
+      })
+    })
+
+    describe('when the upload fails', function() {
+      let error
+      beforeEach(async function() {
+        Stream.pipeline
+          .withArgs(MeteredStream, WriteStream, sinon.match.any)
+          .yields(genericError)
+        try {
+          await GcsPersistor.promises.sendStream(bucket, key, ReadStream)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('throws a WriteError', function() {
+        expect(error).to.be.an.instanceOf(Errors.WriteError)
+      })
+
+      it('wraps the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('sendFile', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return GcsPersistor.promises.sendFile(bucket, key, filename)
+      })
+
+      it('should create a read stream for the file', function() {
+        expect(Fs.createReadStream).to.have.been.calledWith(filename)
+      })
+
+      it('should create a write stream', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.createWriteStream).to.have.been.called
+      })
+
+      it('should upload the stream via the meter', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(
+          ReadStream,
+          MeteredStream
+        )
+        expect(Stream.pipeline).to.have.been.calledWith(
+          MeteredStream,
+          WriteStream
+        )
+      })
+    })
+  })
+
+  describe('copyFile', function() {
+    const destinationFile = 'destFile'
+
+    beforeEach(function() {
+      GcsBucket.file.withArgs(destKey).returns(destinationFile)
+    })
+
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return GcsPersistor.promises.copyFile(bucket, key, destKey)
+      })
+
+      it('should copy the object', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.copy).to.have.been.calledWith(destinationFile)
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsFile.copy = sinon.stub().rejects(GcsNotFoundError)
+        try {
+          await GcsPersistor.promises.copyFile(bucket, key, destKey)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should throw a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+    })
+  })
+
+  describe('deleteFile', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return GcsPersistor.promises.deleteFile(bucket, key)
+      })
+
+      it('should delete the object', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.delete).to.have.been.called
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsFile.delete = sinon.stub().rejects(GcsNotFoundError)
+        try {
+          await GcsPersistor.promises.deleteFile(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should not throw an error', function() {
+        expect(error).not.to.exist
+      })
+    })
+  })
+
+  describe('deleteDirectory', function() {
+    const directoryName = `${ObjectId()}/${ObjectId()}`
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        console.log(key)
+        return GcsPersistor.promises.deleteDirectory(bucket, directoryName)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.getFiles).to.have.been.calledWith({
+          directory: directoryName
+        })
+      })
+
+      it('should delete the files', function() {
+        expect(GcsFile.delete).to.have.been.calledTwice
+      })
+    })
+
+    describe('when there is an error listing the objects', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsBucket.getFiles = sinon.stub().rejects(genericError)
+        try {
+          await GcsPersistor.promises.deleteDirectory(bucket, directoryName)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a WriteError', function() {
+        expect(error).to.be.an.instanceOf(Errors.WriteError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('directorySize', function() {
+    describe('with valid parameters', function() {
+      let size
+
+      beforeEach(async function() {
+        size = await GcsPersistor.promises.directorySize(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
+      })
+
+      it('should return the directory size', function() {
+        expect(size).to.equal(filesSize)
+      })
+    })
+
+    describe('when there are no files', function() {
+      let size
+
+      beforeEach(async function() {
+        GcsBucket.getFiles.resolves([[]])
+        size = await GcsPersistor.promises.directorySize(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
+      })
+
+      it('should return zero', function() {
+        expect(size).to.equal(0)
+      })
+    })
+
+    describe('when there is an error listing the objects', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsBucket.getFiles.rejects(genericError)
+        try {
+          await GcsPersistor.promises.directorySize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('checkIfFileExists', function() {
+    describe('when the file exists', function() {
+      let exists
+
+      beforeEach(async function() {
+        exists = await GcsPersistor.promises.checkIfFileExists(bucket, key)
+      })
+
+      it('should ask the file if it exists', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.exists).to.have.been.called
+      })
+
+      it('should return that the file exists', function() {
+        expect(exists).to.equal(true)
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let exists
+
+      beforeEach(async function() {
+        GcsFile.exists = sinon.stub().resolves([false])
+        exists = await GcsPersistor.promises.checkIfFileExists(bucket, key)
+      })
+
+      it('should get the object header', function() {
+        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
+        expect(GcsBucket.file).to.have.been.calledWith(key)
+        expect(GcsFile.exists).to.have.been.called
+      })
+
+      it('should return that the file does not exist', function() {
+        expect(exists).to.equal(false)
+      })
+    })
+
+    describe('when there is an error', function() {
+      let error
+
+      beforeEach(async function() {
+        GcsFile.exists = sinon.stub().rejects(genericError)
+        try {
+          await GcsPersistor.promises.checkIfFileExists(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+})

services/filestore/test/unit/js/S3PersistorTests.js

@@ -583,48 +583,6 @@ describe('S3PersistorTests', function() {
         })
       })
     })
-
-    describe('when the file does not exist', function() {
-      let error
-
-      beforeEach(async function() {
-        Fs.createReadStream = sinon.stub().throws(FileNotFoundError)
-        try {
-          await S3Persistor.promises.sendFile(bucket, key, filename)
-        } catch (err) {
-          error = err
-        }
-      })
-
-      it('returns a NotFoundError', function() {
-        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
-      })
-
-      it('wraps the error', function() {
-        expect(error.cause).to.equal(FileNotFoundError)
-      })
-    })
-
-    describe('when reading the file throws an error', function() {
-      let error
-
-      beforeEach(async function() {
-        Fs.createReadStream = sinon.stub().throws(genericError)
-        try {
-          await S3Persistor.promises.sendFile(bucket, key, filename)
-        } catch (err) {
-          error = err
-        }
-      })
-
-      it('returns a ReadError', function() {
-        expect(error).to.be.an.instanceOf(Errors.ReadError)
-      })
-
-      it('wraps the error', function() {
-        expect(error.cause).to.equal(genericError)
-      })
-    })
   })
 
   describe('copyFile', function() {
@@ -675,25 +633,6 @@ describe('S3PersistorTests', function() {
         })
       })
     })
-
-    describe('when the file does not exist', function() {
-      let error
-
-      beforeEach(async function() {
-        S3Client.deleteObject = sinon.stub().returns({
-          promise: sinon.stub().rejects(S3NotFoundError)
-        })
-        try {
-          await S3Persistor.promises.deleteFile(bucket, key)
-        } catch (err) {
-          error = err
-        }
-      })
-
-      it('should throw a NotFoundError', function() {
-        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
-      })
-    })
   })
 
   describe('deleteDirectory', function() {