diff --git a/services/filestore/.github/PULL_REQUEST_TEMPLATE.md b/services/filestore/.github/PULL_REQUEST_TEMPLATE.md index ed25ee83c1..12bb2eeb3f 100644 --- a/services/filestore/.github/PULL_REQUEST_TEMPLATE.md +++ b/services/filestore/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,7 @@ - + + + + ### Description diff --git a/services/filestore/Jenkinsfile b/services/filestore/Jenkinsfile index 35bd318ab2..dd741ce239 100644 --- a/services/filestore/Jenkinsfile +++ b/services/filestore/Jenkinsfile @@ -4,10 +4,10 @@ pipeline { agent any environment { - GIT_PROJECT = "filestore-sharelatex" + GIT_PROJECT = "filestore" JENKINS_WORKFLOW = "filestore-sharelatex" TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline" - GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT" + GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT" } triggers { diff --git a/services/filestore/Makefile b/services/filestore/Makefile index db33518816..e83a0696e0 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.24 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -26,14 +26,16 @@ test: test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "filestore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run +test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run + +test_acceptance_run: @[ ! -d test/acceptance ] && echo "filestore has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: $(DOCKER_COMPOSE) down -v -t 0 test_acceptance_pre_run: - @[ ! -f test/acceptance/scripts/pre-run ] && echo "filestore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run + @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "filestore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ diff --git a/services/filestore/README.md b/services/filestore/README.md index cd126ab053..2772b71494 100644 --- a/services/filestore/README.md +++ b/services/filestore/README.md @@ -1,11 +1,9 @@ -filestore-sharelatex +overleaf/filestore ==================== An API for CRUD operations on binary files stored in S3 -[![Build Status](https://travis-ci.org/sharelatex/filestore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/filestore-sharelatex) - -filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/sharelatex/filestore-sharelatex/blob/master/app.coffee . URLs are of the form: +filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/overleaf/filestore/blob/master/app.coffee . URLs are of the form: * `/project/:project_id/file/:file_id` * `/template/:template_id/v/:version/:format` @@ -22,4 +20,4 @@ License The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the `LICENSE` file. -Copyright (c) ShareLaTeX, 2014. +Copyright (c) Overleaf, 2014-2019. diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 3e927a2b51..6c21686826 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -63,22 +63,23 @@ app.use (req, res, next) -> Metrics.injectMetricsRoute(app) +app.head "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFileHead app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile +app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile +app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile -app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile -app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile - +app.head "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFileHead app.get "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile app.get "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile +app.head "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFileHead app.get "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile - -app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile -app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile +app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile +app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index 5be80506f5..168fc68d54 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -1,3 +1,8 @@ +# This module is not used in production, which currently uses +# S3PersistorManager. The intention is to migrate S3PersistorManager to use the +# latest aws-sdk and delete this module so that PersistorManager would load the +# same backend for both the 's3' and 'aws-sdk' options. + logger = require "logger-sharelatex" aws = require "aws-sdk" _ = require "underscore" diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index ef3effbaf2..61e497a4fb 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -1,5 +1,6 @@ logger = require("logger-sharelatex") fs = require("fs") +path = require("path") LocalFileWriter = require("./LocalFileWriter") Errors = require('./Errors') rimraf = require("rimraf") @@ -35,28 +36,38 @@ module.exports = if err? logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" return callback err - @sendFile location, target, fsPath, (err) -> + @sendFile location, target, fsPath, (err) -> # delete the temporary file created above and return the original error LocalFileWriter.deleteFile fsPath, () -> callback(err) # opts may be {start: Number, end: Number} - getFileStream: (location, name, opts, _callback = (err, res)->) -> - callback = _.once _callback + getFileStream: (location, name, opts, callback = (err, res)->) -> filteredName = filterName name logger.log location:location, filteredName:filteredName, "getting file" - sourceStream = fs.createReadStream "#{location}/#{filteredName}", opts - sourceStream.on 'error', (err) -> - logger.err err:err, location:location, filteredName:name, "Error reading from file" - if err.code == 'ENOENT' - return callback new Errors.NotFoundError(err.message), null - else - return callback err, null - sourceStream.on 'readable', () -> - # This can be called multiple times, but the callback wrapper - # ensures the callback is only called once + fs.open "#{location}/#{filteredName}", 'r', (err, fd) -> + if err? + logger.err err:err, location:location, filteredName:name, "Error reading from file" + if err.code == 'ENOENT' + return callback new Errors.NotFoundError(err.message), null + else + return callback err, null + opts.fd = fd + sourceStream = fs.createReadStream null, opts return callback null, sourceStream + getFileSize: (location, filename, callback) -> + fullPath = path.join(location, filterName(filename)) + fs.stat fullPath, (err, stats) -> + if err? + if err.code == 'ENOENT' + logger.log({location:location, filename:filename}, "file not found") + callback(new Errors.NotFoundError(err.message)) + else + logger.err({err:err, location:location, filename:filename}, "failed to stat file") + callback(err) + return + callback(null, stats.size) copyFile: (location, fromName, toName, callback = (err)->)-> filteredFromName=filterName fromName diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee index 60dcc207b7..f98dbd1e49 100644 --- a/services/filestore/app/coffee/FileController.coffee +++ b/services/filestore/app/coffee/FileController.coffee @@ -21,7 +21,7 @@ module.exports = FileController = style: style, } metrics.inc "getFile" - logger.log key:key, bucket:bucket, format:format, style: style, "reciving request to get file" + logger.log key:key, bucket:bucket, format:format, style: style, "receiving request to get file" if req.headers.range? range = FileController._get_range(req.headers.range) options.start = range.start @@ -41,10 +41,24 @@ module.exports = FileController = logger.log key:key, bucket:bucket, format:format, style:style, "sending file to response" fileStream.pipe res + getFileHead: (req, res) -> + {key, bucket} = req + metrics.inc("getFileSize") + logger.log({ key: key, bucket: bucket }, "receiving request to get file metadata") + FileHandler.getFileSize bucket, key, (err, fileSize) -> + if err? + if err instanceof Errors.NotFoundError + res.status(404).end() + else + res.status(500).end() + return + res.set("Content-Length", fileSize) + res.status(200).end() + insertFile: (req, res)-> metrics.inc "insertFile" {key, bucket} = req - logger.log key:key, bucket:bucket, "reciving request to insert file" + logger.log key:key, bucket:bucket, "receiving request to insert file" FileHandler.insertFile bucket, key, req, (err)-> if err? logger.log err: err, key: key, bucket: bucket, "error inserting file" @@ -57,7 +71,7 @@ module.exports = FileController = {key, bucket} = req oldProject_id = req.body.source.project_id oldFile_id = req.body.source.file_id - logger.log key:key, bucket:bucket, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "reciving request to copy file" + logger.log key:key, bucket:bucket, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "receiving request to copy file" PersistorManager.copyFile bucket, "#{oldProject_id}/#{oldFile_id}", key, (err)-> if err? if err instanceof Errors.NotFoundError @@ -71,7 +85,7 @@ module.exports = FileController = deleteFile: (req, res)-> metrics.inc "deleteFile" {key, bucket} = req - logger.log key:key, bucket:bucket, "reciving request to delete file" + logger.log key:key, bucket:bucket, "receiving request to delete file" FileHandler.deleteFile bucket, key, (err)-> if err? logger.log err:err, key:key, bucket:bucket, "something went wrong deleting file" @@ -90,7 +104,7 @@ module.exports = FileController = directorySize: (req, res)-> metrics.inc "projectSize" {project_id, bucket} = req - logger.log project_id:project_id, bucket:bucket, "reciving request to project size" + logger.log project_id:project_id, bucket:bucket, "receiving request to project size" FileHandler.getDirectorySize bucket, project_id, (err, size)-> if err? logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index 6189c8a906..cb8d78a0fe 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -31,6 +31,9 @@ module.exports = FileHandler = else @_getConvertedFile bucket, key, opts, callback + getFileSize: (bucket, key, callback) -> + PersistorManager.getFileSize(bucket, key, callback) + _getStandardFile: (bucket, key, opts, callback)-> PersistorManager.getFileStream bucket, key, opts, (err, fileStream)-> if err? and !(err instanceof Errors.NotFoundError) diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 2b183730d6..89522a4643 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -1,3 +1,7 @@ +# This module is the one which is used in production. It needs to be migrated +# to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for +# details. The knox library is unmaintained and has bugs. + http = require('http') http.globalAgent.maxSockets = 300 https = require('https') @@ -5,6 +9,7 @@ https.globalAgent.maxSockets = 300 settings = require("settings-sharelatex") request = require("request") logger = require("logger-sharelatex") +metrics = require("metrics-sharelatex") fs = require("fs") knox = require("knox") path = require("path") @@ -12,10 +17,15 @@ LocalFileWriter = require("./LocalFileWriter") Errors = require("./Errors") _ = require("underscore") awsS3 = require "aws-sdk/clients/s3" +URL = require('url') thirtySeconds = 30 * 1000 buildDefaultOptions = (bucketName, method, key)-> + if settings.filestore.s3.endpoint + endpoint = "#{settings.filestore.s3.endpoint}/#{bucketName}" + else + endpoint = "https://#{bucketName}.s3.amazonaws.com" return { aws: key: settings.filestore.s3.key @@ -23,23 +33,51 @@ buildDefaultOptions = (bucketName, method, key)-> bucket: bucketName method: method timeout: thirtySeconds - uri:"https://#{bucketName}.s3.amazonaws.com/#{key}" + uri:"#{endpoint}/#{key}" } -s3 = new awsS3({ - credentials: - accessKeyId: settings.filestore.s3.key, - secretAccessKey: settings.filestore.s3.secret -}) +getS3Options = (credentials) -> + options = + credentials: + accessKeyId: credentials.auth_key + secretAccessKey: credentials.auth_secret + + if settings.filestore.s3.endpoint + endpoint = URL.parse(settings.filestore.s3.endpoint) + options.endpoint = settings.filestore.s3.endpoint + options.sslEnabled = endpoint.protocol == 'https' + + return options + +defaultS3Client = new awsS3(getS3Options({ + auth_key: settings.filestore.s3.key, + auth_secret: settings.filestore.s3.secret +})) + +getS3Client = (credentials) -> + if credentials? + return new awsS3(getS3Options(credentials)) + else + return defaultS3Client + +getKnoxClient = (bucketName) => + options = + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: bucketName + if settings.filestore.s3.endpoint + endpoint = URL.parse(settings.filestore.s3.endpoint) + options.endpoint = endpoint.hostname + options.port = endpoint.port + return knox.createClient(options) module.exports = sendFile: (bucketName, key, fsPath, callback)-> - s3Client = knox.createClient - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: bucketName + s3Client = getKnoxClient(bucketName) + uploaded = 0 putEventEmiter = s3Client.putFile fsPath, key, (err, res)-> + metrics.count 's3.egress', uploaded if err? logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath,"something went wrong uploading file to s3" return callback(err) @@ -54,6 +92,8 @@ module.exports = putEventEmiter.on "error", (err)-> logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file" callback err + putEventEmiter.on "progress", (progress)-> + uploaded = progress.written sendStream: (bucketName, key, readStream, callback)-> logger.log bucketName:bucketName, key:key, "sending file to s3" @@ -71,36 +111,69 @@ module.exports = # opts may be {start: Number, end: Number} getFileStream: (bucketName, key, opts, callback = (err, res)->)-> opts = opts || {} - headers = {} - if opts.start? and opts.end? - headers['Range'] = "bytes=#{opts.start}-#{opts.end}" - callback = _.once callback + callback = _.once(callback) logger.log bucketName:bucketName, key:key, "getting file from s3" - s3Client = knox.createClient - key: opts.credentials?.auth_key || settings.filestore.s3.key - secret: opts.credentials?.auth_secret || settings.filestore.s3.secret - bucket: bucketName - s3Stream = s3Client.get(key, headers) - s3Stream.end() - s3Stream.on 'response', (res) -> - if res.statusCode in [403, 404] + + s3 = getS3Client(opts.credentials) + s3Params = { + Bucket: bucketName + Key: key + } + if opts.start? and opts.end? + s3Params['Range'] = "bytes=#{opts.start}-#{opts.end}" + s3Request = s3.getObject(s3Params) + + s3Request.on 'httpHeaders', (statusCode, headers, response, statusMessage) => + if statusCode in [403, 404] # S3 returns a 403 instead of a 404 when the user doesn't have # permission to list the bucket contents. - logger.log bucketName:bucketName, key:key, "file not found in s3" - return callback new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null - else if res.statusCode not in [200, 206] - logger.log bucketName:bucketName, key:key, "error getting file from s3: #{res.statusCode}" - return callback new Error("Got non-200 response from S3: #{res.statusCode}"), null - else - return callback null, res - s3Stream.on 'error', (err) -> - logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3" - callback err + logger.log({ bucketName: bucketName, key: key }, "file not found in s3") + return callback(new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null) + if statusCode not in [200, 206] + logger.log({bucketName: bucketName, key: key }, "error getting file from s3: #{statusCode}") + return callback(new Error("Got non-200 response from S3: #{statusCode} #{statusMessage}"), null) + stream = response.httpResponse.createUnbufferedStream() + stream.on 'data', (data) -> + metrics.count 's3.ingress', data.byteLength + + callback(null, stream) + + s3Request.on 'error', (err) => + logger.err({ err: err, bucketName: bucketName, key: key }, "error getting file stream from s3") + callback(err) + + s3Request.send() + + getFileSize: (bucketName, key, callback) -> + logger.log({ bucketName: bucketName, key: key }, "getting file size from S3") + s3 = getS3Client() + s3.headObject { Bucket: bucketName, Key: key }, (err, data) -> + if err? + if err.statusCode in [403, 404] + # S3 returns a 403 instead of a 404 when the user doesn't have + # permission to list the bucket contents. + logger.log({ + bucketName: bucketName, + key: key + }, "file not found in s3") + callback( + new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}") + ) + else + logger.err({ + bucketName: bucketName, + key: key, + err: err + }, "error performing S3 HeadObject") + callback(err) + return + callback(null, data.ContentLength) copyFile: (bucketName, sourceKey, destKey, callback)-> logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3" source = bucketName + '/' + sourceKey # use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) + s3 = getS3Client() s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) -> if err? if err.code is 'NoSuchKey' @@ -127,10 +200,7 @@ module.exports = _callback = () -> logger.log key: key, bucketName: bucketName, "deleting directory" - s3Client = knox.createClient - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: bucketName + s3Client = getKnoxClient(bucketName) s3Client.list prefix:key, (err, data)-> if err? logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" @@ -156,10 +226,7 @@ module.exports = directorySize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" - s3Client = knox.createClient - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: bucketName + s3Client = getKnoxClient(bucketName) s3Client.list prefix:key, (err, data)-> if err? logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index 0ba90519b3..dc8c156383 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -5,4 +5,6 @@ filestore --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --build-target=docker ---script-version=1.1.12 +--script-version=1.1.24 +--env-pass-through= +--public-repo=True diff --git a/services/filestore/config/settings.defaults.coffee b/services/filestore/config/settings.defaults.coffee index 550cfd2694..29010e85ee 100644 --- a/services/filestore/config/settings.defaults.coffee +++ b/services/filestore/config/settings.defaults.coffee @@ -16,6 +16,7 @@ settings = s3: key: process.env['AWS_KEY'] secret: process.env['AWS_SECRET'] + endpoint: process.env['AWS_S3_ENDPOINT'] stores: user_files: process.env['AWS_S3_USER_FILES_BUCKET_NAME'] template_files: process.env['AWS_S3_TEMPLATE_FILES_BUCKET_NAME'] diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index e7ac6e84a7..42c6ae37b5 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -1,15 +1,17 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.24 -version: "2" +version: "2.1" services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node command: npm run test:unit:_run + environment: + NODE_ENV: test test_acceptance: @@ -20,11 +22,23 @@ services: REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres - MOCHA_GREP: ${MOCHA_GREP} ENABLE_CONVERSIONS: "true" + MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test + USE_PROM_METRICS: "true" + AWS_KEY: fake + AWS_SECRET: fake + AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files + AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files + AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files + AWS_S3_ENDPOINT: http://fakes3:9090 depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + fakes3: + condition: service_healthy user: node command: npm run test:acceptance:_run @@ -43,3 +57,11 @@ services: mongo: image: mongo:3.4 + + fakes3: + image: adobe/s3mock + environment: + - initialBuckets=fake_user_files,fake_template_files,fake_public_files + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9090"] + diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 60d387bf51..65f18f4d78 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.24 -version: "2" +version: "2.1" services: test_unit: @@ -13,6 +13,7 @@ services: working_dir: /app environment: MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test command: npm run test:unit user: node @@ -28,10 +29,23 @@ services: POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} ENABLE_CONVERSIONS: "true" + LOG_LEVEL: ERROR + NODE_ENV: test + USE_PROM_METRICS: "true" + AWS_KEY: fake + AWS_SECRET: fake + AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files + AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files + AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files + AWS_S3_ENDPOINT: http://fakes3:9090 user: node depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + fakes3: + condition: service_healthy command: npm run test:acceptance @@ -50,3 +64,11 @@ services: mongo: image: mongo:3.4 + fakes3: + image: adobe/s3mock + environment: + - initialBuckets=fake_user_files,fake_template_files,fake_public_files + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9090"] + + diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 7caf791bae..66670c6e69 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -3,19 +3,19 @@ "version": "0.1.4", "dependencies": { "@google-cloud/common": { - "version": "0.27.0", - "from": "@google-cloud/common@>=0.27.0 <0.28.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz" + "version": "0.32.1", + "from": "@google-cloud/common@>=0.32.0 <0.33.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz" }, "@google-cloud/debug-agent": { - "version": "3.0.1", + "version": "3.2.0", "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", "dependencies": { "coffeescript": { - "version": "2.3.2", + "version": "2.4.1", "from": "coffeescript@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz" + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz" } } }, @@ -29,15 +29,47 @@ "from": "@google-cloud/common@>=0.26.0 <0.27.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" }, + "@google-cloud/promisify": { + "version": "0.3.1", + "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + }, + "arrify": { + "version": "1.0.1", + "from": "arrify@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + }, + "gcp-metadata": { + "version": "0.9.3", + "from": "gcp-metadata@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + }, + "google-auth-library": { + "version": "2.0.2", + "from": "google-auth-library@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "dependencies": { + "gcp-metadata": { + "version": "0.7.0", + "from": "gcp-metadata@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + } + } + }, "nan": { - "version": "2.12.1", + "version": "2.14.0", "from": "nan@>=2.11.1 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz" + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz" }, "readable-stream": { - "version": "3.1.1", + "version": "3.4.0", "from": "readable-stream@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz" + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz" + }, + "semver": { + "version": "5.7.0", + "from": "semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" }, "string_decoder": { "version": "1.2.0", @@ -45,37 +77,27 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz" }, "through2": { - "version": "3.0.0", + "version": "3.0.1", "from": "through2@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz" } } }, "@google-cloud/projectify": { - "version": "0.3.2", - "from": "@google-cloud/projectify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.2.tgz" + "version": "0.3.3", + "from": "@google-cloud/projectify@>=0.3.3 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz" }, "@google-cloud/promisify": { - "version": "0.3.1", - "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + "version": "0.4.0", + "from": "@google-cloud/promisify@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz" }, "@google-cloud/trace-agent": { - "version": "3.5.2", + "version": "3.6.1", "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.2.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", "dependencies": { - "@google-cloud/common": { - "version": "0.30.2", - "from": "@google-cloud/common@>=0.30.0 <0.31.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.30.2.tgz" - }, - "google-auth-library": { - "version": "3.0.1", - "from": "google-auth-library@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.0.1.tgz" - }, "uuid": { "version": "3.3.2", "from": "uuid@>=3.0.1 <4.0.0", @@ -134,14 +156,46 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" }, "@sindresorhus/is": { - "version": "0.13.0", - "from": "@sindresorhus/is@>=0.13.0 <0.14.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz" + "version": "0.15.0", + "from": "@sindresorhus/is@>=0.15.0 <0.16.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz" + }, + "@sinonjs/commons": { + "version": "1.4.0", + "from": "@sinonjs/commons@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.4.0.tgz", + "dev": true + }, + "@sinonjs/formatio": { + "version": "3.2.1", + "from": "@sinonjs/formatio@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-3.2.1.tgz", + "dev": true, + "dependencies": { + "@sinonjs/samsam": { + "version": "3.3.2", + "from": "@sinonjs/samsam@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.3.2.tgz", + "dev": true + } + } + }, + "@sinonjs/samsam": { + "version": "2.1.3", + "from": "@sinonjs/samsam@>=2.1.2 <3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-2.1.3.tgz", + "dev": true + }, + "@sinonjs/text-encoding": { + "version": "0.7.1", + "from": "@sinonjs/text-encoding@>=0.7.1 <0.8.0", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", + "dev": true }, "@types/caseless": { - "version": "0.12.1", + "version": "0.12.2", "from": "@types/caseless@*", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz" + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz" }, "@types/console-log-level": { "version": "1.4.0", @@ -164,9 +218,9 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" }, "@types/node": { - "version": "10.12.20", + "version": "12.0.8", "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.20.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz" }, "@types/request": { "version": "2.48.1", @@ -183,6 +237,11 @@ "from": "@types/tough-cookie@*", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz" }, + "abort-controller": { + "version": "3.0.0", + "from": "abort-controller@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz" + }, "accept-encoding": { "version": "0.1.0", "from": "accept-encoding@>=0.1.0 <0.2.0", @@ -194,29 +253,35 @@ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz" }, "acorn": { - "version": "5.7.3", - "from": "acorn@>=5.0.3 <6.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz" + "version": "6.1.1", + "from": "acorn@>=6.0.0 <7.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz" }, "agent-base": { - "version": "4.2.1", + "version": "4.3.0", "from": "agent-base@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz" + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz" }, "ajv": { - "version": "6.7.0", + "version": "6.10.0", "from": "ajv@>=6.5.5 <7.0.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz" + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz" }, "array-flatten": { "version": "1.1.1", "from": "array-flatten@1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" }, + "array-from": { + "version": "2.1.1", + "from": "array-from@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz", + "dev": true + }, "arrify": { - "version": "1.0.1", - "from": "arrify@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + "version": "2.0.1", + "from": "arrify@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz" }, "asn1": { "version": "0.2.4", @@ -228,6 +293,12 @@ "from": "assert-plus@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" }, + "assertion-error": { + "version": "1.1.0", + "from": "assertion-error@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "dev": true + }, "async": { "version": "0.2.10", "from": "async@>=0.2.10 <0.3.0", @@ -236,7 +307,14 @@ "async-listener": { "version": "0.6.10", "from": "async-listener@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz" + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "dependencies": { + "semver": { + "version": "5.7.0", + "from": "semver@>=5.3.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + } + } }, "asynckit": { "version": "0.4.0", @@ -271,9 +349,9 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz" }, "axios": { - "version": "0.18.0", + "version": "0.18.1", "from": "axios@>=0.18.0 <0.19.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz" + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz" }, "balanced-match": { "version": "1.0.0", @@ -301,9 +379,9 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" }, "bindings": { - "version": "1.4.0", + "version": "1.5.0", "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz" }, "bintrees": { "version": "1.0.1", @@ -346,14 +424,15 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" }, "builtin-modules": { - "version": "3.0.0", + "version": "3.1.0", "from": "builtin-modules@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz" }, "bunyan": { "version": "1.5.1", "from": "bunyan@1.5.1", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz" + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz", + "dev": true }, "bytes": { "version": "3.0.0", @@ -365,10 +444,22 @@ "from": "caseless@>=0.3.0 <0.4.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.3.0.tgz" }, + "chai": { + "version": "4.2.0", + "from": "chai@4.2.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", + "dev": true + }, + "check-error": { + "version": "1.0.2", + "from": "check-error@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "dev": true + }, "coffee-script": { - "version": "1.12.4", - "from": "coffee-script@1.12.4", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.12.4.tgz" + "version": "1.6.0", + "from": "coffee-script@1.6.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" }, "combined-stream": { "version": "0.0.7", @@ -381,9 +472,9 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" }, "console-log-level": { - "version": "1.4.0", + "version": "1.4.1", "from": "console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz" }, "content-disposition": { "version": "0.5.2", @@ -435,10 +526,16 @@ "from": "debug@2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" }, + "deep-eql": { + "version": "3.0.1", + "from": "deep-eql@>=3.0.1 <4.0.0", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "dev": true + }, "delay": { - "version": "4.1.0", + "version": "4.3.0", "from": "delay@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz" + "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz" }, "delayed-stream": { "version": "0.0.5", @@ -455,16 +552,23 @@ "from": "destroy@>=1.0.4 <1.1.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" }, + "diff": { + "version": "3.5.0", + "from": "diff@>=3.5.0 <4.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "dev": true + }, "dtrace-provider": { "version": "0.6.0", "from": "dtrace-provider@>=0.6.0 <0.7.0", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "dev": true, "optional": true }, "duplexify": { - "version": "3.6.1", + "version": "3.7.1", "from": "duplexify@>=3.6.0 <4.0.0", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "dependencies": { "readable-stream": { "version": "2.3.6", @@ -484,9 +588,9 @@ "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" }, "ecdsa-sig-formatter": { - "version": "1.0.10", - "from": "ecdsa-sig-formatter@1.0.10", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz" + "version": "1.0.11", + "from": "ecdsa-sig-formatter@1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz" }, "ee-first": { "version": "1.1.1", @@ -514,9 +618,9 @@ "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" }, "es6-promise": { - "version": "4.2.5", + "version": "4.2.8", "from": "es6-promise@>=4.0.3 <5.0.0", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz" + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz" }, "es6-promisify": { "version": "5.0.0", @@ -538,6 +642,11 @@ "from": "etag@>=1.8.1 <1.9.0", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" }, + "event-target-shim": { + "version": "5.0.1", + "from": "event-target-shim@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" + }, "events": { "version": "1.1.1", "from": "events@1.1.1", @@ -640,9 +749,9 @@ "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" }, "follow-redirects": { - "version": "1.6.1", - "from": "follow-redirects@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", + "version": "1.5.10", + "from": "follow-redirects@1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", "dependencies": { "debug": { "version": "3.1.0", @@ -689,14 +798,20 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" }, "gaxios": { - "version": "1.2.7", - "from": "gaxios@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.2.7.tgz" + "version": "1.8.4", + "from": "gaxios@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz" }, "gcp-metadata": { - "version": "0.9.3", - "from": "gcp-metadata@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + "version": "1.0.0", + "from": "gcp-metadata@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz" + }, + "get-func-name": { + "version": "2.0.0", + "from": "get-func-name@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "dev": true }, "getpass": { "version": "0.1.7", @@ -715,21 +830,21 @@ "optional": true }, "google-auth-library": { - "version": "2.0.2", - "from": "google-auth-library@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "version": "3.1.2", + "from": "google-auth-library@>=3.1.1 <4.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", "dependencies": { - "gcp-metadata": { - "version": "0.7.0", - "from": "gcp-metadata@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + "semver": { + "version": "5.7.0", + "from": "semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" } } }, "google-p12-pem": { - "version": "1.0.3", + "version": "1.0.4", "from": "google-p12-pem@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz" }, "graceful-fs": { "version": "4.1.11", @@ -737,14 +852,14 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz" }, "gtoken": { - "version": "2.3.2", - "from": "gtoken@>=2.3.0 <3.0.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.2.tgz", + "version": "2.3.3", + "from": "gtoken@>=2.3.2 <3.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", "dependencies": { "mime": { - "version": "2.4.0", + "version": "2.4.4", "from": "mime@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz" } } }, @@ -809,9 +924,9 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" }, "ms": { - "version": "2.1.1", + "version": "2.1.2", "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" } } }, @@ -846,9 +961,9 @@ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" }, "is-buffer": { - "version": "1.1.6", - "from": "is-buffer@>=1.1.5 <2.0.0", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz" + "version": "2.0.3", + "from": "is-buffer@>=2.0.2 <3.0.0", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz" }, "is-typedarray": { "version": "1.0.0", @@ -905,15 +1020,21 @@ "from": "jsprim@>=1.2.2 <2.0.0", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz" }, + "just-extend": { + "version": "4.0.2", + "from": "just-extend@>=4.0.2 <5.0.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", + "dev": true + }, "jwa": { - "version": "1.2.0", - "from": "jwa@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.2.0.tgz" + "version": "1.4.1", + "from": "jwa@>=1.4.1 <2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz" }, "jws": { - "version": "3.2.1", + "version": "3.2.2", "from": "jws@>=3.1.5 <4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.1.tgz" + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz" }, "klaw": { "version": "1.3.1", @@ -932,31 +1053,54 @@ } } }, + "lodash": { + "version": "4.17.11", + "from": "lodash@>=4.17.11 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "dev": true + }, + "lodash.get": { + "version": "4.4.2", + "from": "lodash.get@>=4.4.2 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "dev": true + }, "lodash.pickby": { "version": "4.6.0", "from": "lodash.pickby@>=4.6.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" }, "logger-sharelatex": { - "version": "1.6.0", - "from": "logger-sharelatex@1.6.0", - "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.6.0.tgz", + "version": "1.7.0", + "from": "logger-sharelatex@1.7.0", + "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", "dependencies": { + "bunyan": { + "version": "1.8.12", + "from": "bunyan@1.8.12", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz" + }, "caseless": { "version": "0.12.0", "from": "caseless@>=0.12.0 <0.13.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz" }, "combined-stream": { - "version": "1.0.7", + "version": "1.0.8", "from": "combined-stream@>=1.0.6 <1.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz" + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" }, "delayed-stream": { "version": "1.0.0", "from": "delayed-stream@>=1.0.0 <1.1.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" }, + "dtrace-provider": { + "version": "0.8.7", + "from": "dtrace-provider@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", + "optional": true + }, "forever-agent": { "version": "0.6.1", "from": "forever-agent@>=0.6.1 <0.7.0", @@ -994,6 +1138,12 @@ } } }, + "lolex": { + "version": "3.1.0", + "from": "lolex@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-3.1.0.tgz", + "dev": true + }, "long": { "version": "4.0.0", "from": "long@>=4.0.0 <5.0.0", @@ -1035,15 +1185,10 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.1.1", - "from": "metrics-sharelatex@2.1.1", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.1.1.tgz", + "version": "2.2.0", + "from": "metrics-sharelatex@2.2.0", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.2.0.tgz", "dependencies": { - "coffee-script": { - "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" - }, "underscore": { "version": "1.6.0", "from": "underscore@>=1.6.0 <1.7.0", @@ -1123,6 +1268,12 @@ "from": "module-details-from-path@>=1.0.3 <2.0.0", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" }, + "moment": { + "version": "2.24.0", + "from": "moment@>=2.10.6 <3.0.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "optional": true + }, "ms": { "version": "2.0.0", "from": "ms@2.0.0", @@ -1165,15 +1316,41 @@ "from": "negotiator@0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" }, + "nise": { + "version": "1.5.0", + "from": "nise@>=1.4.6 <2.0.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.0.tgz", + "dev": true, + "dependencies": { + "isarray": { + "version": "0.0.1", + "from": "isarray@0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "dev": true + }, + "lolex": { + "version": "4.1.0", + "from": "lolex@>=4.1.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.1.0.tgz", + "dev": true + }, + "path-to-regexp": { + "version": "1.7.0", + "from": "path-to-regexp@>=1.7.0 <2.0.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", + "dev": true + } + } + }, "node-fetch": { - "version": "2.3.0", - "from": "node-fetch@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz" + "version": "2.6.0", + "from": "node-fetch@>=2.3.0 <3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz" }, "node-forge": { - "version": "0.7.6", - "from": "node-forge@>=0.7.4 <0.8.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz" + "version": "0.8.4", + "from": "node-forge@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz" }, "node-transloadit": { "version": "0.0.4", @@ -1228,14 +1405,14 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz" }, "p-limit": { - "version": "2.1.0", - "from": "p-limit@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz" + "version": "2.2.0", + "from": "p-limit@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz" }, "p-try": { - "version": "2.0.0", + "version": "2.2.0", "from": "p-try@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" }, "parse-duration": { "version": "0.1.1", @@ -1243,9 +1420,9 @@ "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz" }, "parse-ms": { - "version": "2.0.0", + "version": "2.1.0", "from": "parse-ms@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz" }, "parseurl": { "version": "1.3.2", @@ -1267,6 +1444,12 @@ "from": "path-to-regexp@0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" }, + "pathval": { + "version": "1.1.0", + "from": "pathval@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", + "dev": true + }, "performance-now": { "version": "2.1.0", "from": "performance-now@>=2.1.0 <3.0.0", @@ -1293,14 +1476,21 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" }, "prom-client": { - "version": "11.2.1", + "version": "11.5.1", "from": "prom-client@>=11.1.3 <12.0.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz" + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz" }, "protobufjs": { "version": "6.8.8", "from": "protobufjs@>=6.8.6 <6.9.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz" + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "dependencies": { + "@types/node": { + "version": "10.14.9", + "from": "@types/node@>=10.1.0 <11.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz" + } + } }, "proxy-addr": { "version": "2.0.4", @@ -1308,9 +1498,9 @@ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz" }, "psl": { - "version": "1.1.31", + "version": "1.1.32", "from": "psl@>=1.1.24 <2.0.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz" + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.32.tgz" }, "punycode": { "version": "1.3.2", @@ -1333,9 +1523,9 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" }, "raven": { - "version": "1.2.1", - "from": "raven@>=1.1.3 <2.0.0", - "resolved": "https://registry.npmjs.org/raven/-/raven-1.2.1.tgz", + "version": "1.1.3", + "from": "raven@1.1.3", + "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz", "dependencies": { "uuid": { "version": "3.0.0", @@ -1398,9 +1588,21 @@ } }, "require-in-the-middle": { - "version": "3.1.0", - "from": "require-in-the-middle@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-3.1.0.tgz" + "version": "4.0.0", + "from": "require-in-the-middle@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.0.tgz", + "dependencies": { + "debug": { + "version": "4.1.1", + "from": "debug@>=4.1.1 <5.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + }, + "ms": { + "version": "2.1.2", + "from": "ms@^2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + } + } }, "require-like": { "version": "0.1.2", @@ -1409,9 +1611,9 @@ "dev": true }, "resolve": { - "version": "1.10.0", - "from": "resolve@>=1.5.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz" + "version": "1.11.0", + "from": "resolve@>=1.10.0 <2.0.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz" }, "response": { "version": "0.14.0", @@ -1468,9 +1670,9 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz" }, "semver": { - "version": "5.6.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz" + "version": "6.1.1", + "from": "semver@>=6.0.0 <7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz" }, "send": { "version": "0.16.2", @@ -1511,6 +1713,12 @@ "from": "shimmer@>=1.2.0 <2.0.0", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz" }, + "sinon": { + "version": "7.1.1", + "from": "sinon@7.1.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.1.1.tgz", + "dev": true + }, "sntp": { "version": "0.1.4", "from": "sntp@>=0.1.0 <0.2.0", @@ -1583,10 +1791,11 @@ "from": "string_decoder@>=0.10.0 <0.11.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, - "symbol-observable": { - "version": "1.2.0", - "from": "symbol-observable@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz" + "supports-color": { + "version": "5.5.0", + "from": "supports-color@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "dev": true }, "tdigest": { "version": "0.1.1", @@ -1649,6 +1858,12 @@ "from": "tweetnacl@>=0.14.0 <0.15.0", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz" }, + "type-detect": { + "version": "4.0.8", + "from": "type-detect@>=4.0.5 <5.0.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "dev": true + }, "type-is": { "version": "1.6.16", "from": "type-is@>=1.6.16 <1.7.0", diff --git a/services/filestore/package.json b/services/filestore/package.json index a9163666cd..bcd8011f10 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -29,8 +29,8 @@ "fs-extra": "^1.0.0", "heapdump": "^0.3.2", "knox": "~0.9.1", - "logger-sharelatex": "^1.6.0", - "metrics-sharelatex": "^2.1.1", + "logger-sharelatex": "^1.7.0", + "metrics-sharelatex": "^2.2.0", "mocha": "5.2.0", "node-transloadit": "0.0.4", "node-uuid": "~1.4.1", diff --git a/services/filestore/test/acceptance/coffee/FilestoreApp.coffee b/services/filestore/test/acceptance/coffee/FilestoreApp.coffee index 818e90ec6f..1b4cc38834 100644 --- a/services/filestore/test/acceptance/coffee/FilestoreApp.coffee +++ b/services/filestore/test/acceptance/coffee/FilestoreApp.coffee @@ -2,6 +2,9 @@ app = require('../../../app') require("logger-sharelatex").logger.level("info") logger = require("logger-sharelatex") Settings = require("settings-sharelatex") +request = require('request') + +S3_TRIES = 30 module.exports = running: false @@ -21,4 +24,22 @@ module.exports = logger.log("filestore running in dev mode") for callback in @callbacks - callback() \ No newline at end of file + callback() + + waitForS3: (callback, tries) -> + return callback() unless Settings.filestore.s3?.endpoint + tries = 1 unless tries + + request.get "#{Settings.filestore.s3.endpoint}/", (err, response) => + console.log(err, response?.statusCode, tries) + if !err && [200, 404].includes(response?.statusCode) + return callback() + + if tries == S3_TRIES + return callback('timed out waiting for S3') + + setTimeout( + () => + @waitForS3 callback, tries + 1 + 1000 + ) diff --git a/services/filestore/test/acceptance/coffee/SendingFileTest.coffee b/services/filestore/test/acceptance/coffee/SendingFileTest.coffee index cee06e82a3..4e9443fd88 100644 --- a/services/filestore/test/acceptance/coffee/SendingFileTest.coffee +++ b/services/filestore/test/acceptance/coffee/SendingFileTest.coffee @@ -9,9 +9,16 @@ fs = require("fs") request = require("request") settings = require("settings-sharelatex") FilestoreApp = require "./FilestoreApp" +async = require('async') + + +getMetric = (filestoreUrl, metric, cb) -> + request.get "#{filestoreUrl}/metrics", (err, res) -> + expect(res.statusCode).to.equal 200 + metricRegex = new RegExp("^#{metric}{[^}]+} ([0-9]+)$", "m") + cb(parseInt(metricRegex.exec(res.body)?[1] || '0')) describe "Filestore", -> - before (done)-> @localFileReadPath = "/tmp/filestore_acceptence_tests_file_read.txt" @localFileWritePath = "/tmp/filestore_acceptence_tests_file_write.txt" @@ -22,17 +29,28 @@ describe "Filestore", -> "there are 3 lines in all" ].join("\n") - fs.writeFile(@localFileReadPath, @constantFileContent, done) @filestoreUrl = "http://localhost:#{settings.internal.filestore.port}" + fs.writeFile @localFileReadPath, @constantFileContent, (err) -> + return done(err) if err + FilestoreApp.waitForS3(done) beforeEach (done)-> FilestoreApp.ensureRunning => - fs.unlink @localFileWritePath, -> - done() + async.parallel [ + (cb) => + fs.unlink @localFileWritePath, () -> + cb() + (cb) => + getMetric @filestoreUrl, 's3_egress', (metric) => + @previousEgress = metric + cb() + (cb) => + getMetric @filestoreUrl, 's3_ingress', (metric) => + @previousIngress = metric + cb() + ], done - - - it "should send a 200 for status endpoing", (done)-> + it "should send a 200 for status endpoint", (done)-> request "#{@filestoreUrl}/status", (err, response, body)-> response.statusCode.should.equal 200 body.indexOf("filestore").should.not.equal -1 @@ -59,13 +77,32 @@ describe "Filestore", -> response.statusCode.should.equal 404 done() + it 'should record an egress metric for the upload', (done) -> + getMetric @filestoreUrl, 's3_egress', (metric) => + expect(metric - @previousEgress).to.equal @constantFileContent.length + done() + + it "should return the file size on a HEAD request", (done) -> + expectedLength = Buffer.byteLength(@constantFileContent) + request.head @fileUrl, (err, res) => + expect(res.statusCode).to.equal(200) + expect(res.headers['content-length']).to.equal(expectedLength.toString()) + done() + it "should be able get the file back", (done)-> @timeout(1000 * 10) request.get @fileUrl, (err, response, body)=> body.should.equal @constantFileContent done() - it "should be able to get back the first 8 bytes of the file", (done) -> + it "should record an ingress metric when downloading the file", (done)-> + @timeout(1000 * 10) + request.get @fileUrl, () => + getMetric @filestoreUrl, 's3_ingress', (metric) => + expect(metric - @previousIngress).to.equal @constantFileContent.length + done() + + it "should be able to get back the first 9 bytes of the file", (done) -> @timeout(1000 * 10) options = uri: @fileUrl @@ -75,6 +112,17 @@ describe "Filestore", -> body.should.equal 'hello wor' done() + it "should record an ingress metric for a partial download", (done)-> + @timeout(1000 * 10) + options = + uri: @fileUrl + headers: + 'Range': 'bytes=0-8' + request.get options, ()=> + getMetric @filestoreUrl, 's3_ingress', (metric) => + expect(metric - @previousIngress).to.equal 9 + done() + it "should be able to get back bytes 4 through 10 of the file", (done) -> @timeout(1000 * 10) options = @@ -121,11 +169,17 @@ describe "Filestore", -> @file_id = Math.random() @fileUrl = "#{@filestoreUrl}/project/acceptence_tests/file/#{@file_id}" @localFileReadPath = __dirname + '/../../fixtures/test.pdf' + fs.stat @localFileReadPath, (err, stat) => + @localFileSize = stat.size + writeStream = request.post(@fileUrl) - writeStream = request.post(@fileUrl) + writeStream.on "end", done + fs.createReadStream(@localFileReadPath).pipe writeStream - writeStream.on "end", done - fs.createReadStream(@localFileReadPath).pipe writeStream + it 'should record an egress metric for the upload', (done) -> + getMetric @filestoreUrl, 's3_egress', (metric) => + expect(metric - @previousEgress).to.equal @localFileSize + done() it "should be able get the file back", (done)-> @timeout(1000 * 10) diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 7df7adfd8b..6980ce3e30 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -19,9 +19,11 @@ describe "FSPersistorManagerTests", -> rmdir:sinon.stub() exists:sinon.stub() readdir:sinon.stub() + open:sinon.stub() openSync:sinon.stub() fstatSync:sinon.stub() closeSync:sinon.stub() + stat:sinon.stub() @Rimraf = sinon.stub() @LocalFileWriter = writeStream: sinon.stub() @@ -103,20 +105,21 @@ describe "FSPersistorManagerTests", -> @opts = {} it "should use correct file location", (done) -> - @Fs.createReadStream.returns({on: ->}) @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res) => - @Fs.createReadStream.calledWith("#{@location}/#{@name1Filtered}").should.equal true + @Fs.open.calledWith("#{@location}/#{@name1Filtered}").should.equal true done() describe "with start and end options", -> beforeEach -> - @opts = {start: 0, end: 8} + @fd = 2019 + @opts_in = {start: 0, end: 8} + @opts = {start: 0, end: 8, fd: @fd} + @Fs.open.callsArgWith(2, null, @fd) it 'should pass the options to createReadStream', (done) -> - @Fs.createReadStream.returns({on: ->}) - @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=> - @Fs.createReadStream.calledWith("#{@location}/#{@name1Filtered}", @opts).should.equal true + @FSPersistorManager.getFileStream @location, @name1, @opts_in, (err,res)=> + @Fs.createReadStream.calledWith(null, @opts).should.equal true done() describe "error conditions", -> @@ -125,12 +128,10 @@ describe "FSPersistorManagerTests", -> beforeEach -> @fakeCode = 'ENOENT' - @Fs.createReadStream.returns( - on: (key, callback) => - err = new Error() - err.code = @fakeCode - callback(err, null) - ) + err = new Error() + err.code = @fakeCode + @Fs.open.callsArgWith(2, err, null) + it "should give a NotFoundError", (done) -> @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=> expect(res).to.equal null @@ -142,12 +143,9 @@ describe "FSPersistorManagerTests", -> beforeEach -> @fakeCode = 'SOMETHINGHORRIBLE' - @Fs.createReadStream.returns( - on: (key, callback) => - err = new Error() - err.code = @fakeCode - callback(err, null) - ) + err = new Error() + err.code = @fakeCode + @Fs.open.callsArgWith(2, err, null) it "should give an Error", (done) -> @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=> @@ -156,7 +154,35 @@ describe "FSPersistorManagerTests", -> expect(err instanceof Error).to.equal true done() + describe "getFileSize", -> + it "should return the file size", (done) -> + expectedFileSize = 75382 + @Fs.stat.yields(new Error("fs.stat got unexpected arguments")) + @Fs.stat.withArgs("#{@location}/#{@name1Filtered}") + .yields(null, { size: expectedFileSize }) + @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => + if err? + return done(err) + expect(fileSize).to.equal(expectedFileSize) + done() + + it "should throw a NotFoundError if the file does not exist", (done) -> + error = new Error() + error.code = "ENOENT" + @Fs.stat.yields(error) + + @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => + expect(err).to.be.instanceof(@Errors.NotFoundError) + done() + + it "should rethrow any other error", (done) -> + error = new Error() + @Fs.stat.yields(error) + + @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => + expect(err).to.equal(error) + done() describe "copyFile", -> beforeEach -> diff --git a/services/filestore/test/unit/coffee/FileControllerTests.coffee b/services/filestore/test/unit/coffee/FileControllerTests.coffee index 09398b8c01..821aadb68d 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.coffee +++ b/services/filestore/test/unit/coffee/FileControllerTests.coffee @@ -20,6 +20,7 @@ describe "FileController", -> user_files:"user_files" @FileHandler = getFile: sinon.stub() + getFileSize: sinon.stub() deleteFile: sinon.stub() insertFile: sinon.stub() getDirectorySize: sinon.stub() @@ -49,7 +50,8 @@ describe "FileController", -> file_id:@file_id headers: {} @res = - setHeader: -> + set: sinon.stub().returnsThis() + status: sinon.stub().returnsThis() @fileStream = {} describe "getFile", -> @@ -89,6 +91,39 @@ describe "FileController", -> done() @controller.getFile @req, @res + describe "getFileHead", -> + it "should return the file size in a Content-Length header", (done) -> + expectedFileSize = 84921 + @FileHandler.getFileSize.yields( + new Error("FileHandler.getFileSize: unexpected arguments") + ) + @FileHandler.getFileSize.withArgs(@bucket, @key).yields(null, expectedFileSize) + + @res.end = () => + expect(@res.status.lastCall.args[0]).to.equal(200) + expect(@res.set.calledWith("Content-Length", expectedFileSize)).to.equal(true) + done() + + @controller.getFileHead(@req, @res) + + it "should return a 404 is the file is not found", (done) -> + @FileHandler.getFileSize.yields(new @Errors.NotFoundError()) + + @res.end = () => + expect(@res.status.lastCall.args[0]).to.equal(404) + done() + + @controller.getFileHead(@req, @res) + + it "should return a 500 on internal errors", (done) -> + @FileHandler.getFileSize.yields(new Error()) + + @res.end = () => + expect(@res.status.lastCall.args[0]).to.equal(500) + done() + + @controller.getFileHead(@req, @res) + describe "insertFile", -> it "should send bucket name key and res to PersistorManager", (done)-> diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee index 50b8a17524..754366195e 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee +++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee @@ -108,7 +108,6 @@ describe "FileHandler", -> @handler._getConvertedFile.called.should.equal true done() - describe "_getStandardFile", -> beforeEach -> diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 1eee09dc29..a5ab5c2932 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -17,20 +17,31 @@ describe "S3PersistorManagerTests", -> key: "this_key" stores: user_files:"sl_user_files" - @stubbedKnoxClient = + @knoxClient = putFile:sinon.stub() copyFile:sinon.stub() list: sinon.stub() deleteMultiple: sinon.stub() get: sinon.stub() @knox = - createClient: sinon.stub().returns(@stubbedKnoxClient) - @stubbedS3Client = - copyObject:sinon.stub() - @awsS3 = sinon.stub().returns @stubbedS3Client + createClient: sinon.stub().returns(@knoxClient) + @s3EventHandlers = {} + @s3Request = + on: sinon.stub().callsFake (event, callback) => + @s3EventHandlers[event] = callback + send: sinon.stub() + @s3Response = + httpResponse: + createUnbufferedStream: sinon.stub() + @s3Client = + copyObject: sinon.stub() + headObject: sinon.stub() + getObject: sinon.stub().returns(@s3Request) + @awsS3 = sinon.stub().returns(@s3Client) @LocalFileWriter = writeStream: sinon.stub() deleteFile: sinon.stub() + @request = sinon.stub() @requires = "knox": @knox "aws-sdk/clients/s3": @awsS3 @@ -39,142 +50,173 @@ describe "S3PersistorManagerTests", -> "logger-sharelatex": log:-> err:-> + "request": @request "./Errors": @Errors = NotFoundError: sinon.stub() @key = "my/key" @bucketName = "my-bucket" @error = "my errror" + @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires describe "getFileStream", -> - beforeEach -> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - @opts = {} + describe "success", -> + beforeEach () -> + @expectedStream = { expectedStream: true } + @expectedStream.on = sinon.stub() + @s3Request.send.callsFake () => + @s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK") + @s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream) - it "should use correct key", (done)-> - @stubbedKnoxClient.get.returns( - on:-> - end:-> - ) - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback - @stubbedKnoxClient.get.calledWith(@key).should.equal true - done() + it "returns a stream", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + if err? + return done(err) + expect(stream).to.equal(@expectedStream) + done() - it "should use default auth", (done)-> - @stubbedKnoxClient.get.returns( - on:-> - end:-> - ) - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback - clientParams = - key: @settings.filestore.s3.key - secret: @settings.filestore.s3.secret - bucket: @bucketName - @knox.createClient.calledWith(clientParams).should.equal true - done() + it "sets the AWS client up with credentials from settings", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + if err? + return done(err) + expect(@awsS3.lastCall.args).to.deep.equal([{ + credentials: + accessKeyId: @settings.filestore.s3.key + secretAccessKey: @settings.filestore.s3.secret + }]) + done() - describe "with supplied auth", -> - beforeEach -> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - @credentials = - auth_key: "that_key" - auth_secret: "that_secret" - @opts = - credentials: @credentials + it "fetches the right key from the right bucket", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + if err? + return done(err) + expect(@s3Client.getObject.lastCall.args).to.deep.equal([{ + Bucket: @bucketName, + Key: @key + }]) + done() - it "should use supplied auth", (done)-> - @stubbedKnoxClient.get.returns( - on:-> - end:-> - ) - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback - clientParams = - key: @credentials.auth_key - secret: @credentials.auth_secret - bucket: @bucketName - @knox.createClient.calledWith(clientParams).should.equal true - done() + it "accepts alternative credentials", (done) -> + accessKeyId = "that_key" + secret = "that_secret" + opts = { + credentials: + auth_key: accessKeyId + auth_secret: secret + } + @S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) => + if err? + return done(err) + expect(@awsS3.lastCall.args).to.deep.equal([{ + credentials: + accessKeyId: accessKeyId + secretAccessKey: secret + }]) + expect(stream).to.equal(@expectedStream) + done() - describe "with start and end options", -> - beforeEach -> - @opts = - start: 0 - end: 8 - it "should pass headers to the knox.Client.get()", (done) -> - @stubbedKnoxClient.get.returns( - on:-> - end:-> - ) - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback - @stubbedKnoxClient.get.calledWith(@key, {'Range': 'bytes=0-8'}).should.equal true - done() - - describe "error conditions", -> + it "accepts byte range", (done) -> + start = 0 + end = 8 + opts = { start: start, end: end } + @S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) => + if err? + return done(err) + expect(@s3Client.getObject.lastCall.args).to.deep.equal([{ + Bucket: @bucketName + Key: @key + Range: "bytes=#{start}-#{end}" + }]) + expect(stream).to.equal(@expectedStream) + done() + describe "errors", -> describe "when the file doesn't exist", -> - beforeEach -> - @fakeResponse = - statusCode: 404 - @stubbedKnoxClient.get.returns( - on: (key, callback) => - if key == 'response' - callback(@fakeResponse) - end: -> - ) + @s3Request.send.callsFake () => + @s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found") - it "should produce a NotFoundError", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err, stream)=> # empty callback - expect(stream).to.equal null - expect(err).to.not.equal null - expect(err instanceof @Errors.NotFoundError).to.equal true + it "returns a NotFoundError that indicates the bucket and key", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + expect(err).to.be.instanceof(@Errors.NotFoundError) + errMsg = @Errors.NotFoundError.lastCall.args[0] + expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*")) + expect(errMsg).to.match(new RegExp(".*#{@key}.*")) done() - it "should have bucket and key in the Error message", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err, stream)=> # empty callback - error_message = @Errors.NotFoundError.lastCall.args[0] - expect(error_message).to.not.equal null - error_message.should.match(new RegExp(".*#{@bucketName}.*")) - error_message.should.match(new RegExp(".*#{@key}.*")) - done() - - describe "when the S3 service produces an error", -> + describe "when S3 encounters an unkown error", -> beforeEach -> - @fakeResponse = - statusCode: 500 - @stubbedKnoxClient.get.returns( - on: (key, callback) => - if key == 'response' - callback(@fakeResponse) - end: -> - ) + @s3Request.send.callsFake () => + @s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error") - it "should produce an error", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err, stream)=> # empty callback - expect(stream).to.equal null - expect(err).to.not.equal null - expect(err instanceof Error).to.equal true - @Errors.NotFoundError.called.should.equal false + it "returns an error", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + expect(err).to.be.instanceof(Error) done() + describe "when the S3 request errors out before receiving HTTP headers", -> + beforeEach -> + @s3Request.send.callsFake () => + @s3EventHandlers.error(new Error("connection failed")) + + it "returns an error", (done) -> + @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => + expect(err).to.be.instanceof(Error) + done() + + describe "getFileSize", -> + it "should obtain the file size from S3", (done) -> + expectedFileSize = 123 + @s3Client.headObject.yields(new Error( + "s3Client.headObject got unexpected arguments" + )) + @s3Client.headObject.withArgs({ + Bucket: @bucketName + Key: @key + }).yields(null, { ContentLength: expectedFileSize }) + + @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => + if err? + return done(err) + expect(fileSize).to.equal(expectedFileSize) + done() + + [403, 404].forEach (statusCode) -> + it "should throw NotFoundError when S3 responds with #{statusCode}", (done) -> + error = new Error() + error.statusCode = statusCode + @s3Client.headObject.yields(error) + + @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => + expect(err).to.be.an.instanceof(@Errors.NotFoundError) + done() + + it "should rethrow any other error", (done) -> + error = new Error() + @s3Client.headObject.yields(error) + @s3Client.headObject.yields(error) + + @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => + expect(err).to.equal(error) + done() + describe "sendFile", -> beforeEach -> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - @stubbedKnoxClient.putFile.returns on:-> + @knoxClient.putFile.returns on:-> it "should put file with knox", (done)-> @LocalFileWriter.deleteFile.callsArgWith(1) - @stubbedKnoxClient.putFile.callsArgWith(2, @error) + @knoxClient.putFile.callsArgWith(2, @error) @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=> - @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true + @knoxClient.putFile.calledWith(@fsPath, @key).should.equal true err.should.equal @error done() it "should delete the file and pass the error with it", (done)-> @LocalFileWriter.deleteFile.callsArgWith(1) - @stubbedKnoxClient.putFile.callsArgWith(2, @error) + @knoxClient.putFile.callsArgWith(2, @error) @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=> - @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true + @knoxClient.putFile.calledWith(@fsPath, @key).should.equal true err.should.equal @error done() @@ -183,7 +225,6 @@ describe "S3PersistorManagerTests", -> @fsPath = "to/some/where" @origin = on:-> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager.sendFile = sinon.stub().callsArgWith(3) it "should send stream to LocalFileWriter", (done)-> @@ -211,42 +252,36 @@ describe "S3PersistorManagerTests", -> beforeEach -> @sourceKey = "my/key" @destKey = "my/dest/key" - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires it "should use AWS SDK to copy file", (done)-> - @stubbedS3Client.copyObject.callsArgWith(1, @error) + @s3Client.copyObject.callsArgWith(1, @error) @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=> err.should.equal @error - @stubbedS3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true + @s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true done() it "should return a NotFoundError object if the original file does not exist", (done)-> NoSuchKeyError = {code: "NoSuchKey"} - @stubbedS3Client.copyObject.callsArgWith(1, NoSuchKeyError) + @s3Client.copyObject.callsArgWith(1, NoSuchKeyError) @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=> expect(err instanceof @Errors.NotFoundError).to.equal true done() describe "deleteDirectory", -> - beforeEach -> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - it "should list the contents passing them onto multi delete", (done)-> data = Contents: [{Key:"1234"}, {Key: "456"}] - @stubbedKnoxClient.list.callsArgWith(1, null, data) - @stubbedKnoxClient.deleteMultiple.callsArgWith(1) + @knoxClient.list.callsArgWith(1, null, data) + @knoxClient.deleteMultiple.callsArgWith(1) @S3PersistorManager.deleteDirectory @bucketName, @key, (err)=> - @stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true + @knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true done() describe "deleteFile", -> it "should use correct options", (done)-> - @request = sinon.stub().callsArgWith(1) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1) @S3PersistorManager.deleteFile @bucketName, @key, (err)=> opts = @request.args[0][0] @@ -257,9 +292,7 @@ describe "S3PersistorManagerTests", -> done() it "should return the error", (done)-> - @request = sinon.stub().callsArgWith(1, @error) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1, @error) @S3PersistorManager.deleteFile @bucketName, @key, (err)=> err.should.equal @error @@ -268,9 +301,7 @@ describe "S3PersistorManagerTests", -> describe "checkIfFileExists", -> it "should use correct options", (done)-> - @request = sinon.stub().callsArgWith(1, null, statusCode:200) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1, null, statusCode:200) @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> opts = @request.args[0][0] @@ -281,25 +312,21 @@ describe "S3PersistorManagerTests", -> done() it "should return true for a 200", (done)-> - @request = sinon.stub().callsArgWith(1, null, statusCode:200) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1, null, statusCode:200) + @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=> exists.should.equal true done() it "should return false for a non 200", (done)-> - @request = sinon.stub().callsArgWith(1, null, statusCode:404) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1, null, statusCode:404) + @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=> exists.should.equal false done() it "should return the error", (done)-> - @request = sinon.stub().callsArgWith(1, @error, {}) - @requires["request"] = @request - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @request.callsArgWith(1, @error, {}) @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> err.should.equal @error @@ -307,13 +334,10 @@ describe "S3PersistorManagerTests", -> describe "directorySize", -> - beforeEach -> - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - it "should sum directory files size", (done) -> data = Contents: [ {Size: 1024}, {Size: 2048} ] - @stubbedKnoxClient.list.callsArgWith(1, null, data) + @knoxClient.list.callsArgWith(1, null, data) @S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=> totalSize.should.equal 3072 - done() \ No newline at end of file + done()