From feca8933f1ac4241242691bc74c05cac685f565a Mon Sep 17 00:00:00 2001 From: Michael Mazour Date: Wed, 4 Jul 2018 11:18:55 +0100 Subject: [PATCH] Add endpoint for arbitrary bucket fetch Add `/bucket/:bucket/key/*`, which fetches the file from the given bucket at the given path. Uses auth stored at `settings.filestore.s3.{{bucketName}}` if present, and otherwise default auth. --- services/filestore/app.coffee | 7 +- .../app/coffee/BucketController.coffee | 36 ++++++++++ .../app/coffee/S3PersistorManager.coffee | 4 +- .../unit/coffee/BucketControllerTests.coffee | 68 +++++++++++++++++++ .../coffee/S3PersistorManagerTests.coffee | 35 ++++++++++ 5 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 services/filestore/app/coffee/BucketController.coffee create mode 100644 services/filestore/test/unit/coffee/BucketControllerTests.coffee diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index eb97ad48dd..ce10d2c91e 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -4,6 +4,7 @@ logger.initialize("filestore") settings = require("settings-sharelatex") request = require("request") fileController = require("./app/js/FileController") +bucketController = require("./app/js/BucketController") keyBuilder = require("./app/js/KeyBuilder") healthCheckController = require("./app/js/HealthCheckController") domain = require("domain") @@ -18,7 +19,7 @@ Metrics.memory.monitor(logger) app.configure -> app.use Metrics.http.monitor(logger) - + app.configure 'development', -> console.log "Development Enviroment" app.use express.errorHandler({ dumpExceptions: true, showStack: true }) @@ -86,6 +87,8 @@ app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize +app.get "/bucket/:bucket/key/*", bucketController.getFile + app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> res.send filename @@ -103,8 +106,6 @@ app.get '/status', (req, res)-> app.get "/health_check", healthCheckController.check - - app.get '*', (req, res)-> diff --git a/services/filestore/app/coffee/BucketController.coffee b/services/filestore/app/coffee/BucketController.coffee new file mode 100644 index 0000000000..cc1ff03c45 --- /dev/null +++ b/services/filestore/app/coffee/BucketController.coffee @@ -0,0 +1,36 @@ +PersistorManager = require("./PersistorManager") +settings = require("settings-sharelatex") +logger = require("logger-sharelatex") +FileHandler = require("./FileHandler") +metrics = require("metrics-sharelatex") +parseRange = require('range-parser') +Errors = require('./Errors') + +oneDayInSeconds = 60 * 60 * 24 +maxSizeInBytes = 1024 * 1024 * 1024 # 1GB + +module.exports = BucketController = + + getFile: (req, res)-> + {bucket} = req + key = req[0] + {format, style} = req.query + credentials = settings.filestore.s3&[bucket] + options = { + key: key, + bucket: bucket, + credentials: credentials + } + metrics.inc "getFile" + logger.log key:key, bucket:bucket, "receiving request to get file from bucket" + FileHandler.getFile bucket, key, options, (err, fileStream)-> + if err? + logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file from bucket" + if err instanceof Errors.NotFoundError + return res.send 404 + else + return res.send 500 + else + logger.log key:key, bucket:bucket, format:format, style:style, "sending bucket file to response" + fileStream.pipe res + diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index b1a03fb4f4..2bd6eb0e9b 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -68,8 +68,8 @@ module.exports = callback = _.once callback logger.log bucketName:bucketName, key:key, "getting file from s3" s3Client = knox.createClient - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret + key: opts.credentials?.auth_key || settings.filestore.s3.key + secret: opts.credentials?.auth_secret || settings.filestore.s3.secret bucket: bucketName s3Stream = s3Client.get(key, headers) s3Stream.end() diff --git a/services/filestore/test/unit/coffee/BucketControllerTests.coffee b/services/filestore/test/unit/coffee/BucketControllerTests.coffee new file mode 100644 index 0000000000..fc91d08793 --- /dev/null +++ b/services/filestore/test/unit/coffee/BucketControllerTests.coffee @@ -0,0 +1,68 @@ +assert = require("chai").assert +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../app/js/BucketController.js" +SandboxedModule = require('sandboxed-module') + +describe "BucketController", -> + + beforeEach -> + @PersistorManager = + sendStream: sinon.stub() + copyFile: sinon.stub() + deleteFile:sinon.stub() + + @settings = + s3: + buckets: + user_files:"user_files" + filestore: + backend: "s3" + s3: + secret: "secret" + key: "this_key" + + @FileHandler = + getFile: sinon.stub() + deleteFile: sinon.stub() + insertFile: sinon.stub() + getDirectorySize: sinon.stub() + @LocalFileWriter = {} + @controller = SandboxedModule.require modulePath, requires: + "./LocalFileWriter":@LocalFileWriter + "./FileHandler": @FileHandler + "./PersistorManager":@PersistorManager + "settings-sharelatex": @settings + "logger-sharelatex": + log:-> + err:-> + @project_id = "project_id" + @file_id = "file_id" + @bucket = "user_files" + @key = "#{@project_id}/#{@file_id}" + @req = + bucket:@bucket + 0:@key + query:{} + headers: {} + @res = + setHeader: -> + @fileStream = {} + + describe "getFile", -> + + it "should pipe the stream", (done)-> + @FileHandler.getFile.callsArgWith(3, null, @fileStream) + @fileStream.pipe = (res)=> + res.should.equal @res + done() + @controller.getFile @req, @res + + it "should send a 500 if there is a problem", (done)-> + @FileHandler.getFile.callsArgWith(3, "error") + @res.send = (code)=> + code.should.equal 500 + done() + @controller.getFile @req, @res diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 3a3e7b0d86..b48fde7820 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -55,6 +55,41 @@ describe "S3PersistorManagerTests", -> @stubbedKnoxClient.get.calledWith(@key).should.equal true done() + it "should use default auth", (done)-> + @stubbedKnoxClient.get.returns( + on:-> + end:-> + ) + @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback + clientParams = + key: @settings.filestore.s3.key + secret: @settings.filestore.s3.secret + bucket: @bucketName + @knox.createClient.calledWith(clientParams).should.equal true + done() + + describe "with supplied auth", -> + beforeEach -> + @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + @credentials = + auth_key: "that_key" + auth_secret: "that_secret" + @opts = + credentials: @credentials + + it "should use supplied auth", (done)-> + @stubbedKnoxClient.get.returns( + on:-> + end:-> + ) + @S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback + clientParams = + key: @credentials.auth_key + secret: @credentials.auth_secret + bucket: @bucketName + @knox.createClient.calledWith(clientParams).should.equal true + done() + describe "with start and end options", -> beforeEach -> @opts =