Add endpoint for arbitrary bucket fetch

Add `/bucket/:bucket/key/*`, which fetches the file from the given bucket at the given path. Uses auth stored at `settings.filestore.s3.{{bucketName}}` if present, and otherwise default auth.

services/filestore/app.coffee

@@ -4,6 +4,7 @@ logger.initialize("filestore")
 settings = require("settings-sharelatex")
 request = require("request")
 fileController = require("./app/js/FileController")
+bucketController = require("./app/js/BucketController")
 keyBuilder = require("./app/js/KeyBuilder")
 healthCheckController = require("./app/js/HealthCheckController")
 domain = require("domain")
@@ -86,6 +87,8 @@ app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey,
 
 app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize
 
+app.get "/bucket/:bucket/key/*", bucketController.getFile
+
 app.get "/heapdump", (req, res)->
 	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)->
 		res.send filename
@@ -105,8 +108,6 @@ app.get '/status', (req, res)->
 app.get "/health_check", healthCheckController.check
 
 
-
-
 app.get '*', (req, res)->
 	res.send 404
 

services/filestore/app/coffee/BucketController.coffee

@@ -0,0 +1,36 @@
+PersistorManager = require("./PersistorManager")
+settings = require("settings-sharelatex")
+logger = require("logger-sharelatex")
+FileHandler = require("./FileHandler")
+metrics = require("metrics-sharelatex")
+parseRange = require('range-parser')
+Errors = require('./Errors')
+
+oneDayInSeconds = 60 * 60 * 24
+maxSizeInBytes = 1024 * 1024 * 1024 # 1GB
+
+module.exports = BucketController =
+
+	getFile: (req, res)->
+		{bucket} = req
+		key = req[0]
+		{format, style} = req.query
+		credentials = settings.filestore.s3&[bucket]
+		options = {
+			key: key,
+			bucket: bucket,
+			credentials: credentials
+		}
+		metrics.inc "getFile"
+		logger.log key:key, bucket:bucket, "receiving request to get file from bucket"
+		FileHandler.getFile bucket, key, options, (err, fileStream)->
+			if err?
+				logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file from bucket"
+				if err instanceof Errors.NotFoundError
+					return res.send 404
+				else
+					return res.send 500
+			else
+				logger.log key:key, bucket:bucket, format:format, style:style, "sending bucket file to response"
+				fileStream.pipe res
+

services/filestore/app/coffee/S3PersistorManager.coffee

@@ -68,8 +68,8 @@ module.exports =
 		callback = _.once callback
 		logger.log bucketName:bucketName, key:key, "getting file from s3"
 		s3Client = knox.createClient
-			key: settings.filestore.s3.key
-			secret: settings.filestore.s3.secret
+			key: opts.credentials?.auth_key || settings.filestore.s3.key
+			secret: opts.credentials?.auth_secret || settings.filestore.s3.secret
 			bucket: bucketName
 		s3Stream = s3Client.get(key, headers)
 		s3Stream.end()

services/filestore/test/unit/coffee/BucketControllerTests.coffee

@@ -0,0 +1,68 @@
+assert = require("chai").assert
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../app/js/BucketController.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "BucketController", ->
+
+	beforeEach ->
+		@PersistorManager =
+			sendStream: sinon.stub()
+			copyFile: sinon.stub()
+			deleteFile:sinon.stub()
+
+		@settings =
+			s3:
+				buckets:
+					user_files:"user_files"
+			filestore:
+				backend: "s3"
+				s3:
+					secret: "secret"
+					key: "this_key"
+
+		@FileHandler =
+			getFile: sinon.stub()
+			deleteFile: sinon.stub()
+			insertFile: sinon.stub()
+			getDirectorySize: sinon.stub()
+		@LocalFileWriter = {}
+		@controller = SandboxedModule.require modulePath, requires:
+			"./LocalFileWriter":@LocalFileWriter
+			"./FileHandler": @FileHandler
+			"./PersistorManager":@PersistorManager
+			"settings-sharelatex": @settings
+			"logger-sharelatex":
+				log:->
+				err:->
+		@project_id = "project_id"
+		@file_id = "file_id"
+		@bucket = "user_files"
+		@key = "#{@project_id}/#{@file_id}"
+		@req =
+			bucket:@bucket
+			0:@key
+			query:{}
+			headers: {}
+		@res =
+			setHeader: ->
+		@fileStream = {}
+
+	describe "getFile", ->
+
+		it "should pipe the stream", (done)->
+			@FileHandler.getFile.callsArgWith(3, null, @fileStream)
+			@fileStream.pipe = (res)=>
+				res.should.equal @res
+				done()
+			@controller.getFile @req, @res
+
+		it "should send a 500 if there is a problem", (done)->
+			@FileHandler.getFile.callsArgWith(3, "error")
+			@res.send = (code)=>
+				code.should.equal 500
+				done()
+			@controller.getFile @req, @res

services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee

@@ -55,6 +55,41 @@ describe "S3PersistorManagerTests", ->
 			@stubbedKnoxClient.get.calledWith(@key).should.equal true
 			done()
 
+		it "should use default auth", (done)->
+			@stubbedKnoxClient.get.returns(
+				on:->
+				end:->
+			)
+			@S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback
+			clientParams =
+				key: @settings.filestore.s3.key
+				secret: @settings.filestore.s3.secret
+				bucket: @bucketName
+			@knox.createClient.calledWith(clientParams).should.equal true
+			done()
+
+		describe "with supplied auth", ->
+			beforeEach ->
+				@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
+				@credentials =
+					auth_key: "that_key"
+					auth_secret: "that_secret"
+				@opts =
+					credentials: @credentials
+
+			it "should use supplied auth", (done)->
+				@stubbedKnoxClient.get.returns(
+					on:->
+					end:->
+				)
+				@S3PersistorManager.getFileStream @bucketName, @key, @opts, (err)=> # empty callback
+				clientParams =
+					key: @credentials.auth_key
+					secret: @credentials.auth_secret
+					bucket: @bucketName
+				@knox.createClient.calledWith(clientParams).should.equal true
+				done()
+
 		describe "with start and end options", ->
 			beforeEach ->
 				@opts =