From f920fd0b16bd2b7d0664731927c974fb007109d1 Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Tue, 4 Mar 2014 15:01:13 +0000 Subject: [PATCH] match refactor_config on cwoac/sharelatex --- .../filestore/app/coffee/KeyBuilder.coffee | 6 ++-- .../app/coffee/PersistorManager.coffee | 8 ++--- .../app/coffee/S3PersistorManager.coffee | 20 ++++++------- .../config/settings.development.coffee | 28 ++++++++++++----- .../unit/coffee/PersistorManagerTests.coffee | 6 ++-- .../coffee/S3PersistorManagerTests.coffee | 30 ++++++++++--------- 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 113c0eac57..45aa351487 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -20,15 +20,15 @@ module.exports = userFileKey: (req, res, next)-> {project_id, file_id} = req.params req.key = "#{project_id}/#{file_id}" - req.bucket = settings.s3.buckets.user_files + req.bucket = settings.filestore.stores.user_files next() templateFileKey: (req, res, next)-> {template_id, format, version} = req.params req.key = "#{template_id}/#{version}/#{format}" - req.bucket = settings.s3.buckets.template_files + req.bucket = settings.filestore.stores.template_files req.version = version opts = req.query next() - \ No newline at end of file + diff --git a/services/filestore/app/coffee/PersistorManager.coffee b/services/filestore/app/coffee/PersistorManager.coffee index 1b200c58d3..1dad923098 100644 --- a/services/filestore/app/coffee/PersistorManager.coffee +++ b/services/filestore/app/coffee/PersistorManager.coffee @@ -2,14 +2,14 @@ settings = require("settings-sharelatex") logger = require("logger-sharelatex") # assume s3 if none specified -settings.filestoreBackend ||= "s3" +settings.filestore.backend ||= "s3" -logger.log backend:settings.filestoreBackend, "Loading backend" -module.exports = switch settings.filestoreBackend +logger.log backend:settings.filestore.backend, "Loading backend" +module.exports = switch settings.filestore.backend when "s3" require("./S3PersistorManager") when "fs" require("./FSPersistorManager") else - throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" ) + throw new Error( "Unknown filestore backend: #{settings.filestore.backend}" ) diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index dd65b79abc..0b35ea7b52 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -24,8 +24,8 @@ printSockets() buildDefaultOptions = (bucketName, method, key)-> return { aws: - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName method: method timeout: thirtySeconds @@ -36,8 +36,8 @@ module.exports = sendFile: (bucketName, key, fsPath, callback)-> s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName putEventEmiter = s3Client.putFile fsPath, key, (err, res)-> if err? @@ -70,8 +70,8 @@ module.exports = getFileStream: (bucketName, key, callback = (err, res)->)-> logger.log bucketName:bucketName, key:key, "getting file from s3" s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Stream = s3Client.get(key) s3Stream.end() @@ -84,8 +84,8 @@ module.exports = copyFile: (bucketName, sourceKey, destKey, callback)-> logger.log bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "copying file in s3" s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Client.copyFile sourceKey, destKey, (err)-> if err? @@ -102,8 +102,8 @@ module.exports = deleteDirectory: (bucketName, key, callback)-> s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Client.list prefix:key, (err, data)-> keys = _.map data.Contents, (entry)-> diff --git a/services/filestore/config/settings.development.coffee b/services/filestore/config/settings.development.coffee index db79f35ca6..19088d3c76 100644 --- a/services/filestore/config/settings.development.coffee +++ b/services/filestore/config/settings.development.coffee @@ -3,13 +3,27 @@ module.exports = filestore: port: 3009 host: "localhost" - - # which persistor to use for file storage - # current options are: - # "s3" - Amazon S3 - # "fs" - local filesystem - # if no persistor is chosen, s3 will be used by default - filestoreBackend: "s3" + + filestore: + # which backend persistor to use. + # choices are + # s3 - Amazon S3 + # fs - local filesystem + backend: "s3" + stores: + # where to store user and template binary files + # + # For Amazon S3 this is the bucket name to store binary files in + # Must contain full url like: .s3.amazonaws.com + # + # For local filesystem this is the directory to store the files in. + # Must contain full path, e.g. "/var/lib/sharelatex/data" + # This path must exist, not be tmpfs and be writable to by the user sharelatex is run as. + user_files: "" + s3: + # if you are using S3, then fill in your S3 details below + key: "" + secret: "" # ShareLaTeX stores binary files like images in S3. # Fill in your Amazon S3 credentials below. diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/PersistorManagerTests.coffee index af11fa7408..ca7a82cbaa 100644 --- a/services/filestore/test/unit/coffee/PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/PersistorManagerTests.coffee @@ -22,7 +22,8 @@ describe "PersistorManagerTests", -> describe "test s3 mixin", -> beforeEach -> @settings = - filestoreBackend: "s3" + filestore: + backend: "s3" @requires = "./S3PersistorManager": @S3PersistorManager "settings-sharelatex": @settings @@ -81,7 +82,8 @@ describe "PersistorManagerTests", -> describe "test invalid mixins", -> it "should not load an invalid wrapper", (done) -> @settings = - filestoreBackend:"magic" + filestore: + backend:"magic" @requires = "./S3PersistorManager": @S3PersistorManager "settings-sharelatex": @settings diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 76872fb140..fe70f1008d 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -9,24 +9,26 @@ SandboxedModule = require('sandboxed-module') describe "S3PersistorManagerTests", -> beforeEach -> - @settings = - s3: - secret: "secret" - key: "this_key" - buckets: - user_files:"sl_user_files" - @stubbedKnoxClient = + @settings = + filestore: + backend: "s3" + s3: + secret: "secret" + key: "this_key" + stores: + user_files:"sl_user_files" + @stubbedKnoxClient = putFile:sinon.stub() copyFile:sinon.stub() list: sinon.stub() deleteMultiple: sinon.stub() get: sinon.stub() - @knox = + @knox = createClient: sinon.stub().returns(@stubbedKnoxClient) - @LocalFileWriter = + @LocalFileWriter = writeStream: sinon.stub() deleteFile: sinon.stub() - @requires = + @requires = "knox": @knox "settings-sharelatex": @settings "./LocalFileWriter":@LocalFileWriter @@ -48,7 +50,7 @@ describe "S3PersistorManagerTests", -> end:-> ) @S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=> - @stubbedKnoxClient.get.calledWith(@key).should.equal true + @stubbedKnoxClient.get.calledWith(@key).should.equal true done() describe "sendFile", -> @@ -121,7 +123,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires it "should list the contents passing them onto multi delete", (done)-> - data = + data = Contents: [{Key:"1234"}, {Key: "456"}] @stubbedKnoxClient.list.callsArgWith(1, null, data) @stubbedKnoxClient.deleteMultiple.callsArgWith(1) @@ -138,7 +140,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.deleteFile @bucketName, @key, (err)=> opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) opts.method.should.equal "delete" opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}" @@ -162,7 +164,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) opts.method.should.equal "head" opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"