make s3 chosen if no config option specified

rename *wrapper  to FilestoreManager
remove ToS3 from filestore commands
rename configuration option for filestore.
add details of interface to FilestoreWrapper.coffee for reference.
remove api comments
rename settings.filestoreManager to settings.persistorManager
This commit is contained in:
Oliver Matthews 2014-02-25 16:38:13 +00:00
parent c9af5e5420
commit 4b5f9ad4fb
8 changed files with 114 additions and 110 deletions

View file

@ -13,7 +13,7 @@ module.exports =
insertFile: (bucket, key, stream, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key)
PersistorManager.deleteDirectory bucket, convetedKey, ->
PersistorManager.sendStreamToS3 bucket, key, stream, ->
PersistorManager.sendStream bucket, key, stream, ->
callback()
deleteFile: (bucket, key, callback)->
@ -53,7 +53,7 @@ module.exports =
if err?
logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file"
return callback(err)
PersistorManager.sendFileToS3 bucket, convetedKey, fsPath, (err)->
PersistorManager.sendFile bucket, convetedKey, fsPath, (err)->
if err?
logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file"
return callback(err)
@ -67,7 +67,7 @@ module.exports =
FileConverter.thumbnail origonalFsPath, callback
else if opts.style == "preview"
FileConverter.preview origonalFsPath, callback
else
else
throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")

View file

@ -1,10 +1,10 @@
settings = require("settings-sharelatex")
logger = require("logger-sharelatex")
s3Wrapper = require("./s3Wrapper")
S3PersistorManager = require("./S3PersistorManager")
logger.log backend:settings.filestoreBackend, "Loading backend"
module.exports = switch settings.filestoreBackend
when "s3"
s3Wrapper
logger.log backend:settings.persistorManager, "Loading backend"
module.exports = switch settings.persistorManager
when "s3",null
S3PersistorManager
else
throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" )
throw new Error( "Unknown filestore backend: #{settings.persistorManager}" )

View file

@ -34,7 +34,7 @@ buildDefaultOptions = (bucketName, method, key)->
module.exports =
sendFileToS3: (bucketName, key, fsPath, callback)->
sendFile: (bucketName, key, fsPath, callback)->
s3Client = knox.createClient
key: settings.s3.key
secret: settings.s3.secret
@ -57,7 +57,7 @@ module.exports =
callback err
sendStreamToS3: (bucketName, key, readStream, callback)->
sendStream: (bucketName, key, readStream, callback)->
logger.log bucketName:bucketName, key:key, "sending file to s3"
readStream.on "error", (err)->
logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
@ -65,7 +65,7 @@ module.exports =
if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err)
@sendFileToS3 bucketName, key, fsPath, callback
@sendFile bucketName, key, fsPath, callback
getFileStream: (bucketName, key, callback = (err, res)->)->
logger.log bucketName:bucketName, key:key, "getting file from s3"

View file

@ -4,10 +4,11 @@ module.exports =
port: 3009
host: "localhost"
# which backend to use
# which persistor to use for file storage
# current options are:
# "s3" - Amazon S3
filestoreBackend: "s3"
# if no persistor is chosen, s3 will be used by default
persistorManager: "s3"
# ShareLaTeX stores binary files like images in S3.
# Fill in your Amazon S3 credentials below.
@ -20,7 +21,7 @@ module.exports =
# Filestore health check
# ----------------------
# Project and file details to check in filestore when calling /health_check
# Project and file details to check in persistor when calling /health_check
# health_check:
# project_id: ""
# file_id: ""

View file

@ -9,8 +9,8 @@ SandboxedModule = require('sandboxed-module')
describe "FileController", ->
beforeEach ->
@FsWrapper =
sendStreamToS3: sinon.stub()
@PersistorManager =
sendStream: sinon.stub()
copyFile: sinon.stub()
deleteFile:sinon.stub()
@ -26,7 +26,7 @@ describe "FileController", ->
@controller = SandboxedModule.require modulePath, requires:
"./LocalFileWriter":@LocalFileWriter
"./FileHandler": @FileHandler
"./PersistorManager":@FsWrapper
"./PersistorManager":@PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
@ -73,7 +73,7 @@ describe "FileController", ->
describe "insertFile", ->
it "should send bucket name key and res to FsWrapper", (done)->
it "should send bucket name key and res to PersistorManager", (done)->
@FileHandler.insertFile.callsArgWith(3)
@res.send = =>
@FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true
@ -90,17 +90,17 @@ describe "FileController", ->
project_id: @oldProject_id
file_id: @oldFile_id
it "should send bucket name and both keys to FsWrapper", (done)->
@FsWrapper.copyFile.callsArgWith(3)
it "should send bucket name and both keys to PersistorManager", (done)->
@PersistorManager.copyFile.callsArgWith(3)
@res.send = (code)=>
code.should.equal 200
@FsWrapper.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
@PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
done()
@controller.copyFile @req, @res
it "should send a 500 if there was an error", (done)->
@FsWrapper.copyFile.callsArgWith(3, "error")
@PersistorManager.copyFile.callsArgWith(3, "error")
@res.send = (code)=>
code.should.equal 500
done()

View file

@ -13,12 +13,12 @@ describe "FileHandler", ->
s3:
buckets:
user_files:"user_files"
@FsWrapper =
@PersistorManager =
getFileStream: sinon.stub()
checkIfFileExists: sinon.stub()
deleteFile: sinon.stub()
deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub()
sendStream: sinon.stub()
insertFile: sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
@ -33,7 +33,7 @@ describe "FileHandler", ->
compressPng: sinon.stub()
@handler = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings
"./PersistorManager":@FsWrapper
"./PersistorManager":@PersistorManager
"./LocalFileWriter":@LocalFileWriter
"./FileConverter":@FileConverter
"./KeyBuilder": @keyBuilder
@ -50,33 +50,33 @@ describe "FileHandler", ->
describe "insertFile", ->
beforeEach ->
@stream = {}
@FsWrapper.deleteDirectory.callsArgWith(2)
@FsWrapper.sendStreamToS3.callsArgWith(3)
@PersistorManager.deleteDirectory.callsArgWith(2)
@PersistorManager.sendStream.callsArgWith(3)
it "should send file to s3", (done)->
it "should send file to the filestore", (done)->
@handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.sendStreamToS3.calledWith(@bucket, @key, @stream).should.equal true
@PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true
done()
it "should delete the convetedKey folder", (done)->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
@PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done()
describe "deleteFile", ->
beforeEach ->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@FsWrapper.deleteFile.callsArgWith(2)
@PersistorManager.deleteFile.callsArgWith(2)
it "should tell the s3 wrapper to delete the file", (done)->
it "should tell the filestore manager to delete the file", (done)->
@handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @key).should.equal true
@PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true
done()
it "should tell the s3 wrapper to delete the cached foler", (done)->
it "should tell the filestore manager to delete the cached foler", (done)->
@handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
@PersistorManager.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done()
describe "getFile", ->
@ -102,11 +102,11 @@ describe "FileHandler", ->
beforeEach ->
@fileStream = {on:->}
@FsWrapper.getFileStream.callsArgWith(2, "err", @fileStream)
@PersistorManager.getFileStream.callsArgWith(2, "err", @fileStream)
it "should get the stream from s3 ", (done)->
it "should get the stream", (done)->
@handler.getFile @bucket, @key, null, =>
@FsWrapper.getFileStream.calledWith(@bucket, @key).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true
done()
it "should return the stream and error", (done)->
@ -118,14 +118,14 @@ describe "FileHandler", ->
describe "_getConvertedFile", ->
it "should getFileStream if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, true)
@FsWrapper.getFileStream.callsArgWith(2)
@PersistorManager.checkIfFileExists.callsArgWith(2, null, true)
@PersistorManager.getFileStream.callsArgWith(2)
@handler._getConvertedFile @bucket, @key, {}, =>
@FsWrapper.getFileStream.calledWith(@bucket).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket).should.equal true
done()
it "should call _getConvertedFileAndCache if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, false)
@PersistorManager.checkIfFileExists.callsArgWith(2, null, false)
@handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4)
@handler._getConvertedFile @bucket, @key, {}, =>
@handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true
@ -134,15 +134,15 @@ describe "FileHandler", ->
describe "_getConvertedFileAndCache", ->
it "should _convertFile ", (done)->
@FsWrapper.sendFileToS3 = sinon.stub().callsArgWith(3)
@FsWrapper.getFileStream = sinon.stub().callsArgWith(2)
@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
@PersistorManager.getFileStream = sinon.stub().callsArgWith(2)
@convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, =>
@handler._convertFile.called.should.equal true
@FsWrapper.sendFileToS3.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@FsWrapper.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
done()

View file

@ -11,20 +11,20 @@ SandboxedModule = require('sandboxed-module')
describe "PersistorManagerTests", ->
beforeEach ->
@s3Wrapper =
@S3PersistorManager =
getFileStream: sinon.stub()
checkIfFileExists: sinon.stub()
deleteFile: sinon.stub()
deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub()
sendStream: sinon.stub()
insertFile: sinon.stub()
describe "test s3 mixin", ->
beforeEach ->
@settings =
filestoreBackend: "s3"
persistorManager: "s3"
@requires =
"./s3Wrapper": @s3Wrapper
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
@ -34,63 +34,66 @@ describe "PersistorManagerTests", ->
it "should load getFileStream", (done) ->
@PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream()
@s3Wrapper.getFileStream.calledOnce.should.equal true
@S3PersistorManager.getFileStream.calledOnce.should.equal true
done()
it "should load checkIfFileExists", (done) ->
@PersistorManager.checkIfFileExists()
@s3Wrapper.checkIfFileExists.calledOnce.should.equal true
@S3PersistorManager.checkIfFileExists.calledOnce.should.equal true
done()
it "should load deleteFile", (done) ->
@PersistorManager.deleteFile()
@s3Wrapper.deleteFile.calledOnce.should.equal true
@S3PersistorManager.deleteFile.calledOnce.should.equal true
done()
it "should load deleteDirectory", (done) ->
@PersistorManager.deleteDirectory()
@s3Wrapper.deleteDirectory.calledOnce.should.equal true
@S3PersistorManager.deleteDirectory.calledOnce.should.equal true
done()
it "should load sendStreamToS3", (done) ->
@PersistorManager.sendStreamToS3()
@s3Wrapper.sendStreamToS3.calledOnce.should.equal true
it "should load sendStream", (done) ->
@PersistorManager.sendStream()
@S3PersistorManager.sendStream.calledOnce.should.equal true
done()
it "should load insertFile", (done) ->
@PersistorManager.insertFile()
@s3Wrapper.insertFile.calledOnce.should.equal true
@S3PersistorManager.insertFile.calledOnce.should.equal true
done()
describe "test unspecified mixins", ->
it "should load s3 when no wrapper specified", (done) ->
@settings =
@requires =
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
@PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream()
@S3PersistorManager.getFileStream.calledOnce.should.equal true
done()
describe "test invalid mixins", ->
it "should not load a null wrapper", (done) ->
@settings =
@requires =
"./s3Wrapper": @s3Wrapper
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error
assert.equal("Unknown filestore backend: null",error.message)
done()
it "should not load an invalid wrapper", (done) ->
@settings =
filestoreBackend:"magic"
persistorManager:"magic"
@requires =
"./s3Wrapper": @s3Wrapper
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
@fsWrapper=null
try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error
assert.equal("Unknown filestore backend: magic",error.message)
assert.isNull(@fsWrapper)
done()

View file

@ -3,10 +3,10 @@ sinon = require('sinon')
chai = require('chai')
should = chai.should()
expect = chai.expect
modulePath = "../../../app/js/s3Wrapper.js"
modulePath = "../../../app/js/S3PersistorManager.js"
SandboxedModule = require('sandboxed-module')
describe "s3WrapperTests", ->
describe "S3PersistorManagerTests", ->
beforeEach ->
@settings =
@ -39,7 +39,7 @@ describe "s3WrapperTests", ->
describe "getFileStream", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use correct key", (done)->
@ -47,20 +47,20 @@ describe "s3WrapperTests", ->
on:->
end:->
)
@s3Wrapper.getFileStream @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.get.calledWith(@key).should.equal true
done()
describe "sendFileToS3", ->
describe "sendFile", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@stubbedKnoxClient.putFile.returns on:->
it "should put file with knox", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error
done()
@ -68,49 +68,49 @@ describe "s3WrapperTests", ->
it "should delete the file and pass the error with it", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error
done()
describe "sendStreamToS3", ->
describe "sendStream", ->
beforeEach ->
@fsPath = "to/some/where"
@origin =
@origin =
on:->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.sendFileToS3 = sinon.stub().callsArgWith(3)
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
it "should send stream to LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, =>
@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
done()
it "should return the error from LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, @error)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=>
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
err.should.equal @error
done()
it "should send the file to s3", (done)->
it "should send the file to the filestore", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=>
@s3Wrapper.sendFileToS3.called.should.equal true
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
@S3PersistorManager.sendFile.called.should.equal true
done()
describe "copyFile", ->
beforeEach ->
@sourceKey = "my/key"
@destKey = "my/dest/key"
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use knox to copy file", (done)->
@stubbedKnoxClient.copyFile.callsArgWith(2, @error)
@s3Wrapper.copyFile @bucketName, @sourceKey, @destKey, (err)=>
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
err.should.equal @error
@stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true
done()
@ -118,14 +118,14 @@ describe "s3WrapperTests", ->
describe "deleteDirectory", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should list the contents passing them onto multi delete", (done)->
data =
Contents: [{Key:"1234"}, {Key: "456"}]
@stubbedKnoxClient.list.callsArgWith(1, null, data)
@stubbedKnoxClient.deleteMultiple.callsArgWith(1)
@s3Wrapper.deleteDirectory @bucketName, @key, (err)=>
@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
@stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
done()
@ -134,9 +134,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=>
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "delete"
@ -147,9 +147,9 @@ describe "s3WrapperTests", ->
it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=>
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
err.should.equal @error
done()
@ -158,9 +158,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=>
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "head"
@ -171,24 +171,24 @@ describe "s3WrapperTests", ->
it "should return true for a 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=>
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal true
done()
it "should return false for a non 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:404)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=>
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal false
done()
it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error, {})
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=>
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
err.should.equal @error
done()
done()