Merge pull request #10 from cwoac/rename_to_persistor

Finish abstracting persistor manager
This commit is contained in:
James Allen 2014-02-26 12:19:52 +00:00
commit 0306a3ad4a
8 changed files with 103 additions and 95 deletions

View file

@ -13,7 +13,7 @@ module.exports =
insertFile: (bucket, key, stream, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key)
PersistorManager.deleteDirectory bucket, convetedKey, ->
PersistorManager.sendStreamToS3 bucket, key, stream, ->
PersistorManager.sendStream bucket, key, stream, ->
callback()
deleteFile: (bucket, key, callback)->
@ -53,7 +53,7 @@ module.exports =
if err?
logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file"
return callback(err)
PersistorManager.sendFileToS3 bucket, convetedKey, fsPath, (err)->
PersistorManager.sendFile bucket, convetedKey, fsPath, (err)->
if err?
logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file"
return callback(err)

View file

@ -1,10 +1,14 @@
settings = require("settings-sharelatex")
logger = require("logger-sharelatex")
s3Wrapper = require("./s3Wrapper")
S3PersistorManager = require("./S3PersistorManager")
# assume s3 if none specified
settings.filestoreBackend ||= "s3"
logger.log backend:settings.filestoreBackend, "Loading backend"
module.exports = switch settings.filestoreBackend
when "s3"
s3Wrapper
S3PersistorManager
else
throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" )

View file

@ -34,7 +34,7 @@ buildDefaultOptions = (bucketName, method, key)->
module.exports =
sendFileToS3: (bucketName, key, fsPath, callback)->
sendFile: (bucketName, key, fsPath, callback)->
s3Client = knox.createClient
key: settings.s3.key
secret: settings.s3.secret
@ -57,7 +57,7 @@ module.exports =
callback err
sendStreamToS3: (bucketName, key, readStream, callback)->
sendStream: (bucketName, key, readStream, callback)->
logger.log bucketName:bucketName, key:key, "sending file to s3"
readStream.on "error", (err)->
logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
@ -65,7 +65,7 @@ module.exports =
if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err)
@sendFileToS3 bucketName, key, fsPath, callback
@sendFile bucketName, key, fsPath, callback
getFileStream: (bucketName, key, callback = (err, res)->)->
logger.log bucketName:bucketName, key:key, "getting file from s3"

View file

@ -4,9 +4,10 @@ module.exports =
port: 3009
host: "localhost"
# which backend to use
# which persistor to use for file storage
# current options are:
# "s3" - Amazon S3
# if no persistor is chosen, s3 will be used by default
filestoreBackend: "s3"
# ShareLaTeX stores binary files like images in S3.
@ -20,7 +21,7 @@ module.exports =
# Filestore health check
# ----------------------
# Project and file details to check in filestore when calling /health_check
# Project and file details to check in persistor when calling /health_check
# health_check:
# project_id: ""
# file_id: ""

View file

@ -9,8 +9,8 @@ SandboxedModule = require('sandboxed-module')
describe "FileController", ->
beforeEach ->
@FsWrapper =
sendStreamToS3: sinon.stub()
@PersistorManager =
sendStream: sinon.stub()
copyFile: sinon.stub()
deleteFile:sinon.stub()
@ -26,7 +26,7 @@ describe "FileController", ->
@controller = SandboxedModule.require modulePath, requires:
"./LocalFileWriter":@LocalFileWriter
"./FileHandler": @FileHandler
"./PersistorManager":@FsWrapper
"./PersistorManager":@PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
@ -73,7 +73,7 @@ describe "FileController", ->
describe "insertFile", ->
it "should send bucket name key and res to FsWrapper", (done)->
it "should send bucket name key and res to PersistorManager", (done)->
@FileHandler.insertFile.callsArgWith(3)
@res.send = =>
@FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true
@ -90,17 +90,17 @@ describe "FileController", ->
project_id: @oldProject_id
file_id: @oldFile_id
it "should send bucket name and both keys to FsWrapper", (done)->
@FsWrapper.copyFile.callsArgWith(3)
it "should send bucket name and both keys to PersistorManager", (done)->
@PersistorManager.copyFile.callsArgWith(3)
@res.send = (code)=>
code.should.equal 200
@FsWrapper.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
@PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
done()
@controller.copyFile @req, @res
it "should send a 500 if there was an error", (done)->
@FsWrapper.copyFile.callsArgWith(3, "error")
@PersistorManager.copyFile.callsArgWith(3, "error")
@res.send = (code)=>
code.should.equal 500
done()

View file

@ -13,12 +13,12 @@ describe "FileHandler", ->
s3:
buckets:
user_files:"user_files"
@FsWrapper =
@PersistorManager =
getFileStream: sinon.stub()
checkIfFileExists: sinon.stub()
deleteFile: sinon.stub()
deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub()
sendStream: sinon.stub()
insertFile: sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
@ -33,7 +33,7 @@ describe "FileHandler", ->
compressPng: sinon.stub()
@handler = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings
"./PersistorManager":@FsWrapper
"./PersistorManager":@PersistorManager
"./LocalFileWriter":@LocalFileWriter
"./FileConverter":@FileConverter
"./KeyBuilder": @keyBuilder
@ -50,33 +50,33 @@ describe "FileHandler", ->
describe "insertFile", ->
beforeEach ->
@stream = {}
@FsWrapper.deleteDirectory.callsArgWith(2)
@FsWrapper.sendStreamToS3.callsArgWith(3)
@PersistorManager.deleteDirectory.callsArgWith(2)
@PersistorManager.sendStream.callsArgWith(3)
it "should send file to s3", (done)->
it "should send file to the filestore", (done)->
@handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.sendStreamToS3.calledWith(@bucket, @key, @stream).should.equal true
@PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true
done()
it "should delete the convetedKey folder", (done)->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
@PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done()
describe "deleteFile", ->
beforeEach ->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@FsWrapper.deleteFile.callsArgWith(2)
@PersistorManager.deleteFile.callsArgWith(2)
it "should tell the s3 wrapper to delete the file", (done)->
it "should tell the filestore manager to delete the file", (done)->
@handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @key).should.equal true
@PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true
done()
it "should tell the s3 wrapper to delete the cached foler", (done)->
it "should tell the filestore manager to delete the cached foler", (done)->
@handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
@PersistorManager.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done()
describe "getFile", ->
@ -102,11 +102,11 @@ describe "FileHandler", ->
beforeEach ->
@fileStream = {on:->}
@FsWrapper.getFileStream.callsArgWith(2, "err", @fileStream)
@PersistorManager.getFileStream.callsArgWith(2, "err", @fileStream)
it "should get the stream from s3 ", (done)->
it "should get the stream", (done)->
@handler.getFile @bucket, @key, null, =>
@FsWrapper.getFileStream.calledWith(@bucket, @key).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true
done()
it "should return the stream and error", (done)->
@ -118,14 +118,14 @@ describe "FileHandler", ->
describe "_getConvertedFile", ->
it "should getFileStream if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, true)
@FsWrapper.getFileStream.callsArgWith(2)
@PersistorManager.checkIfFileExists.callsArgWith(2, null, true)
@PersistorManager.getFileStream.callsArgWith(2)
@handler._getConvertedFile @bucket, @key, {}, =>
@FsWrapper.getFileStream.calledWith(@bucket).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket).should.equal true
done()
it "should call _getConvertedFileAndCache if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, false)
@PersistorManager.checkIfFileExists.callsArgWith(2, null, false)
@handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4)
@handler._getConvertedFile @bucket, @key, {}, =>
@handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true
@ -134,15 +134,15 @@ describe "FileHandler", ->
describe "_getConvertedFileAndCache", ->
it "should _convertFile ", (done)->
@FsWrapper.sendFileToS3 = sinon.stub().callsArgWith(3)
@FsWrapper.getFileStream = sinon.stub().callsArgWith(2)
@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
@PersistorManager.getFileStream = sinon.stub().callsArgWith(2)
@convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, =>
@handler._convertFile.called.should.equal true
@FsWrapper.sendFileToS3.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@FsWrapper.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
done()

View file

@ -11,12 +11,12 @@ SandboxedModule = require('sandboxed-module')
describe "PersistorManagerTests", ->
beforeEach ->
@s3Wrapper =
@S3PersistorManager =
getFileStream: sinon.stub()
checkIfFileExists: sinon.stub()
deleteFile: sinon.stub()
deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub()
sendStream: sinon.stub()
insertFile: sinon.stub()
describe "test s3 mixin", ->
@ -24,7 +24,7 @@ describe "PersistorManagerTests", ->
@settings =
filestoreBackend: "s3"
@requires =
"./s3Wrapper": @s3Wrapper
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
@ -34,63 +34,66 @@ describe "PersistorManagerTests", ->
it "should load getFileStream", (done) ->
@PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream()
@s3Wrapper.getFileStream.calledOnce.should.equal true
@S3PersistorManager.getFileStream.calledOnce.should.equal true
done()
it "should load checkIfFileExists", (done) ->
@PersistorManager.checkIfFileExists()
@s3Wrapper.checkIfFileExists.calledOnce.should.equal true
@S3PersistorManager.checkIfFileExists.calledOnce.should.equal true
done()
it "should load deleteFile", (done) ->
@PersistorManager.deleteFile()
@s3Wrapper.deleteFile.calledOnce.should.equal true
@S3PersistorManager.deleteFile.calledOnce.should.equal true
done()
it "should load deleteDirectory", (done) ->
@PersistorManager.deleteDirectory()
@s3Wrapper.deleteDirectory.calledOnce.should.equal true
@S3PersistorManager.deleteDirectory.calledOnce.should.equal true
done()
it "should load sendStreamToS3", (done) ->
@PersistorManager.sendStreamToS3()
@s3Wrapper.sendStreamToS3.calledOnce.should.equal true
it "should load sendStream", (done) ->
@PersistorManager.sendStream()
@S3PersistorManager.sendStream.calledOnce.should.equal true
done()
it "should load insertFile", (done) ->
@PersistorManager.insertFile()
@s3Wrapper.insertFile.calledOnce.should.equal true
@S3PersistorManager.insertFile.calledOnce.should.equal true
done()
describe "test invalid mixins", ->
describe "test unspecified mixins", ->
it "should not load a null wrapper", (done) ->
it "should load s3 when no wrapper specified", (done) ->
@settings =
@requires =
"./s3Wrapper": @s3Wrapper
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error
assert.equal("Unknown filestore backend: null",error.message)
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
@PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream()
@S3PersistorManager.getFileStream.calledOnce.should.equal true
done()
describe "test invalid mixins", ->
it "should not load an invalid wrapper", (done) ->
@settings =
filestoreBackend:"magic"
@requires =
"./s3Wrapper": @s3Wrapper
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
@fsWrapper=null
try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error
assert.equal("Unknown filestore backend: magic",error.message)
assert.isNull(@fsWrapper)
done()

View file

@ -3,10 +3,10 @@ sinon = require('sinon')
chai = require('chai')
should = chai.should()
expect = chai.expect
modulePath = "../../../app/js/s3Wrapper.js"
modulePath = "../../../app/js/S3PersistorManager.js"
SandboxedModule = require('sandboxed-module')
describe "s3WrapperTests", ->
describe "S3PersistorManagerTests", ->
beforeEach ->
@settings =
@ -39,7 +39,7 @@ describe "s3WrapperTests", ->
describe "getFileStream", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use correct key", (done)->
@ -47,20 +47,20 @@ describe "s3WrapperTests", ->
on:->
end:->
)
@s3Wrapper.getFileStream @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.get.calledWith(@key).should.equal true
done()
describe "sendFileToS3", ->
describe "sendFile", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@stubbedKnoxClient.putFile.returns on:->
it "should put file with knox", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error
done()
@ -68,49 +68,49 @@ describe "s3WrapperTests", ->
it "should delete the file and pass the error with it", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=>
@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error
done()
describe "sendStreamToS3", ->
describe "sendStream", ->
beforeEach ->
@fsPath = "to/some/where"
@origin =
on:->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.sendFileToS3 = sinon.stub().callsArgWith(3)
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
it "should send stream to LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, =>
@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
done()
it "should return the error from LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, @error)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=>
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
err.should.equal @error
done()
it "should send the file to s3", (done)->
it "should send the file to the filestore", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=>
@s3Wrapper.sendFileToS3.called.should.equal true
@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
@S3PersistorManager.sendFile.called.should.equal true
done()
describe "copyFile", ->
beforeEach ->
@sourceKey = "my/key"
@destKey = "my/dest/key"
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use knox to copy file", (done)->
@stubbedKnoxClient.copyFile.callsArgWith(2, @error)
@s3Wrapper.copyFile @bucketName, @sourceKey, @destKey, (err)=>
@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
err.should.equal @error
@stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true
done()
@ -118,14 +118,14 @@ describe "s3WrapperTests", ->
describe "deleteDirectory", ->
beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should list the contents passing them onto multi delete", (done)->
data =
Contents: [{Key:"1234"}, {Key: "456"}]
@stubbedKnoxClient.list.callsArgWith(1, null, data)
@stubbedKnoxClient.deleteMultiple.callsArgWith(1)
@s3Wrapper.deleteDirectory @bucketName, @key, (err)=>
@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
@stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
done()
@ -134,9 +134,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=>
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "delete"
@ -147,9 +147,9 @@ describe "s3WrapperTests", ->
it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=>
@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
err.should.equal @error
done()
@ -158,9 +158,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=>
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "head"
@ -171,24 +171,24 @@ describe "s3WrapperTests", ->
it "should return true for a 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=>
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal true
done()
it "should return false for a non 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:404)
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=>
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal false
done()
it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error, {})
@requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires
@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=>
@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
err.should.equal @error
done()