github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-14 20:40:17 -05:00
Code Issues Projects Releases Packages Wiki Activity

make s3 chosen if no config option specified

Browse source 
rename *wrapper  to FilestoreManager
remove ToS3 from filestore commands
rename configuration option for filestore.
add details of interface to FilestoreWrapper.coffee for reference.
remove api comments
rename settings.filestoreManager to settings.persistorManager
This commit is contained in:
Oliver Matthews 2014-02-25 16:38:13 +00:00
parent c9af5e5420
commit 4b5f9ad4fb
8 changed files with 114 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
services/filestore/app/coffee/FileHandler.coffee
View file

@ -13,7 +13,7 @@ module.exports =
insertFile: (bucket, key, stream, callback)-> insertFile: (bucket, key, stream, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key) convetedKey = KeyBuilder.getConvertedFolderKey(key)
PersistorManager.deleteDirectory bucket, convetedKey, -> PersistorManager.deleteDirectory bucket, convetedKey, ->
PersistorManager.sendStreamToS3 bucket, key, stream, -> PersistorManager.sendStream bucket, key, stream, ->
callback() callback()
deleteFile: (bucket, key, callback)-> deleteFile: (bucket, key, callback)->
@ -53,7 +53,7 @@ module.exports =
if err? if err?
logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file" logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file"
return callback(err) return callback(err)
PersistorManager.sendFileToS3 bucket, convetedKey, fsPath, (err)-> PersistorManager.sendFile bucket, convetedKey, fsPath, (err)->
if err? if err?
logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file" logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file"
return callback(err) return callback(err)
@ -67,7 +67,7 @@ module.exports =
FileConverter.thumbnail origonalFsPath, callback FileConverter.thumbnail origonalFsPath, callback
else if opts.style == "preview" else if opts.style == "preview"
FileConverter.preview origonalFsPath, callback FileConverter.preview origonalFsPath, callback
else else
throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}") throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")

12
services/filestore/app/coffee/PersistorManager.coffee
View file

@ -1,10 +1,10 @@
settings = require("settings-sharelatex") settings = require("settings-sharelatex")
logger = require("logger-sharelatex") logger = require("logger-sharelatex")
s3Wrapper = require("./s3Wrapper") S3PersistorManager = require("./S3PersistorManager")
logger.log backend:settings.filestoreBackend, "Loading backend" logger.log backend:settings.persistorManager, "Loading backend"
module.exports = switch settings.filestoreBackend module.exports = switch settings.persistorManager
when "s3" when "s3",null
s3Wrapper S3PersistorManager
else else
throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" ) throw new Error( "Unknown filestore backend: #{settings.persistorManager}" )

6
services/filestore/app/coffee/s3Wrapper.coffee → services/filestore/app/coffee/S3PersistorManager.coffee
View file

@ -34,7 +34,7 @@ buildDefaultOptions = (bucketName, method, key)->
module.exports = module.exports =
sendFileToS3: (bucketName, key, fsPath, callback)-> sendFile: (bucketName, key, fsPath, callback)->
s3Client = knox.createClient s3Client = knox.createClient
key: settings.s3.key key: settings.s3.key
secret: settings.s3.secret secret: settings.s3.secret
@ -57,7 +57,7 @@ module.exports =
callback err callback err
sendStreamToS3: (bucketName, key, readStream, callback)-> sendStream: (bucketName, key, readStream, callback)->
logger.log bucketName:bucketName, key:key, "sending file to s3" logger.log bucketName:bucketName, key:key, "sending file to s3"
readStream.on "error", (err)-> readStream.on "error", (err)->
logger.err bucketName:bucketName, key:key, "error on stream to send to s3" logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
@ -65,7 +65,7 @@ module.exports =
if err? if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk" logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err) return callback(err)
@sendFileToS3 bucketName, key, fsPath, callback @sendFile bucketName, key, fsPath, callback
getFileStream: (bucketName, key, callback = (err, res)->)-> getFileStream: (bucketName, key, callback = (err, res)->)->
logger.log bucketName:bucketName, key:key, "getting file from s3" logger.log bucketName:bucketName, key:key, "getting file from s3"

7
services/filestore/config/settings.development.coffee
View file

@ -4,10 +4,11 @@ module.exports =
port: 3009 port: 3009
host: "localhost" host: "localhost"
# which backend to use # which persistor to use for file storage
# current options are: # current options are:
# "s3" - Amazon S3 # "s3" - Amazon S3
filestoreBackend: "s3" # if no persistor is chosen, s3 will be used by default
persistorManager: "s3"
# ShareLaTeX stores binary files like images in S3. # ShareLaTeX stores binary files like images in S3.
# Fill in your Amazon S3 credentials below. # Fill in your Amazon S3 credentials below.
@ -20,7 +21,7 @@ module.exports =
# Filestore health check # Filestore health check
# ---------------------- # ----------------------
# Project and file details to check in filestore when calling /health_check # Project and file details to check in persistor when calling /health_check
# health_check: # health_check:
# project_id: "" # project_id: ""
# file_id: "" # file_id: ""

16
services/filestore/test/unit/coffee/FileControllerTests.coffee
View file

@ -9,8 +9,8 @@ SandboxedModule = require('sandboxed-module')
describe "FileController", -> describe "FileController", ->
beforeEach -> beforeEach ->
@FsWrapper = @PersistorManager =
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
copyFile: sinon.stub() copyFile: sinon.stub()
deleteFile:sinon.stub() deleteFile:sinon.stub()
@ -26,7 +26,7 @@ describe "FileController", ->
@controller = SandboxedModule.require modulePath, requires: @controller = SandboxedModule.require modulePath, requires:
"./LocalFileWriter":@LocalFileWriter "./LocalFileWriter":@LocalFileWriter
"./FileHandler": @FileHandler "./FileHandler": @FileHandler
"./PersistorManager":@FsWrapper "./PersistorManager":@PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
@ -73,7 +73,7 @@ describe "FileController", ->
describe "insertFile", -> describe "insertFile", ->
it "should send bucket name key and res to FsWrapper", (done)-> it "should send bucket name key and res to PersistorManager", (done)->
@FileHandler.insertFile.callsArgWith(3) @FileHandler.insertFile.callsArgWith(3)
@res.send = => @res.send = =>
@FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true @FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true
@ -90,17 +90,17 @@ describe "FileController", ->
project_id: @oldProject_id project_id: @oldProject_id
file_id: @oldFile_id file_id: @oldFile_id
it "should send bucket name and both keys to FsWrapper", (done)-> it "should send bucket name and both keys to PersistorManager", (done)->
@FsWrapper.copyFile.callsArgWith(3) @PersistorManager.copyFile.callsArgWith(3)
@res.send = (code)=> @res.send = (code)=>
code.should.equal 200 code.should.equal 200
@FsWrapper.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true @PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
done() done()
@controller.copyFile @req, @res @controller.copyFile @req, @res
it "should send a 500 if there was an error", (done)-> it "should send a 500 if there was an error", (done)->
@FsWrapper.copyFile.callsArgWith(3, "error") @PersistorManager.copyFile.callsArgWith(3, "error")
@res.send = (code)=> @res.send = (code)=>
code.should.equal 500 code.should.equal 500
done() done()

48
services/filestore/test/unit/coffee/FileHandlerTests.coffee
View file

@ -13,12 +13,12 @@ describe "FileHandler", ->
s3: s3:
buckets: buckets:
user_files:"user_files" user_files:"user_files"
@FsWrapper = @PersistorManager =
getFileStream: sinon.stub() getFileStream: sinon.stub()
checkIfFileExists: sinon.stub() checkIfFileExists: sinon.stub()
deleteFile: sinon.stub() deleteFile: sinon.stub()
deleteDirectory: sinon.stub() deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
insertFile: sinon.stub() insertFile: sinon.stub()
@LocalFileWriter = @LocalFileWriter =
writeStream: sinon.stub() writeStream: sinon.stub()
@ -33,7 +33,7 @@ describe "FileHandler", ->
compressPng: sinon.stub() compressPng: sinon.stub()
@handler = SandboxedModule.require modulePath, requires: @handler = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings "settings-sharelatex": @settings
"./PersistorManager":@FsWrapper "./PersistorManager":@PersistorManager
"./LocalFileWriter":@LocalFileWriter "./LocalFileWriter":@LocalFileWriter
"./FileConverter":@FileConverter "./FileConverter":@FileConverter
"./KeyBuilder": @keyBuilder "./KeyBuilder": @keyBuilder
@ -50,33 +50,33 @@ describe "FileHandler", ->
describe "insertFile", -> describe "insertFile", ->
beforeEach -> beforeEach ->
@stream = {} @stream = {}
@FsWrapper.deleteDirectory.callsArgWith(2) @PersistorManager.deleteDirectory.callsArgWith(2)
@FsWrapper.sendStreamToS3.callsArgWith(3) @PersistorManager.sendStream.callsArgWith(3)
it "should send file to s3", (done)-> it "should send file to the filestore", (done)->
@handler.insertFile @bucket, @key, @stream, => @handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.sendStreamToS3.calledWith(@bucket, @key, @stream).should.equal true @PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true
done() done()
it "should delete the convetedKey folder", (done)-> it "should delete the convetedKey folder", (done)->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@handler.insertFile @bucket, @key, @stream, => @handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true @PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done() done()
describe "deleteFile", -> describe "deleteFile", ->
beforeEach -> beforeEach ->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@FsWrapper.deleteFile.callsArgWith(2) @PersistorManager.deleteFile.callsArgWith(2)
it "should tell the s3 wrapper to delete the file", (done)-> it "should tell the filestore manager to delete the file", (done)->
@handler.deleteFile @bucket, @key, => @handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @key).should.equal true @PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true
done() done()
it "should tell the s3 wrapper to delete the cached foler", (done)-> it "should tell the filestore manager to delete the cached foler", (done)->
@handler.deleteFile @bucket, @key, => @handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true @PersistorManager.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done() done()
describe "getFile", -> describe "getFile", ->
@ -102,11 +102,11 @@ describe "FileHandler", ->
beforeEach -> beforeEach ->
@fileStream = {on:->} @fileStream = {on:->}
@FsWrapper.getFileStream.callsArgWith(2, "err", @fileStream) @PersistorManager.getFileStream.callsArgWith(2, "err", @fileStream)
it "should get the stream from s3 ", (done)-> it "should get the stream", (done)->
@handler.getFile @bucket, @key, null, => @handler.getFile @bucket, @key, null, =>
@FsWrapper.getFileStream.calledWith(@bucket, @key).should.equal true @PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true
done() done()
it "should return the stream and error", (done)-> it "should return the stream and error", (done)->
@ -118,14 +118,14 @@ describe "FileHandler", ->
describe "_getConvertedFile", -> describe "_getConvertedFile", ->
it "should getFileStream if it does exists", (done)-> it "should getFileStream if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, true) @PersistorManager.checkIfFileExists.callsArgWith(2, null, true)
@FsWrapper.getFileStream.callsArgWith(2) @PersistorManager.getFileStream.callsArgWith(2)
@handler._getConvertedFile @bucket, @key, {}, => @handler._getConvertedFile @bucket, @key, {}, =>
@FsWrapper.getFileStream.calledWith(@bucket).should.equal true @PersistorManager.getFileStream.calledWith(@bucket).should.equal true
done() done()
it "should call _getConvertedFileAndCache if it does exists", (done)-> it "should call _getConvertedFileAndCache if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, false) @PersistorManager.checkIfFileExists.callsArgWith(2, null, false)
@handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4) @handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4)
@handler._getConvertedFile @bucket, @key, {}, => @handler._getConvertedFile @bucket, @key, {}, =>
@handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true @handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true
@ -134,15 +134,15 @@ describe "FileHandler", ->
describe "_getConvertedFileAndCache", -> describe "_getConvertedFileAndCache", ->
it "should _convertFile ", (done)-> it "should _convertFile ", (done)->
@FsWrapper.sendFileToS3 = sinon.stub().callsArgWith(3) @PersistorManager.sendFile = sinon.stub().callsArgWith(3)
@FsWrapper.getFileStream = sinon.stub().callsArgWith(2) @PersistorManager.getFileStream = sinon.stub().callsArgWith(2)
@convetedKey = @key+"converted" @convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath) @handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) @ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, => @handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, =>
@handler._convertFile.called.should.equal true @handler._convertFile.called.should.equal true
@FsWrapper.sendFileToS3.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true @PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@FsWrapper.getFileStream.calledWith(@bucket, @convetedKey).should.equal true @PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true @ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
done() done()

61
services/filestore/test/unit/coffee/PersistorManagerTests.coffee
View file

@ -11,20 +11,20 @@ SandboxedModule = require('sandboxed-module')
describe "PersistorManagerTests", -> describe "PersistorManagerTests", ->
beforeEach -> beforeEach ->
@s3Wrapper = @S3PersistorManager =
getFileStream: sinon.stub() getFileStream: sinon.stub()
checkIfFileExists: sinon.stub() checkIfFileExists: sinon.stub()
deleteFile: sinon.stub() deleteFile: sinon.stub()
deleteDirectory: sinon.stub() deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
insertFile: sinon.stub() insertFile: sinon.stub()
describe "test s3 mixin", -> describe "test s3 mixin", ->
beforeEach -> beforeEach ->
@settings = @settings =
filestoreBackend: "s3" persistorManager: "s3"
@requires = @requires =
"./s3Wrapper": @s3Wrapper "./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
@ -34,63 +34,66 @@ describe "PersistorManagerTests", ->
it "should load getFileStream", (done) -> it "should load getFileStream", (done) ->
@PersistorManager.should.respondTo("getFileStream") @PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream() @PersistorManager.getFileStream()
@s3Wrapper.getFileStream.calledOnce.should.equal true @S3PersistorManager.getFileStream.calledOnce.should.equal true
done() done()
it "should load checkIfFileExists", (done) -> it "should load checkIfFileExists", (done) ->
@PersistorManager.checkIfFileExists() @PersistorManager.checkIfFileExists()
@s3Wrapper.checkIfFileExists.calledOnce.should.equal true @S3PersistorManager.checkIfFileExists.calledOnce.should.equal true
done() done()
it "should load deleteFile", (done) -> it "should load deleteFile", (done) ->
@PersistorManager.deleteFile() @PersistorManager.deleteFile()
@s3Wrapper.deleteFile.calledOnce.should.equal true @S3PersistorManager.deleteFile.calledOnce.should.equal true
done() done()
it "should load deleteDirectory", (done) -> it "should load deleteDirectory", (done) ->
@PersistorManager.deleteDirectory() @PersistorManager.deleteDirectory()
@s3Wrapper.deleteDirectory.calledOnce.should.equal true @S3PersistorManager.deleteDirectory.calledOnce.should.equal true
done() done()
it "should load sendStreamToS3", (done) -> it "should load sendStream", (done) ->
@PersistorManager.sendStreamToS3() @PersistorManager.sendStream()
@s3Wrapper.sendStreamToS3.calledOnce.should.equal true @S3PersistorManager.sendStream.calledOnce.should.equal true
done() done()
it "should load insertFile", (done) -> it "should load insertFile", (done) ->
@PersistorManager.insertFile() @PersistorManager.insertFile()
@s3Wrapper.insertFile.calledOnce.should.equal true @S3PersistorManager.insertFile.calledOnce.should.equal true
done()
describe "test unspecified mixins", ->
it "should load s3 when no wrapper specified", (done) ->
@settings =
@requires =
"./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
@PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream()
@S3PersistorManager.getFileStream.calledOnce.should.equal true
done() done()
describe "test invalid mixins", -> describe "test invalid mixins", ->
it "should not load a null wrapper", (done) ->
@settings =
@requires =
"./s3Wrapper": @s3Wrapper
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error
assert.equal("Unknown filestore backend: null",error.message)
done()
it "should not load an invalid wrapper", (done) -> it "should not load an invalid wrapper", (done) ->
@settings = @settings =
filestoreBackend:"magic" persistorManager:"magic"
@requires = @requires =
"./s3Wrapper": @s3Wrapper "./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
err:-> err:->
@fsWrapper=null
try try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires @PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error catch error
assert.equal("Unknown filestore backend: magic",error.message) assert.equal("Unknown filestore backend: magic",error.message)
assert.isNull(@fsWrapper)
done() done()

68
services/filestore/test/unit/coffee/s3WrapperTests.coffee → services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee
View file

@ -3,10 +3,10 @@ sinon = require('sinon')
chai = require('chai') chai = require('chai')
should = chai.should() should = chai.should()
expect = chai.expect expect = chai.expect
modulePath = "../../../app/js/s3Wrapper.js" modulePath = "../../../app/js/S3PersistorManager.js"
SandboxedModule = require('sandboxed-module') SandboxedModule = require('sandboxed-module')
describe "s3WrapperTests", -> describe "S3PersistorManagerTests", ->
beforeEach -> beforeEach ->
@settings = @settings =
@ -39,7 +39,7 @@ describe "s3WrapperTests", ->
describe "getFileStream", -> describe "getFileStream", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use correct key", (done)-> it "should use correct key", (done)->
@ -47,20 +47,20 @@ describe "s3WrapperTests", ->
on:-> on:->
end:-> end:->
) )
@s3Wrapper.getFileStream @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.get.calledWith(@key).should.equal true @stubbedKnoxClient.get.calledWith(@key).should.equal true
done() done()
describe "sendFileToS3", -> describe "sendFile", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@stubbedKnoxClient.putFile.returns on:-> @stubbedKnoxClient.putFile.returns on:->
it "should put file with knox", (done)-> it "should put file with knox", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error) @stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error err.should.equal @error
done() done()
@ -68,49 +68,49 @@ describe "s3WrapperTests", ->
it "should delete the file and pass the error with it", (done)-> it "should delete the file and pass the error with it", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error) @stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error err.should.equal @error
done() done()
describe "sendStreamToS3", -> describe "sendStream", ->
beforeEach -> beforeEach ->
@fsPath = "to/some/where" @fsPath = "to/some/where"
@origin = @origin =
on:-> on:->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.sendFileToS3 = sinon.stub().callsArgWith(3) @S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
it "should send stream to LocalFileWriter", (done)-> it "should send stream to LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath) @LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, => @S3PersistorManager.sendStream @bucketName, @key, @origin, =>
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true @LocalFileWriter.writeStream.calledWith(@origin).should.equal true
done() done()
it "should return the error from LocalFileWriter", (done)-> it "should return the error from LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, @error) @LocalFileWriter.writeStream.callsArgWith(2, @error)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=> @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
err.should.equal @error err.should.equal @error
done() done()
it "should send the file to s3", (done)-> it "should send the file to the filestore", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2) @LocalFileWriter.writeStream.callsArgWith(2)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=> @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
@s3Wrapper.sendFileToS3.called.should.equal true @S3PersistorManager.sendFile.called.should.equal true
done() done()
describe "copyFile", -> describe "copyFile", ->
beforeEach -> beforeEach ->
@sourceKey = "my/key" @sourceKey = "my/key"
@destKey = "my/dest/key" @destKey = "my/dest/key"
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use knox to copy file", (done)-> it "should use knox to copy file", (done)->
@stubbedKnoxClient.copyFile.callsArgWith(2, @error) @stubbedKnoxClient.copyFile.callsArgWith(2, @error)
@s3Wrapper.copyFile @bucketName, @sourceKey, @destKey, (err)=> @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
err.should.equal @error err.should.equal @error
@stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true @stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true
done() done()
@ -118,14 +118,14 @@ describe "s3WrapperTests", ->
describe "deleteDirectory", -> describe "deleteDirectory", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should list the contents passing them onto multi delete", (done)-> it "should list the contents passing them onto multi delete", (done)->
data = data =
Contents: [{Key:"1234"}, {Key: "456"}] Contents: [{Key:"1234"}, {Key: "456"}]
@stubbedKnoxClient.list.callsArgWith(1, null, data) @stubbedKnoxClient.list.callsArgWith(1, null, data)
@stubbedKnoxClient.deleteMultiple.callsArgWith(1) @stubbedKnoxClient.deleteMultiple.callsArgWith(1)
@s3Wrapper.deleteDirectory @bucketName, @key, (err)=> @S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
@stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true @stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
done() done()
@ -134,9 +134,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)-> it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1) @request = sinon.stub().callsArgWith(1)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=> @S3PersistorManager.deleteFile @bucketName, @key, (err)=>
opts = @request.args[0][0] opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "delete" opts.method.should.equal "delete"
@ -147,9 +147,9 @@ describe "s3WrapperTests", ->
it "should return the error", (done)-> it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error) @request = sinon.stub().callsArgWith(1, @error)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=> @S3PersistorManager.deleteFile @bucketName, @key, (err)=>
err.should.equal @error err.should.equal @error
done() done()
@ -158,9 +158,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)-> it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200) @request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
opts = @request.args[0][0] opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "head" opts.method.should.equal "head"
@ -171,24 +171,24 @@ describe "s3WrapperTests", ->
it "should return true for a 200", (done)-> it "should return true for a 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200) @request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal true exists.should.equal true
done() done()
it "should return false for a non 200", (done)-> it "should return false for a non 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:404) @request = sinon.stub().callsArgWith(1, null, statusCode:404)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal false exists.should.equal false
done() done()
it "should return the error", (done)-> it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error, {}) @request = sinon.stub().callsArgWith(1, @error, {})
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
err.should.equal @error err.should.equal @error
done() done()