github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #10 from cwoac/rename_to_persistor

Browse source 
Finish abstracting persistor manager
This commit is contained in:
James Allen 2014-02-26 12:19:52 +00:00
commit 0306a3ad4a
8 changed files with 103 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
services/filestore/app/coffee/FileHandler.coffee
View file

@ -13,7 +13,7 @@ module.exports =
insertFile: (bucket, key, stream, callback)-> insertFile: (bucket, key, stream, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key) convetedKey = KeyBuilder.getConvertedFolderKey(key)
PersistorManager.deleteDirectory bucket, convetedKey, -> PersistorManager.deleteDirectory bucket, convetedKey, ->
PersistorManager.sendStreamToS3 bucket, key, stream, -> PersistorManager.sendStream bucket, key, stream, ->
callback() callback()
deleteFile: (bucket, key, callback)-> deleteFile: (bucket, key, callback)->
@ -53,7 +53,7 @@ module.exports =
if err? if err?
logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file" logger.err err:err, fsPath:fsPath, bucket:bucket, key:key, opts:opts, "something went wrong optimising png file"
return callback(err) return callback(err)
PersistorManager.sendFileToS3 bucket, convetedKey, fsPath, (err)-> PersistorManager.sendFile bucket, convetedKey, fsPath, (err)->
if err? if err?
logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file" logger.err err:err, bucket:bucket, key:key, convetedKey:convetedKey, opts:opts, "something went wrong sending the file"
return callback(err) return callback(err)
@ -67,7 +67,7 @@ module.exports =
FileConverter.thumbnail origonalFsPath, callback FileConverter.thumbnail origonalFsPath, callback
else if opts.style == "preview" else if opts.style == "preview"
FileConverter.preview origonalFsPath, callback FileConverter.preview origonalFsPath, callback
else else
throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}") throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")

8
services/filestore/app/coffee/PersistorManager.coffee
View file

@ -1,10 +1,14 @@
settings = require("settings-sharelatex") settings = require("settings-sharelatex")
logger = require("logger-sharelatex") logger = require("logger-sharelatex")
s3Wrapper = require("./s3Wrapper") S3PersistorManager = require("./S3PersistorManager")
# assume s3 if none specified
settings.filestoreBackend ||= "s3"
logger.log backend:settings.filestoreBackend, "Loading backend" logger.log backend:settings.filestoreBackend, "Loading backend"
module.exports = switch settings.filestoreBackend module.exports = switch settings.filestoreBackend
when "s3" when "s3"
s3Wrapper S3PersistorManager
else else
throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" ) throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" )

6
services/filestore/app/coffee/s3Wrapper.coffee → services/filestore/app/coffee/S3PersistorManager.coffee
View file

@ -34,7 +34,7 @@ buildDefaultOptions = (bucketName, method, key)->
module.exports = module.exports =
sendFileToS3: (bucketName, key, fsPath, callback)-> sendFile: (bucketName, key, fsPath, callback)->
s3Client = knox.createClient s3Client = knox.createClient
key: settings.s3.key key: settings.s3.key
secret: settings.s3.secret secret: settings.s3.secret
@ -57,7 +57,7 @@ module.exports =
callback err callback err
sendStreamToS3: (bucketName, key, readStream, callback)-> sendStream: (bucketName, key, readStream, callback)->
logger.log bucketName:bucketName, key:key, "sending file to s3" logger.log bucketName:bucketName, key:key, "sending file to s3"
readStream.on "error", (err)-> readStream.on "error", (err)->
logger.err bucketName:bucketName, key:key, "error on stream to send to s3" logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
@ -65,7 +65,7 @@ module.exports =
if err? if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk" logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err) return callback(err)
@sendFileToS3 bucketName, key, fsPath, callback @sendFile bucketName, key, fsPath, callback
getFileStream: (bucketName, key, callback = (err, res)->)-> getFileStream: (bucketName, key, callback = (err, res)->)->
logger.log bucketName:bucketName, key:key, "getting file from s3" logger.log bucketName:bucketName, key:key, "getting file from s3"

5
services/filestore/config/settings.development.coffee
View file

@ -4,9 +4,10 @@ module.exports =
port: 3009 port: 3009
host: "localhost" host: "localhost"
# which backend to use # which persistor to use for file storage
# current options are: # current options are:
# "s3" - Amazon S3 # "s3" - Amazon S3
# if no persistor is chosen, s3 will be used by default
filestoreBackend: "s3" filestoreBackend: "s3"
# ShareLaTeX stores binary files like images in S3. # ShareLaTeX stores binary files like images in S3.
@ -20,7 +21,7 @@ module.exports =
# Filestore health check # Filestore health check
# ---------------------- # ----------------------
# Project and file details to check in filestore when calling /health_check # Project and file details to check in persistor when calling /health_check
# health_check: # health_check:
# project_id: "" # project_id: ""
# file_id: "" # file_id: ""

16
services/filestore/test/unit/coffee/FileControllerTests.coffee
View file

@ -9,8 +9,8 @@ SandboxedModule = require('sandboxed-module')
describe "FileController", -> describe "FileController", ->
beforeEach -> beforeEach ->
@FsWrapper = @PersistorManager =
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
copyFile: sinon.stub() copyFile: sinon.stub()
deleteFile:sinon.stub() deleteFile:sinon.stub()
@ -26,7 +26,7 @@ describe "FileController", ->
@controller = SandboxedModule.require modulePath, requires: @controller = SandboxedModule.require modulePath, requires:
"./LocalFileWriter":@LocalFileWriter "./LocalFileWriter":@LocalFileWriter
"./FileHandler": @FileHandler "./FileHandler": @FileHandler
"./PersistorManager":@FsWrapper "./PersistorManager":@PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
@ -73,7 +73,7 @@ describe "FileController", ->
describe "insertFile", -> describe "insertFile", ->
it "should send bucket name key and res to FsWrapper", (done)-> it "should send bucket name key and res to PersistorManager", (done)->
@FileHandler.insertFile.callsArgWith(3) @FileHandler.insertFile.callsArgWith(3)
@res.send = => @res.send = =>
@FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true @FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true
@ -90,17 +90,17 @@ describe "FileController", ->
project_id: @oldProject_id project_id: @oldProject_id
file_id: @oldFile_id file_id: @oldFile_id
it "should send bucket name and both keys to FsWrapper", (done)-> it "should send bucket name and both keys to PersistorManager", (done)->
@FsWrapper.copyFile.callsArgWith(3) @PersistorManager.copyFile.callsArgWith(3)
@res.send = (code)=> @res.send = (code)=>
code.should.equal 200 code.should.equal 200
@FsWrapper.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true @PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
done() done()
@controller.copyFile @req, @res @controller.copyFile @req, @res
it "should send a 500 if there was an error", (done)-> it "should send a 500 if there was an error", (done)->
@FsWrapper.copyFile.callsArgWith(3, "error") @PersistorManager.copyFile.callsArgWith(3, "error")
@res.send = (code)=> @res.send = (code)=>
code.should.equal 500 code.should.equal 500
done() done()

48
services/filestore/test/unit/coffee/FileHandlerTests.coffee
View file

@ -13,12 +13,12 @@ describe "FileHandler", ->
s3: s3:
buckets: buckets:
user_files:"user_files" user_files:"user_files"
@FsWrapper = @PersistorManager =
getFileStream: sinon.stub() getFileStream: sinon.stub()
checkIfFileExists: sinon.stub() checkIfFileExists: sinon.stub()
deleteFile: sinon.stub() deleteFile: sinon.stub()
deleteDirectory: sinon.stub() deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
insertFile: sinon.stub() insertFile: sinon.stub()
@LocalFileWriter = @LocalFileWriter =
writeStream: sinon.stub() writeStream: sinon.stub()
@ -33,7 +33,7 @@ describe "FileHandler", ->
compressPng: sinon.stub() compressPng: sinon.stub()
@handler = SandboxedModule.require modulePath, requires: @handler = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings "settings-sharelatex": @settings
"./PersistorManager":@FsWrapper "./PersistorManager":@PersistorManager
"./LocalFileWriter":@LocalFileWriter "./LocalFileWriter":@LocalFileWriter
"./FileConverter":@FileConverter "./FileConverter":@FileConverter
"./KeyBuilder": @keyBuilder "./KeyBuilder": @keyBuilder
@ -50,33 +50,33 @@ describe "FileHandler", ->
describe "insertFile", -> describe "insertFile", ->
beforeEach -> beforeEach ->
@stream = {} @stream = {}
@FsWrapper.deleteDirectory.callsArgWith(2) @PersistorManager.deleteDirectory.callsArgWith(2)
@FsWrapper.sendStreamToS3.callsArgWith(3) @PersistorManager.sendStream.callsArgWith(3)
it "should send file to s3", (done)-> it "should send file to the filestore", (done)->
@handler.insertFile @bucket, @key, @stream, => @handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.sendStreamToS3.calledWith(@bucket, @key, @stream).should.equal true @PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true
done() done()
it "should delete the convetedKey folder", (done)-> it "should delete the convetedKey folder", (done)->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@handler.insertFile @bucket, @key, @stream, => @handler.insertFile @bucket, @key, @stream, =>
@FsWrapper.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true @PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done() done()
describe "deleteFile", -> describe "deleteFile", ->
beforeEach -> beforeEach ->
@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
@FsWrapper.deleteFile.callsArgWith(2) @PersistorManager.deleteFile.callsArgWith(2)
it "should tell the s3 wrapper to delete the file", (done)-> it "should tell the filestore manager to delete the file", (done)->
@handler.deleteFile @bucket, @key, => @handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @key).should.equal true @PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true
done() done()
it "should tell the s3 wrapper to delete the cached foler", (done)-> it "should tell the filestore manager to delete the cached foler", (done)->
@handler.deleteFile @bucket, @key, => @handler.deleteFile @bucket, @key, =>
@FsWrapper.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true @PersistorManager.deleteFile.calledWith(@bucket, @stubbedConvetedKey).should.equal true
done() done()
describe "getFile", -> describe "getFile", ->
@ -102,11 +102,11 @@ describe "FileHandler", ->
beforeEach -> beforeEach ->
@fileStream = {on:->} @fileStream = {on:->}
@FsWrapper.getFileStream.callsArgWith(2, "err", @fileStream) @PersistorManager.getFileStream.callsArgWith(2, "err", @fileStream)
it "should get the stream from s3 ", (done)-> it "should get the stream", (done)->
@handler.getFile @bucket, @key, null, => @handler.getFile @bucket, @key, null, =>
@FsWrapper.getFileStream.calledWith(@bucket, @key).should.equal true @PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true
done() done()
it "should return the stream and error", (done)-> it "should return the stream and error", (done)->
@ -118,14 +118,14 @@ describe "FileHandler", ->
describe "_getConvertedFile", -> describe "_getConvertedFile", ->
it "should getFileStream if it does exists", (done)-> it "should getFileStream if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, true) @PersistorManager.checkIfFileExists.callsArgWith(2, null, true)
@FsWrapper.getFileStream.callsArgWith(2) @PersistorManager.getFileStream.callsArgWith(2)
@handler._getConvertedFile @bucket, @key, {}, => @handler._getConvertedFile @bucket, @key, {}, =>
@FsWrapper.getFileStream.calledWith(@bucket).should.equal true @PersistorManager.getFileStream.calledWith(@bucket).should.equal true
done() done()
it "should call _getConvertedFileAndCache if it does exists", (done)-> it "should call _getConvertedFileAndCache if it does exists", (done)->
@FsWrapper.checkIfFileExists.callsArgWith(2, null, false) @PersistorManager.checkIfFileExists.callsArgWith(2, null, false)
@handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4) @handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4)
@handler._getConvertedFile @bucket, @key, {}, => @handler._getConvertedFile @bucket, @key, {}, =>
@handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true @handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true
@ -134,15 +134,15 @@ describe "FileHandler", ->
describe "_getConvertedFileAndCache", -> describe "_getConvertedFileAndCache", ->
it "should _convertFile ", (done)-> it "should _convertFile ", (done)->
@FsWrapper.sendFileToS3 = sinon.stub().callsArgWith(3) @PersistorManager.sendFile = sinon.stub().callsArgWith(3)
@FsWrapper.getFileStream = sinon.stub().callsArgWith(2) @PersistorManager.getFileStream = sinon.stub().callsArgWith(2)
@convetedKey = @key+"converted" @convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath) @handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) @ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, => @handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, =>
@handler._convertFile.called.should.equal true @handler._convertFile.called.should.equal true
@FsWrapper.sendFileToS3.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true @PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@FsWrapper.getFileStream.calledWith(@bucket, @convetedKey).should.equal true @PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true @ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
done() done()

41
services/filestore/test/unit/coffee/PersistorManagerTests.coffee
View file

@ -11,12 +11,12 @@ SandboxedModule = require('sandboxed-module')
describe "PersistorManagerTests", -> describe "PersistorManagerTests", ->
beforeEach -> beforeEach ->
@s3Wrapper = @S3PersistorManager =
getFileStream: sinon.stub() getFileStream: sinon.stub()
checkIfFileExists: sinon.stub() checkIfFileExists: sinon.stub()
deleteFile: sinon.stub() deleteFile: sinon.stub()
deleteDirectory: sinon.stub() deleteDirectory: sinon.stub()
sendStreamToS3: sinon.stub() sendStream: sinon.stub()
insertFile: sinon.stub() insertFile: sinon.stub()
describe "test s3 mixin", -> describe "test s3 mixin", ->
@ -24,7 +24,7 @@ describe "PersistorManagerTests", ->
@settings = @settings =
filestoreBackend: "s3" filestoreBackend: "s3"
@requires = @requires =
"./s3Wrapper": @s3Wrapper "./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
@ -34,63 +34,66 @@ describe "PersistorManagerTests", ->
it "should load getFileStream", (done) -> it "should load getFileStream", (done) ->
@PersistorManager.should.respondTo("getFileStream") @PersistorManager.should.respondTo("getFileStream")
@PersistorManager.getFileStream() @PersistorManager.getFileStream()
@s3Wrapper.getFileStream.calledOnce.should.equal true @S3PersistorManager.getFileStream.calledOnce.should.equal true
done() done()
it "should load checkIfFileExists", (done) -> it "should load checkIfFileExists", (done) ->
@PersistorManager.checkIfFileExists() @PersistorManager.checkIfFileExists()
@s3Wrapper.checkIfFileExists.calledOnce.should.equal true @S3PersistorManager.checkIfFileExists.calledOnce.should.equal true
done() done()
it "should load deleteFile", (done) -> it "should load deleteFile", (done) ->
@PersistorManager.deleteFile() @PersistorManager.deleteFile()
@s3Wrapper.deleteFile.calledOnce.should.equal true @S3PersistorManager.deleteFile.calledOnce.should.equal true
done() done()
it "should load deleteDirectory", (done) -> it "should load deleteDirectory", (done) ->
@PersistorManager.deleteDirectory() @PersistorManager.deleteDirectory()
@s3Wrapper.deleteDirectory.calledOnce.should.equal true @S3PersistorManager.deleteDirectory.calledOnce.should.equal true
done() done()
it "should load sendStreamToS3", (done) -> it "should load sendStream", (done) ->
@PersistorManager.sendStreamToS3() @PersistorManager.sendStream()
@s3Wrapper.sendStreamToS3.calledOnce.should.equal true @S3PersistorManager.sendStream.calledOnce.should.equal true
done() done()
it "should load insertFile", (done) -> it "should load insertFile", (done) ->
@PersistorManager.insertFile() @PersistorManager.insertFile()
@s3Wrapper.insertFile.calledOnce.should.equal true @S3PersistorManager.insertFile.calledOnce.should.equal true
done() done()
describe "test invalid mixins", -> describe "test unspecified mixins", ->
it "should not load a null wrapper", (done) -> it "should load s3 when no wrapper specified", (done) ->
@settings = @settings =
@requires = @requires =
"./s3Wrapper": @s3Wrapper "./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
err:-> err:->
try @PersistorManager=SandboxedModule.require modulePath, requires: @requires
@PersistorManager=SandboxedModule.require modulePath, requires: @requires @PersistorManager.should.respondTo("getFileStream")
catch error @PersistorManager.getFileStream()
assert.equal("Unknown filestore backend: null",error.message) @S3PersistorManager.getFileStream.calledOnce.should.equal true
done() done()
describe "test invalid mixins", ->
it "should not load an invalid wrapper", (done) -> it "should not load an invalid wrapper", (done) ->
@settings = @settings =
filestoreBackend:"magic" filestoreBackend:"magic"
@requires = @requires =
"./s3Wrapper": @s3Wrapper "./S3PersistorManager": @S3PersistorManager
"settings-sharelatex": @settings "settings-sharelatex": @settings
"logger-sharelatex": "logger-sharelatex":
log:-> log:->
err:-> err:->
@fsWrapper=null
try try
@PersistorManager=SandboxedModule.require modulePath, requires: @requires @PersistorManager=SandboxedModule.require modulePath, requires: @requires
catch error catch error
assert.equal("Unknown filestore backend: magic",error.message) assert.equal("Unknown filestore backend: magic",error.message)
assert.isNull(@fsWrapper)
done() done()

68
services/filestore/test/unit/coffee/s3WrapperTests.coffee → services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee
View file

@ -3,10 +3,10 @@ sinon = require('sinon')
chai = require('chai') chai = require('chai')
should = chai.should() should = chai.should()
expect = chai.expect expect = chai.expect
modulePath = "../../../app/js/s3Wrapper.js" modulePath = "../../../app/js/S3PersistorManager.js"
SandboxedModule = require('sandboxed-module') SandboxedModule = require('sandboxed-module')
describe "s3WrapperTests", -> describe "S3PersistorManagerTests", ->
beforeEach -> beforeEach ->
@settings = @settings =
@ -39,7 +39,7 @@ describe "s3WrapperTests", ->
describe "getFileStream", -> describe "getFileStream", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use correct key", (done)-> it "should use correct key", (done)->
@ -47,20 +47,20 @@ describe "s3WrapperTests", ->
on:-> on:->
end:-> end:->
) )
@s3Wrapper.getFileStream @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.get.calledWith(@key).should.equal true @stubbedKnoxClient.get.calledWith(@key).should.equal true
done() done()
describe "sendFileToS3", -> describe "sendFile", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@stubbedKnoxClient.putFile.returns on:-> @stubbedKnoxClient.putFile.returns on:->
it "should put file with knox", (done)-> it "should put file with knox", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error) @stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error err.should.equal @error
done() done()
@ -68,49 +68,49 @@ describe "s3WrapperTests", ->
it "should delete the file and pass the error with it", (done)-> it "should delete the file and pass the error with it", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@stubbedKnoxClient.putFile.callsArgWith(2, @error) @stubbedKnoxClient.putFile.callsArgWith(2, @error)
@s3Wrapper.sendFileToS3 @bucketName, @key, @fsPath, (err)=> @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
@stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true @stubbedKnoxClient.putFile.calledWith(@fsPath, @key).should.equal true
err.should.equal @error err.should.equal @error
done() done()
describe "sendStreamToS3", -> describe "sendStream", ->
beforeEach -> beforeEach ->
@fsPath = "to/some/where" @fsPath = "to/some/where"
@origin = @origin =
on:-> on:->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.sendFileToS3 = sinon.stub().callsArgWith(3) @S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
it "should send stream to LocalFileWriter", (done)-> it "should send stream to LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath) @LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, => @S3PersistorManager.sendStream @bucketName, @key, @origin, =>
@LocalFileWriter.writeStream.calledWith(@origin).should.equal true @LocalFileWriter.writeStream.calledWith(@origin).should.equal true
done() done()
it "should return the error from LocalFileWriter", (done)-> it "should return the error from LocalFileWriter", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2, @error) @LocalFileWriter.writeStream.callsArgWith(2, @error)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=> @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
err.should.equal @error err.should.equal @error
done() done()
it "should send the file to s3", (done)-> it "should send the file to the filestore", (done)->
@LocalFileWriter.deleteFile.callsArgWith(1) @LocalFileWriter.deleteFile.callsArgWith(1)
@LocalFileWriter.writeStream.callsArgWith(2) @LocalFileWriter.writeStream.callsArgWith(2)
@s3Wrapper.sendStreamToS3 @bucketName, @key, @origin, (err)=> @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
@s3Wrapper.sendFileToS3.called.should.equal true @S3PersistorManager.sendFile.called.should.equal true
done() done()
describe "copyFile", -> describe "copyFile", ->
beforeEach -> beforeEach ->
@sourceKey = "my/key" @sourceKey = "my/key"
@destKey = "my/dest/key" @destKey = "my/dest/key"
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should use knox to copy file", (done)-> it "should use knox to copy file", (done)->
@stubbedKnoxClient.copyFile.callsArgWith(2, @error) @stubbedKnoxClient.copyFile.callsArgWith(2, @error)
@s3Wrapper.copyFile @bucketName, @sourceKey, @destKey, (err)=> @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
err.should.equal @error err.should.equal @error
@stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true @stubbedKnoxClient.copyFile.calledWith(@sourceKey, @destKey).should.equal true
done() done()
@ -118,14 +118,14 @@ describe "s3WrapperTests", ->
describe "deleteDirectory", -> describe "deleteDirectory", ->
beforeEach -> beforeEach ->
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
it "should list the contents passing them onto multi delete", (done)-> it "should list the contents passing them onto multi delete", (done)->
data = data =
Contents: [{Key:"1234"}, {Key: "456"}] Contents: [{Key:"1234"}, {Key: "456"}]
@stubbedKnoxClient.list.callsArgWith(1, null, data) @stubbedKnoxClient.list.callsArgWith(1, null, data)
@stubbedKnoxClient.deleteMultiple.callsArgWith(1) @stubbedKnoxClient.deleteMultiple.callsArgWith(1)
@s3Wrapper.deleteDirectory @bucketName, @key, (err)=> @S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
@stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true @stubbedKnoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
done() done()
@ -134,9 +134,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)-> it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1) @request = sinon.stub().callsArgWith(1)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=> @S3PersistorManager.deleteFile @bucketName, @key, (err)=>
opts = @request.args[0][0] opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "delete" opts.method.should.equal "delete"
@ -147,9 +147,9 @@ describe "s3WrapperTests", ->
it "should return the error", (done)-> it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error) @request = sinon.stub().callsArgWith(1, @error)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.deleteFile @bucketName, @key, (err)=> @S3PersistorManager.deleteFile @bucketName, @key, (err)=>
err.should.equal @error err.should.equal @error
done() done()
@ -158,9 +158,9 @@ describe "s3WrapperTests", ->
it "should use correct options", (done)-> it "should use correct options", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200) @request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
opts = @request.args[0][0] opts = @request.args[0][0]
assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName})
opts.method.should.equal "head" opts.method.should.equal "head"
@ -171,24 +171,24 @@ describe "s3WrapperTests", ->
it "should return true for a 200", (done)-> it "should return true for a 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:200) @request = sinon.stub().callsArgWith(1, null, statusCode:200)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal true exists.should.equal true
done() done()
it "should return false for a non 200", (done)-> it "should return false for a non 200", (done)->
@request = sinon.stub().callsArgWith(1, null, statusCode:404) @request = sinon.stub().callsArgWith(1, null, statusCode:404)
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err, exists)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
exists.should.equal false exists.should.equal false
done() done()
it "should return the error", (done)-> it "should return the error", (done)->
@request = sinon.stub().callsArgWith(1, @error, {}) @request = sinon.stub().callsArgWith(1, @error, {})
@requires["request"] = @request @requires["request"] = @request
@s3Wrapper = SandboxedModule.require modulePath, requires: @requires @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
@s3Wrapper.checkIfFileExists @bucketName, @key, (err)=> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
err.should.equal @error err.should.equal @error
done() done()