Merge pull request #19 from EDP-Sciences/master

Add a PersistorManager library using the aws-sdk library
This commit is contained in:
Henry Oswald 2015-11-25 14:50:38 +00:00
commit d6b3262ac7
7 changed files with 367 additions and 31 deletions

View file

@ -40,6 +40,7 @@ test/IntergrationTests/js/*
data/*/*
app.js
cluster.js
app/js/*
test/IntergrationTests/js/*
test/UnitTests/js/*

View file

@ -0,0 +1,87 @@
logger = require "logger-sharelatex"
aws = require "aws-sdk"
_ = require "underscore"
fs = require "fs"
Errors = require "./Errors"
s3 = new aws.S3()
module.exports =
sendFile: (bucketName, key, fsPath, callback)->
logger.log bucketName:bucketName, key:key, "send file data to s3"
stream = fs.createReadStream fsPath
s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
if err?
logger.err err: err, Bucket: bucketName, Key: key, "error sending file data to s3"
callback err
sendStream: (bucketName, key, stream, callback)->
logger.log bucketName:bucketName, key:key, "send file stream to s3"
s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
if err?
logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3"
callback err
getFileStream: (bucketName, key, opts, callback = (err, res)->)->
logger.log bucketName:bucketName, key:key, "get file stream from s3"
callback = _.once callback
params =
Bucket:bucketName
Key: key
if opts.start? and opts.end?
params['Range'] = "bytes=#{opts.start}-#{opts.end}"
request = s3.getObject params
stream = request.createReadStream()
stream.on 'readable', () ->
callback null, stream
stream.on 'error', (err) ->
logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3"
if err.code == 'NoSuchKey'
return callback new Errors.NotFoundError "File not found in S3: #{bucketName}:#{key}"
callback err
copyFile: (bucketName, sourceKey, destKey, callback)->
logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3"
source = bucketName + '/' + sourceKey
s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) ->
if err?
logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in s3"
callback err
deleteFile: (bucketName, key, callback)->
logger.log bucketName:bucketName, key:key, "delete file in s3"
s3.deleteObject {Bucket: bucketName, Key: key}, (err) ->
if err?
logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting file in s3"
callback err
deleteDirectory: (bucketName, key, callback)->
logger.log bucketName:bucketName, key:key, "delete directory in s3"
s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) ->
if err?
logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3"
return callback err
if data.Contents.length == 0
logger.log bucketName:bucketName, key:key, "the directory is empty"
return callback()
keys = _.map data.Contents, (entry)->
Key: entry.Key
s3.deleteObjects
Bucket: bucketName
Delete:
Objects: keys
Quiet: true
, (err) ->
if err?
logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3"
callback err
checkIfFileExists:(bucketName, key, callback)->
logger.log bucketName:bucketName, key:key, "check file existence in s3"
s3.headObject {Bucket: bucketName, Key: key}, (err, data) ->
if err?
return (callback null, false) if err.code == 'NotFound'
logger.err err:err, bucketName:bucketName, key:key, "something went wrong checking head in s3"
return callback err
callback null, data.ETag?

View file

@ -3,6 +3,7 @@ fs = require("fs")
LocalFileWriter = require("./LocalFileWriter")
Errors = require('./Errors')
rimraf = require("rimraf")
_ = require "underscore"
filterName = (key) ->
return key.replace /\//g, "_"
@ -29,9 +30,7 @@ module.exports =
# opts may be {start: Number, end: Number}
getFileStream: (location, name, opts, _callback = (err, res)->) ->
callback = (args...) ->
_callback(args...)
_callback = () ->
callback = _.once _callback
filteredName = filterName name
logger.log location:location, name:filteredName, "getting file"
sourceStream = fs.createReadStream "#{location}/#{filteredName}", opts

View file

@ -10,16 +10,16 @@ ImageOptimiser = require("./ImageOptimiser")
module.exports =
insertFile: (bucket, key, stream, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key)
PersistorManager.deleteDirectory bucket, convetedKey, (error) ->
convertedKey = KeyBuilder.getConvertedFolderKey key
PersistorManager.deleteDirectory bucket, convertedKey, (error) ->
return callback(error) if error?
PersistorManager.sendStream bucket, key, stream, callback
deleteFile: (bucket, key, callback)->
convetedKey = KeyBuilder.getConvertedFolderKey(key)
convertedKey = KeyBuilder.getConvertedFolderKey key
async.parallel [
(done)-> PersistorManager.deleteFile bucket, key, done
(done)-> PersistorManager.deleteDirectory bucket, convetedKey, done
(done)-> PersistorManager.deleteDirectory bucket, convertedKey, done
], callback
getFile: (bucket, key, opts = {}, callback)->
@ -36,49 +36,48 @@ module.exports =
callback err, fileStream
_getConvertedFile: (bucket, key, opts, callback)->
convetedKey = KeyBuilder.addCachingToKey(key, opts)
PersistorManager.checkIfFileExists bucket, convetedKey, (err, exists)=>
convertedKey = KeyBuilder.addCachingToKey key, opts
PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=>
if err?
return callback(err)
return callback err
if exists
PersistorManager.getFileStream bucket, convetedKey, opts, callback
PersistorManager.getFileStream bucket, convertedKey, opts, callback
else
@_getConvertedFileAndCache bucket, key, convetedKey, opts, callback
@_getConvertedFileAndCache bucket, key, convertedKey, opts, callback
_getConvertedFileAndCache: (bucket, key, convetedKey, opts, callback)->
self = @
_getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)->
convertedFsPath = ""
async.series [
(cb)->
self._convertFile bucket, key, opts, (err, fileSystemPath)->
(cb) =>
@_convertFile bucket, key, opts, (err, fileSystemPath) ->
convertedFsPath = fileSystemPath
cb err
(cb)->
ImageOptimiser.compressPng convertedFsPath, cb
(cb)->
PersistorManager.sendFile bucket, convetedKey, convertedFsPath, cb
PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb
], (err)->
if err?
return callback(err)
PersistorManager.getFileStream bucket, convetedKey, opts, callback
PersistorManager.getFileStream bucket, convertedKey, opts, callback
_convertFile: (bucket, origonalKey, opts, callback)->
@_writeS3FileToDisk bucket, origonalKey, opts, (err, origonalFsPath)->
_convertFile: (bucket, originalKey, opts, callback)->
@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->
if err?
return callback(err)
done = (err, destPath)->
if err?
logger.err err:err, bucket:bucket, origonalKey:origonalKey, opts:opts, "error converting file"
logger.err err:err, bucket:bucket, originalKey:originalKey, opts:opts, "error converting file"
return callback(err)
LocalFileWriter.deleteFile origonalFsPath, ->
LocalFileWriter.deleteFile originalFsPath, ->
callback(err, destPath)
if opts.format?
FileConverter.convert origonalFsPath, opts.format, done
FileConverter.convert originalFsPath, opts.format, done
else if opts.style == "thumbnail"
FileConverter.thumbnail origonalFsPath, done
FileConverter.thumbnail originalFsPath, done
else if opts.style == "preview"
FileConverter.preview origonalFsPath, done
FileConverter.preview originalFsPath, done
else
return callback(new Error("should have specified opts to convert file with #{JSON.stringify(opts)}"))

View file

@ -7,6 +7,8 @@ settings.filestore.backend ||= "s3"
logger.log backend:settings.filestore.backend, "Loading backend"
module.exports = switch settings.filestore.backend
when "aws-sdk"
require "./AWSSDKPersistorManager"
when "s3"
require("./S3PersistorManager")
when "fs"

View file

@ -8,17 +8,12 @@
},
"dependencies": {
"async": "~0.2.10",
"bunyan": "^1.3.5",
"aws-sdk": "^2.1.39",
"coffee-script": "~1.7.1",
"express": "~3.4.8",
"grunt-bunyan": "^0.5.0",
"grunt-execute": "^0.2.2",
"grunt-mocha-test": "~0.8.2",
"heapdump": "^0.3.2",
"knox": "~0.9.1",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0",
"longjohn": "~0.2.2",
"lynx": "0.0.11",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0",
"node-transloadit": "0.0.4",
"node-uuid": "~1.4.1",
@ -36,7 +31,11 @@
"sinon": "",
"chai": "",
"sandboxed-module": "",
"bunyan": "^1.3.5",
"grunt": "0.4.1",
"grunt-bunyan": "^0.5.0",
"grunt-execute": "^0.2.2",
"grunt-mocha-test": "~0.8.2",
"grunt-contrib-requirejs": "0.4.1",
"grunt-contrib-coffee": "0.7.0",
"grunt-contrib-watch": "0.5.3",

View file

@ -0,0 +1,249 @@
sinon = require 'sinon'
chai = require 'chai'
should = chai.should()
expect = chai.expect
modulePath = "../../../app/js/AWSSDKPersistorManager.js"
SandboxedModule = require 'sandboxed-module'
describe "AWSSDKPersistorManager", ->
beforeEach ->
@settings =
filestore:
backend: "aws-sdk"
@s3 =
upload: sinon.stub()
getObject: sinon.stub()
copyObject: sinon.stub()
deleteObject: sinon.stub()
listObjects: sinon.stub()
deleteObjects: sinon.stub()
headObject: sinon.stub()
@awssdk =
S3: sinon.stub().returns @s3
@requires =
"aws-sdk": @awssdk
"settings-sharelatex": @settings
"logger-sharelatex":
log:->
err:->
"fs": @fs =
createReadStream: sinon.stub()
"./Errors": @Errors =
NotFoundError: sinon.stub()
@key = "my/key"
@bucketName = "my-bucket"
@error = "my error"
@AWSSDKPersistorManager = SandboxedModule.require modulePath, requires: @requires
describe "sendFile", ->
beforeEach ->
@stream = {}
@fsPath = "/usr/local/some/file"
@fs.createReadStream.returns @stream
it "should put the file with s3.upload", (done) ->
@s3.upload.callsArgWith 1
@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
expect(err).to.not.be.ok
expect(@s3.upload.calledOnce, "called only once").to.be.true
expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream)
, "called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.upload", (done) ->
@s3.upload.callsArgWith 1, @error
@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
expect(err).to.equal @error
done()
describe "sendStream", ->
beforeEach ->
@stream = {}
it "should put the file with s3.upload", (done) ->
@s3.upload.callsArgWith 1
@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
expect(err).to.not.be.ok
expect(@s3.upload.calledOnce, "called only once").to.be.true
expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream),
"called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.upload", (done) ->
@s3.upload.callsArgWith 1, @error
@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
expect(err).to.equal @error
done()
describe "getFileStream", ->
beforeEach ->
@opts = {}
@stream = {}
@read_stream =
on: @read_stream_on = sinon.stub()
@object =
createReadStream: sinon.stub().returns @read_stream
@s3.getObject.returns @object
it "should return a stream from s3.getObject", (done) ->
@read_stream_on.withArgs('readable').callsArgWith 1
@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
expect(@read_stream_on.calledTwice)
expect(err).to.not.be.ok
expect(stream, "returned the stream").to.equal @read_stream
expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key),
"called with correct arguments").to.be.true
done()
describe "with start and end options", ->
beforeEach ->
@opts =
start: 0
end: 8
it "should pass headers to the s3.GetObject", (done) ->
@read_stream_on.withArgs('readable').callsArgWith 1
@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key, Range: 'bytes=0-8'),
"called with correct arguments").to.be.true
done()
describe "error conditions", ->
describe "when the file doesn't exist", ->
beforeEach ->
@error = new Error()
@error.code = 'NoSuchKey'
it "should produce a NotFoundError", (done) ->
@read_stream_on.withArgs('error').callsArgWith 1, @error
@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
expect(stream).to.not.be.ok
expect(err).to.be.ok
expect(err instanceof @Errors.NotFoundError, "error is a correct instance").to.equal true
done()
describe "when there is some other error", ->
beforeEach ->
@error = new Error()
it "should dispatch the error from s3 object stream", (done) ->
@read_stream_on.withArgs('error').callsArgWith 1, @error
@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
expect(stream).to.not.be.ok
expect(err).to.be.ok
expect(err).to.equal @error
done()
describe "copyFile", ->
beforeEach ->
@destKey = "some/key"
@stream = {}
it "should copy the file with s3.copyObject", (done) ->
@s3.copyObject.callsArgWith 1
@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
expect(err).to.not.be.ok
expect(@s3.copyObject.calledOnce, "called only once").to.be.true
expect((@s3.copyObject.calledWith Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key),
"called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.copyObject", (done) ->
@s3.copyObject.callsArgWith 1, @error
@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
expect(err).to.equal @error
done()
describe "deleteFile", ->
it "should delete the file with s3.deleteObject", (done) ->
@s3.deleteObject.callsArgWith 1
@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
expect(err).to.not.be.ok
expect(@s3.deleteObject.calledOnce, "called only once").to.be.true
expect((@s3.deleteObject.calledWith Bucket: @bucketName, Key: @key),
"called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.deleteObject", (done) ->
@s3.deleteObject.callsArgWith 1, @error
@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
expect(err).to.equal @error
done()
describe "deleteDirectory", ->
it "should list the directory content using s3.listObjects", (done) ->
@s3.listObjects.callsArgWith 1, null, Contents: []
@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
expect(err).to.not.be.ok
expect(@s3.listObjects.calledOnce, "called only once").to.be.true
expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key),
"called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.listObjects", (done) ->
@s3.listObjects.callsArgWith 1, @error
@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
expect(err).to.equal @error
done()
describe "with directory content", ->
beforeEach ->
@fileList = [
Key: 'foo'
, Key: 'bar'
, Key: 'baz'
]
it "should forward the file keys to s3.deleteObjects", (done) ->
@s3.listObjects.callsArgWith 1, null, Contents: @fileList
@s3.deleteObjects.callsArgWith 1
@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
expect(err).to.not.be.ok
expect(@s3.deleteObjects.calledOnce, "called only once").to.be.true
expect((@s3.deleteObjects.calledWith
Bucket: @bucketName
Delete:
Quiet: true
Objects: @fileList),
"called with correct arguments").to.be.true
done()
it "should dispatch the error from s3.deleteObjects", (done) ->
@s3.listObjects.callsArgWith 1, null, Contents: @fileList
@s3.deleteObjects.callsArgWith 1, @error
@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
expect(err).to.equal @error
done()
describe "checkIfFileExists", ->
it "should check for the file with s3.headObject", (done) ->
@s3.headObject.callsArgWith 1, null, {}
@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
expect(err).to.not.be.ok
expect(@s3.headObject.calledOnce, "called only once").to.be.true
expect((@s3.headObject.calledWith Bucket: @bucketName, Key: @key),
"called with correct arguments").to.be.true
done()
it "should return false on an inexistant file", (done) ->
@s3.headObject.callsArgWith 1, null, {}
@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
expect(exists).to.be.false
done()
it "should return true on an existing file", (done) ->
@s3.headObject.callsArgWith 1, null, ETag: "etag"
@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
expect(exists).to.be.true
done()
it "should dispatch the error from s3.headObject", (done) ->
@s3.headObject.callsArgWith 1, @error
@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
expect(err).to.equal @error
done()