diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index 808b0b5219..311997ed03 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -7,7 +7,7 @@ s3 = aws.S3() module.exports = sendFile: (bucketName, key, fsPath, callback)-> - logger.log bucketName:bucketName, key, "send file data to s3" + logger.log bucketName:bucketName, key:key, "send file data to s3" stream = fs.createReadStream fsPath s3.putObject Bucket: bucketName, Key: key, Body: stream, (err, data) -> if err? @@ -15,18 +15,19 @@ module.exports = callback err sendStream: (bucketName, key, stream, callback)-> - logger.log bucketName:bucketName, key, "send file stream to s3" + logger.log bucketName:bucketName, key:key, "send file stream to s3" s3.putObject Bucket: bucketName, Key: key, Body: stream, (err, data) -> if err? logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3" callback err getFileStream: (bucketName, key, callback = (err, res)->)-> - logger.log bucketName:bucketName, key, "get file stream from s3" + logger.log bucketName:bucketName, key:key, "get file stream from s3" callback = _.once callback - stream = s3.getObject(Bucket:bucketName, Key: key).createReadStream() - stream.on 'response', (res) -> - callback null, res + request = s3.getObject(Bucket:bucketName, Key: key) + stream = request.createReadStream() + stream.on 'readable', () -> + callback null, stream stream.on 'error', (err) -> logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3" callback err @@ -48,20 +49,23 @@ module.exports = deleteDirectory: (bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "delete directory in s3" - s3.listObjects {Bucket: bucketName, prefix: key}, (err, data) -> + s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> if err? logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" return callback err + if data.Contents.length == 0 + logger.log bucketName:bucketName, key:key, "the directory is empty" + return callback() keys = _.map data.Contents, (entry)-> Key: entry.Key - s3.deleteMultiple + s3.deleteObjects Bucket: bucketName Delete: Objects: keys Quiet: true , (err) -> if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting directory in s3" + logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3" callback err checkIfFileExists:(bucketName, key, callback)-> diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee index 3b83e203fd..921faa6c6e 100644 --- a/services/filestore/app/coffee/FileController.coffee +++ b/services/filestore/app/coffee/FileController.coffee @@ -11,7 +11,7 @@ module.exports = metrics.inc "getFile" {key, bucket} = req {format, style} = req.query - logger.log key:key, bucket:bucket, format:format, style:style, "reciving request to get file" + logger.log key:key, bucket:bucket, format:format, style:style, "receiving request to get file" FileHandler.getFile bucket, key, {format:format,style:style}, (err, fileStream)-> if err? logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file" diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index 51aec4bba6..ece26c6164 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -10,16 +10,16 @@ ImageOptimiser = require("./ImageOptimiser") module.exports = insertFile: (bucket, key, stream, callback)-> - convetedKey = KeyBuilder.getConvertedFolderKey(key) - PersistorManager.deleteDirectory bucket, convetedKey, (error) -> + convertedKey = KeyBuilder.getConvertedFolderKey key + PersistorManager.deleteDirectory bucket, convertedKey, (error) -> return callback(error) if error? PersistorManager.sendStream bucket, key, stream, callback deleteFile: (bucket, key, callback)-> - convetedKey = KeyBuilder.getConvertedFolderKey(key) + convertedKey = KeyBuilder.getConvertedFolderKey key async.parallel [ (done)-> PersistorManager.deleteFile bucket, key, done - (done)-> PersistorManager.deleteDirectory bucket, convetedKey, done + (done)-> PersistorManager.deleteDirectory bucket, convertedKey, done ], callback getFile: (bucket, key, opts = {}, callback)-> @@ -36,14 +36,14 @@ module.exports = callback err, fileStream _getConvertedFile: (bucket, key, opts, callback)-> - convetedKey = KeyBuilder.addCachingToKey(key, opts) - PersistorManager.checkIfFileExists bucket, convetedKey, (err, exists)=> + convertedKey = KeyBuilder.addCachingToKey key, opts + PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=> if exists - PersistorManager.getFileStream bucket, convetedKey, callback + PersistorManager.getFileStream bucket, convertedKey, callback else - @_getConvertedFileAndCache bucket, key, convetedKey, opts, callback + @_getConvertedFileAndCache bucket, key, convertedKey, opts, callback - _getConvertedFileAndCache: (bucket, key, convetedKey, opts, callback)-> + _getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)-> self = @ convertedFsPath = "" async.series [ @@ -54,27 +54,27 @@ module.exports = (cb)-> ImageOptimiser.compressPng convertedFsPath, cb (cb)-> - PersistorManager.sendFile bucket, convetedKey, convertedFsPath, cb + PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb ], (err)-> if err? return callback(err) - PersistorManager.getFileStream bucket, convetedKey, callback + PersistorManager.getFileStream bucket, convertedKey, callback - _convertFile: (bucket, origonalKey, opts, callback)-> - @_writeS3FileToDisk bucket, origonalKey, (err, origonalFsPath)-> + _convertFile: (bucket, originalKey, opts, callback)-> + @_writeS3FileToDisk bucket, originalKey, (err, originalFsPath)-> done = (err, destPath)-> if err? - logger.err err:err, bucket:bucket, origonalKey:origonalKey, opts:opts, "error converting file" + logger.err err:err, bucket:bucket, originalKey:originalKey, opts:opts, "error converting file" return callback(err) - LocalFileWriter.deleteFile origonalFsPath, -> + LocalFileWriter.deleteFile originalFsPath, -> callback(err, destPath) if opts.format? - FileConverter.convert origonalFsPath, opts.format, done + FileConverter.convert originalFsPath, opts.format, done else if opts.style == "thumbnail" - FileConverter.thumbnail origonalFsPath, done + FileConverter.thumbnail originalFsPath, done else if opts.style == "preview" - FileConverter.preview origonalFsPath, done + FileConverter.preview originalFsPath, done else throw new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")