Remove AWSSDKPersistorManager

This commit is contained in:
Simon Detheridge 2020-01-03 18:21:21 +00:00
parent e27cf4db7b
commit 473aea4e60
2 changed files with 0 additions and 706 deletions

View file

@ -1,197 +0,0 @@
/* eslint-disable
handle-callback-err,
no-return-assign,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
// This module is not used in production, which currently uses
// S3PersistorManager. The intention is to migrate S3PersistorManager to use the
// latest aws-sdk and delete this module so that PersistorManager would load the
// same backend for both the 's3' and 'aws-sdk' options.
const logger = require('logger-sharelatex')
const aws = require('aws-sdk')
const _ = require('underscore')
const fs = require('fs')
const Errors = require('./Errors')
const s3 = new aws.S3()
module.exports = {
sendFile(bucketName, key, fsPath, callback) {
logger.log({ bucketName, key }, 'send file data to s3')
const stream = fs.createReadStream(fsPath)
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
err,
data
) {
if (err != null) {
logger.err(
{ err, Bucket: bucketName, Key: key },
'error sending file data to s3'
)
}
return callback(err)
})
},
sendStream(bucketName, key, stream, callback) {
logger.log({ bucketName, key }, 'send file stream to s3')
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
err,
data
) {
if (err != null) {
logger.err(
{ err, Bucket: bucketName, Key: key },
'error sending file stream to s3'
)
}
return callback(err)
})
},
getFileStream(bucketName, key, opts, callback) {
if (callback == null) {
callback = function(err, res) {}
}
logger.log({ bucketName, key }, 'get file stream from s3')
callback = _.once(callback)
const params = {
Bucket: bucketName,
Key: key
}
if (opts.start != null && opts.end != null) {
params.Range = `bytes=${opts.start}-${opts.end}`
}
const request = s3.getObject(params)
const stream = request.createReadStream()
stream.on('readable', () => callback(null, stream))
return stream.on('error', function(err) {
logger.err({ err, bucketName, key }, 'error getting file stream from s3')
if (err.code === 'NoSuchKey') {
return callback(
new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
)
}
return callback(err)
})
},
copyFile(bucketName, sourceKey, destKey, callback) {
logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
const source = bucketName + '/' + sourceKey
return s3.copyObject(
{ Bucket: bucketName, Key: destKey, CopySource: source },
function(err) {
if (err != null) {
logger.err(
{ err, bucketName, sourceKey, destKey },
'something went wrong copying file in s3'
)
}
return callback(err)
}
)
},
deleteFile(bucketName, key, callback) {
logger.log({ bucketName, key }, 'delete file in s3')
return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) {
if (err != null) {
logger.err(
{ err, bucketName, key },
'something went wrong deleting file in s3'
)
}
return callback(err)
})
},
deleteDirectory(bucketName, key, callback) {
logger.log({ bucketName, key }, 'delete directory in s3')
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
err,
data
) {
if (err != null) {
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in s3'
)
return callback(err)
}
if (data.Contents.length === 0) {
logger.log({ bucketName, key }, 'the directory is empty')
return callback()
}
const keys = _.map(data.Contents, entry => ({
Key: entry.Key
}))
return s3.deleteObjects(
{
Bucket: bucketName,
Delete: {
Objects: keys,
Quiet: true
}
},
function(err) {
if (err != null) {
logger.err(
{ err, bucketName, key: keys },
'something went wrong deleting directory in s3'
)
}
return callback(err)
}
)
})
},
checkIfFileExists(bucketName, key, callback) {
logger.log({ bucketName, key }, 'check file existence in s3')
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
if (err != null) {
if (err.code === 'NotFound') {
return callback(null, false)
}
logger.err(
{ err, bucketName, key },
'something went wrong checking head in s3'
)
return callback(err)
}
return callback(null, data.ETag != null)
})
},
directorySize(bucketName, key, callback) {
logger.log({ bucketName, key }, 'get project size in s3')
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
err,
data
) {
if (err != null) {
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in s3'
)
return callback(err)
}
if (data.Contents.length === 0) {
logger.log({ bucketName, key }, 'the directory is empty')
return callback()
}
let totalSize = 0
_.each(data.Contents, entry => (totalSize += entry.Size))
return callback(null, totalSize)
})
}
}

View file

@ -1,509 +0,0 @@
/* eslint-disable
handle-callback-err,
no-dupe-keys,
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const chai = require('chai')
const should = chai.should()
const { expect } = chai
const modulePath = '../../../app/js/AWSSDKPersistorManager.js'
const SandboxedModule = require('sandboxed-module')
describe('AWSSDKPersistorManager', function() {
beforeEach(function() {
this.settings = {
filestore: {
backend: 'aws-sdk'
}
}
this.s3 = {
upload: sinon.stub(),
getObject: sinon.stub(),
copyObject: sinon.stub(),
deleteObject: sinon.stub(),
listObjects: sinon.stub(),
deleteObjects: sinon.stub(),
headObject: sinon.stub()
}
this.awssdk = { S3: sinon.stub().returns(this.s3) }
this.requires = {
'aws-sdk': this.awssdk,
'settings-sharelatex': this.settings,
'logger-sharelatex': {
log() {},
err() {}
},
fs: (this.fs = { createReadStream: sinon.stub() }),
'./Errors': (this.Errors = { NotFoundError: sinon.stub() })
}
this.key = 'my/key'
this.bucketName = 'my-bucket'
this.error = 'my error'
return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, {
requires: this.requires
}))
})
describe('sendFile', function() {
beforeEach(function() {
this.stream = {}
this.fsPath = '/usr/local/some/file'
return this.fs.createReadStream.returns(this.stream)
})
it('should put the file with s3.upload', function(done) {
this.s3.upload.callsArgWith(1)
return this.AWSSDKPersistorManager.sendFile(
this.bucketName,
this.key,
this.fsPath,
err => {
expect(err).to.not.be.ok
expect(this.s3.upload.calledOnce, 'called only once').to.be.true
expect(
this.s3.upload.calledWith({
Bucket: this.bucketName,
Key: this.key,
Body: this.stream
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.upload', function(done) {
this.s3.upload.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.sendFile(
this.bucketName,
this.key,
this.fsPath,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
describe('sendStream', function() {
beforeEach(function() {
return (this.stream = {})
})
it('should put the file with s3.upload', function(done) {
this.s3.upload.callsArgWith(1)
return this.AWSSDKPersistorManager.sendStream(
this.bucketName,
this.key,
this.stream,
err => {
expect(err).to.not.be.ok
expect(this.s3.upload.calledOnce, 'called only once').to.be.true
expect(
this.s3.upload.calledWith({
Bucket: this.bucketName,
Key: this.key,
Body: this.stream
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.upload', function(done) {
this.s3.upload.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.sendStream(
this.bucketName,
this.key,
this.stream,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
describe('getFileStream', function() {
beforeEach(function() {
this.opts = {}
this.stream = {}
this.read_stream = { on: (this.read_stream_on = sinon.stub()) }
this.object = { createReadStream: sinon.stub().returns(this.read_stream) }
return this.s3.getObject.returns(this.object)
})
it('should return a stream from s3.getObject', function(done) {
this.read_stream_on.withArgs('readable').callsArgWith(1)
return this.AWSSDKPersistorManager.getFileStream(
this.bucketName,
this.key,
this.opts,
(err, stream) => {
expect(this.read_stream_on.calledTwice)
expect(err).to.not.be.ok
expect(stream, 'returned the stream').to.equal(this.read_stream)
expect(
this.s3.getObject.calledWith({
Bucket: this.bucketName,
Key: this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
describe('with start and end options', function() {
beforeEach(function() {
return (this.opts = {
start: 0,
end: 8
})
})
return it('should pass headers to the s3.GetObject', function(done) {
this.read_stream_on.withArgs('readable').callsArgWith(1)
this.AWSSDKPersistorManager.getFileStream(
this.bucketName,
this.key,
this.opts,
(err, stream) => {
return expect(
this.s3.getObject.calledWith({
Bucket: this.bucketName,
Key: this.key,
Range: 'bytes=0-8'
}),
'called with correct arguments'
).to.be.true
}
)
return done()
})
})
return describe('error conditions', function() {
describe("when the file doesn't exist", function() {
beforeEach(function() {
this.error = new Error()
return (this.error.code = 'NoSuchKey')
})
return it('should produce a NotFoundError', function(done) {
this.read_stream_on.withArgs('error').callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.getFileStream(
this.bucketName,
this.key,
this.opts,
(err, stream) => {
expect(stream).to.not.be.ok
expect(err).to.be.ok
expect(
err instanceof this.Errors.NotFoundError,
'error is a correct instance'
).to.equal(true)
return done()
}
)
})
})
return describe('when there is some other error', function() {
beforeEach(function() {
return (this.error = new Error())
})
return it('should dispatch the error from s3 object stream', function(done) {
this.read_stream_on.withArgs('error').callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.getFileStream(
this.bucketName,
this.key,
this.opts,
(err, stream) => {
expect(stream).to.not.be.ok
expect(err).to.be.ok
expect(err).to.equal(this.error)
return done()
}
)
})
})
})
})
describe('copyFile', function() {
beforeEach(function() {
this.destKey = 'some/key'
return (this.stream = {})
})
it('should copy the file with s3.copyObject', function(done) {
this.s3.copyObject.callsArgWith(1)
return this.AWSSDKPersistorManager.copyFile(
this.bucketName,
this.key,
this.destKey,
err => {
expect(err).to.not.be.ok
expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true
expect(
this.s3.copyObject.calledWith({
Bucket: this.bucketName,
Key: this.destKey,
CopySource: this.bucketName + '/' + this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.copyObject', function(done) {
this.s3.copyObject.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.copyFile(
this.bucketName,
this.key,
this.destKey,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
describe('deleteFile', function() {
it('should delete the file with s3.deleteObject', function(done) {
this.s3.deleteObject.callsArgWith(1)
return this.AWSSDKPersistorManager.deleteFile(
this.bucketName,
this.key,
err => {
expect(err).to.not.be.ok
expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true
expect(
this.s3.deleteObject.calledWith({
Bucket: this.bucketName,
Key: this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.deleteObject', function(done) {
this.s3.deleteObject.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.deleteFile(
this.bucketName,
this.key,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
describe('deleteDirectory', function() {
it('should list the directory content using s3.listObjects', function(done) {
this.s3.listObjects.callsArgWith(1, null, { Contents: [] })
return this.AWSSDKPersistorManager.deleteDirectory(
this.bucketName,
this.key,
err => {
expect(err).to.not.be.ok
expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true
expect(
this.s3.listObjects.calledWith({
Bucket: this.bucketName,
Prefix: this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
it('should dispatch the error from s3.listObjects', function(done) {
this.s3.listObjects.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.deleteDirectory(
this.bucketName,
this.key,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
return describe('with directory content', function() {
beforeEach(function() {
return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }])
})
it('should forward the file keys to s3.deleteObjects', function(done) {
this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList })
this.s3.deleteObjects.callsArgWith(1)
return this.AWSSDKPersistorManager.deleteDirectory(
this.bucketName,
this.key,
err => {
expect(err).to.not.be.ok
expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be
.true
expect(
this.s3.deleteObjects.calledWith({
Bucket: this.bucketName,
Delete: {
Quiet: true,
Objects: this.fileList
}
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.deleteObjects', function(done) {
this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList })
this.s3.deleteObjects.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.deleteDirectory(
this.bucketName,
this.key,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
})
describe('checkIfFileExists', function() {
it('should check for the file with s3.headObject', function(done) {
this.s3.headObject.callsArgWith(1, null, {})
return this.AWSSDKPersistorManager.checkIfFileExists(
this.bucketName,
this.key,
(err, exists) => {
expect(err).to.not.be.ok
expect(this.s3.headObject.calledOnce, 'called only once').to.be.true
expect(
this.s3.headObject.calledWith({
Bucket: this.bucketName,
Key: this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
it('should return false on an inexistant file', function(done) {
this.s3.headObject.callsArgWith(1, null, {})
return this.AWSSDKPersistorManager.checkIfFileExists(
this.bucketName,
this.key,
(err, exists) => {
expect(exists).to.be.false
return done()
}
)
})
it('should return true on an existing file', function(done) {
this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' })
return this.AWSSDKPersistorManager.checkIfFileExists(
this.bucketName,
this.key,
(err, exists) => {
expect(exists).to.be.true
return done()
}
)
})
return it('should dispatch the error from s3.headObject', function(done) {
this.s3.headObject.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.checkIfFileExists(
this.bucketName,
this.key,
(err, exists) => {
expect(err).to.equal(this.error)
return done()
}
)
})
})
return describe('directorySize', function() {
it('should list the directory content using s3.listObjects', function(done) {
this.s3.listObjects.callsArgWith(1, null, { Contents: [] })
return this.AWSSDKPersistorManager.directorySize(
this.bucketName,
this.key,
err => {
expect(err).to.not.be.ok
expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true
expect(
this.s3.listObjects.calledWith({
Bucket: this.bucketName,
Prefix: this.key
}),
'called with correct arguments'
).to.be.true
return done()
}
)
})
it('should dispatch the error from s3.listObjects', function(done) {
this.s3.listObjects.callsArgWith(1, this.error)
return this.AWSSDKPersistorManager.directorySize(
this.bucketName,
this.key,
err => {
expect(err).to.equal(this.error)
return done()
}
)
})
return it('should sum directory files sizes', function(done) {
this.s3.listObjects.callsArgWith(1, null, {
Contents: [{ Size: 1024 }, { Size: 2048 }]
})
return this.AWSSDKPersistorManager.directorySize(
this.bucketName,
this.key,
(err, size) => {
expect(size).to.equal(3072)
return done()
}
)
})
})
})