From 473aea4e60437096dffe77750413563a0fd030fa Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 18:21:21 +0000 Subject: [PATCH] Remove AWSSDKPersistorManager --- .../app/js/AWSSDKPersistorManager.js | 197 ------- .../unit/js/AWSSDKPersistorManagerTests.js | 509 ------------------ 2 files changed, 706 deletions(-) delete mode 100644 services/filestore/app/js/AWSSDKPersistorManager.js delete mode 100644 services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js diff --git a/services/filestore/app/js/AWSSDKPersistorManager.js b/services/filestore/app/js/AWSSDKPersistorManager.js deleted file mode 100644 index 4dbc836280..0000000000 --- a/services/filestore/app/js/AWSSDKPersistorManager.js +++ /dev/null @@ -1,197 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This module is not used in production, which currently uses -// S3PersistorManager. The intention is to migrate S3PersistorManager to use the -// latest aws-sdk and delete this module so that PersistorManager would load the -// same backend for both the 's3' and 'aws-sdk' options. - -const logger = require('logger-sharelatex') -const aws = require('aws-sdk') -const _ = require('underscore') -const fs = require('fs') -const Errors = require('./Errors') - -const s3 = new aws.S3() - -module.exports = { - sendFile(bucketName, key, fsPath, callback) { - logger.log({ bucketName, key }, 'send file data to s3') - const stream = fs.createReadStream(fsPath) - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file data to s3' - ) - } - return callback(err) - }) - }, - - sendStream(bucketName, key, stream, callback) { - logger.log({ bucketName, key }, 'send file stream to s3') - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file stream to s3' - ) - } - return callback(err) - }) - }, - - getFileStream(bucketName, key, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - logger.log({ bucketName, key }, 'get file stream from s3') - callback = _.once(callback) - const params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - params.Range = `bytes=${opts.start}-${opts.end}` - } - const request = s3.getObject(params) - const stream = request.createReadStream() - stream.on('readable', () => callback(null, stream)) - return stream.on('error', function(err) { - logger.err({ err, bucketName, key }, 'error getting file stream from s3') - if (err.code === 'NoSuchKey') { - return callback( - new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) - ) - } - return callback(err) - }) - }, - - copyFile(bucketName, sourceKey, destKey, callback) { - logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') - const source = bucketName + '/' + sourceKey - return s3.copyObject( - { Bucket: bucketName, Key: destKey, CopySource: source }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, sourceKey, destKey }, - 'something went wrong copying file in s3' - ) - } - return callback(err) - } - ) - }, - - deleteFile(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete file in s3') - return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong deleting file in s3' - ) - } - return callback(err) - }) - }, - - deleteDirectory(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete directory in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - const keys = _.map(data.Contents, entry => ({ - Key: entry.Key - })) - return s3.deleteObjects( - { - Bucket: bucketName, - Delete: { - Objects: keys, - Quiet: true - } - }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, key: keys }, - 'something went wrong deleting directory in s3' - ) - } - return callback(err) - } - ) - }) - }, - - checkIfFileExists(bucketName, key, callback) { - logger.log({ bucketName, key }, 'check file existence in s3') - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if (err.code === 'NotFound') { - return callback(null, false) - } - logger.err( - { err, bucketName, key }, - 'something went wrong checking head in s3' - ) - return callback(err) - } - return callback(null, data.ETag != null) - }) - }, - - directorySize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'get project size in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - let totalSize = 0 - _.each(data.Contents, entry => (totalSize += entry.Size)) - return callback(null, totalSize) - }) - } -} diff --git a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js deleted file mode 100644 index ea88da71c3..0000000000 --- a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js +++ /dev/null @@ -1,509 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-dupe-keys, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const sinon = require('sinon') -const chai = require('chai') - -const should = chai.should() -const { expect } = chai - -const modulePath = '../../../app/js/AWSSDKPersistorManager.js' -const SandboxedModule = require('sandboxed-module') - -describe('AWSSDKPersistorManager', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 'aws-sdk' - } - } - this.s3 = { - upload: sinon.stub(), - getObject: sinon.stub(), - copyObject: sinon.stub(), - deleteObject: sinon.stub(), - listObjects: sinon.stub(), - deleteObjects: sinon.stub(), - headObject: sinon.stub() - } - this.awssdk = { S3: sinon.stub().returns(this.s3) } - - this.requires = { - 'aws-sdk': this.awssdk, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - }, - fs: (this.fs = { createReadStream: sinon.stub() }), - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) - } - this.key = 'my/key' - this.bucketName = 'my-bucket' - this.error = 'my error' - return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) - }) - - describe('sendFile', function() { - beforeEach(function() { - this.stream = {} - this.fsPath = '/usr/local/some/file' - return this.fs.createReadStream.returns(this.stream) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('sendStream', function() { - beforeEach(function() { - return (this.stream = {}) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('getFileStream', function() { - beforeEach(function() { - this.opts = {} - this.stream = {} - this.read_stream = { on: (this.read_stream_on = sinon.stub()) } - this.object = { createReadStream: sinon.stub().returns(this.read_stream) } - return this.s3.getObject.returns(this.object) - }) - - it('should return a stream from s3.getObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(this.read_stream_on.calledTwice) - expect(err).to.not.be.ok - expect(stream, 'returned the stream').to.equal(this.read_stream) - expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - describe('with start and end options', function() { - beforeEach(function() { - return (this.opts = { - start: 0, - end: 8 - }) - }) - return it('should pass headers to the s3.GetObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - return expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Range: 'bytes=0-8' - }), - 'called with correct arguments' - ).to.be.true - } - ) - return done() - }) - }) - - return describe('error conditions', function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - this.error = new Error() - return (this.error.code = 'NoSuchKey') - }) - return it('should produce a NotFoundError', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect( - err instanceof this.Errors.NotFoundError, - 'error is a correct instance' - ).to.equal(true) - return done() - } - ) - }) - }) - - return describe('when there is some other error', function() { - beforeEach(function() { - return (this.error = new Error()) - }) - return it('should dispatch the error from s3 object stream', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - }) - - describe('copyFile', function() { - beforeEach(function() { - this.destKey = 'some/key' - return (this.stream = {}) - }) - - it('should copy the file with s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.not.be.ok - expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.copyObject.calledWith({ - Bucket: this.bucketName, - Key: this.destKey, - CopySource: this.bucketName + '/' + this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteFile', function() { - it('should delete the file with s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.deleteObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteDirectory', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return describe('with directory content', function() { - beforeEach(function() { - return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }]) - }) - - it('should forward the file keys to s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be - .true - expect( - this.s3.deleteObjects.calledWith({ - Bucket: this.bucketName, - Delete: { - Quiet: true, - Objects: this.fileList - } - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - - describe('checkIfFileExists', function() { - it('should check for the file with s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.not.be.ok - expect(this.s3.headObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.headObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should return false on an inexistant file', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.false - return done() - } - ) - }) - - it('should return true on an existing file', function(done) { - this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' }) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - return describe('directorySize', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return it('should sum directory files sizes', function(done) { - this.s3.listObjects.callsArgWith(1, null, { - Contents: [{ Size: 1024 }, { Size: 2048 }] - }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - (err, size) => { - expect(size).to.equal(3072) - return done() - } - ) - }) - }) -})