Make S3 partSize a setting

This commit is contained in:
Simon Detheridge 2020-02-17 13:50:21 +00:00
parent 7c4bf97a7f
commit 8acc9ef0ae
4 changed files with 10 additions and 5 deletions

View file

@ -96,7 +96,7 @@ async function sendStream(bucketName, key, readStream, sourceMd5) {
}
const response = await _getClientForBucket(bucketName)
.upload(uploadOptions, { partSize: 100 * 1024 * 1024 })
.upload(uploadOptions, { partSize: settings.filestore.s3.partSize })
.promise()
let destMd5 = _md5FromResponse(response)
if (!destMd5) {

View file

@ -38,6 +38,7 @@ settings =
key: process.env['AWS_ACCESS_KEY_ID']
secret: process.env['AWS_SECRET_ACCESS_KEY']
endpoint: process.env['AWS_S3_ENDPOINT']
partSize: process.env['S3_PARTSIZE'] or (100 * 1024 * 1024)
stores:
user_files: process.env['USER_FILES_BUCKET_NAME']

View file

@ -57,7 +57,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true
pathStyle: true,
partSize: 100 * 1024 * 1024
},
stores: {
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
@ -71,7 +72,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true
pathStyle: true,
partSize: 100 * 1024 * 1024
},
stores: {
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
@ -102,7 +104,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true
pathStyle: true,
partSize: 100 * 1024 * 1024
},
stores: {
user_files: Path.resolve(__dirname, '../../../user_files'),

View file

@ -53,7 +53,8 @@ describe('S3PersistorTests', function() {
backend: 's3',
s3: {
secret: defaultS3Secret,
key: defaultS3Key
key: defaultS3Key,
partSize: 100 * 1024 * 1024
},
stores: {
user_files: 'sl_user_files'