github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Make S3 partSize a setting

Browse source
This commit is contained in:
Simon Detheridge 2020-02-17 13:50:21 +00:00
parent 7c4bf97a7f
commit 8acc9ef0ae
4 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/filestore/app/js/S3Persistor.js
View file

@ -96,7 +96,7 @@ async function sendStream(bucketName, key, readStream, sourceMd5) {
} }
const response = await _getClientForBucket(bucketName) const response = await _getClientForBucket(bucketName)
.upload(uploadOptions, { partSize: 100 * 1024 * 1024 }) .upload(uploadOptions, { partSize: settings.filestore.s3.partSize })
.promise() .promise()
let destMd5 = _md5FromResponse(response) let destMd5 = _md5FromResponse(response)
if (!destMd5) { if (!destMd5) {

1
services/filestore/config/settings.defaults.coffee
View file

@ -38,6 +38,7 @@ settings =
key: process.env['AWS_ACCESS_KEY_ID'] key: process.env['AWS_ACCESS_KEY_ID']
secret: process.env['AWS_SECRET_ACCESS_KEY'] secret: process.env['AWS_SECRET_ACCESS_KEY']
endpoint: process.env['AWS_S3_ENDPOINT'] endpoint: process.env['AWS_S3_ENDPOINT']
partSize: process.env['S3_PARTSIZE'] or (100 * 1024 * 1024)
stores: stores:
user_files: process.env['USER_FILES_BUCKET_NAME'] user_files: process.env['USER_FILES_BUCKET_NAME']

9
services/filestore/test/acceptance/js/FilestoreTests.js
View file

@ -57,7 +57,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID, key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY, secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT, endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true pathStyle: true,
partSize: 100 * 1024 * 1024
}, },
stores: { stores: {
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME, user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
@ -71,7 +72,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID, key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY, secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT, endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true pathStyle: true,
partSize: 100 * 1024 * 1024
}, },
stores: { stores: {
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME, user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
@ -102,7 +104,8 @@ const BackendSettings = {
key: process.env.AWS_ACCESS_KEY_ID, key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY, secret: process.env.AWS_SECRET_ACCESS_KEY,
endpoint: process.env.AWS_S3_ENDPOINT, endpoint: process.env.AWS_S3_ENDPOINT,
pathStyle: true pathStyle: true,
partSize: 100 * 1024 * 1024
}, },
stores: { stores: {
user_files: Path.resolve(__dirname, '../../../user_files'), user_files: Path.resolve(__dirname, '../../../user_files'),

3
services/filestore/test/unit/js/S3PersistorTests.js
View file

@ -53,7 +53,8 @@ describe('S3PersistorTests', function() {
backend: 's3', backend: 's3',
s3: { s3: {
secret: defaultS3Secret, secret: defaultS3Secret,
key: defaultS3Key key: defaultS3Key,
partSize: 100 * 1024 * 1024
}, },
stores: { stores: {
user_files: 'sl_user_files' user_files: 'sl_user_files'