diff --git a/libraries/object-persistor/src/FSPersistor.js b/libraries/object-persistor/src/FSPersistor.js index 05b251b114..01aab72800 100644 --- a/libraries/object-persistor/src/FSPersistor.js +++ b/libraries/object-persistor/src/FSPersistor.js @@ -15,6 +15,12 @@ const glob = promisify(globCallbacks) module.exports = class FSPersistor extends AbstractPersistor { constructor(settings = {}) { + if (settings.storageClass) { + throw new NotImplementedError( + 'FS backend does not support storage classes' + ) + } + super() this.useSubdirectories = Boolean(settings.useSubdirectories) } diff --git a/libraries/object-persistor/src/GcsPersistor.js b/libraries/object-persistor/src/GcsPersistor.js index 542755ee6d..0485373ae5 100644 --- a/libraries/object-persistor/src/GcsPersistor.js +++ b/libraries/object-persistor/src/GcsPersistor.js @@ -2,7 +2,12 @@ const fs = require('node:fs') const { pipeline } = require('node:stream/promises') const { PassThrough } = require('node:stream') const { Storage, IdempotencyStrategy } = require('@google-cloud/storage') -const { WriteError, ReadError, NotFoundError } = require('./Errors') +const { + WriteError, + ReadError, + NotFoundError, + NotImplementedError, +} = require('./Errors') const asyncPool = require('tiny-async-pool') const AbstractPersistor = require('./AbstractPersistor') const PersistorHelper = require('./PersistorHelper') @@ -11,8 +16,13 @@ const zlib = require('node:zlib') module.exports = class GcsPersistor extends AbstractPersistor { constructor(settings) { - super() + if (settings.storageClass) { + throw new NotImplementedError( + 'Use default bucket class for GCS instead of settings.storageClass' + ) + } + super() this.settings = settings // endpoint settings will be null by default except for tests diff --git a/libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js b/libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js index 6c13a611d1..ea90578760 100644 --- a/libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js +++ b/libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js @@ -311,6 +311,16 @@ class PerProjectEncryptedS3Persistor extends S3Persistor { return await super.getObjectSize(bucketName, path, { ...opts, ssecOptions }) } + async getObjectStorageClass(bucketName, path, opts = {}) { + const ssecOptions = + opts.ssecOptions || + (await this.#getExistingDataEncryptionKeyOptions(bucketName, path)) + return await super.getObjectStorageClass(bucketName, path, { + ...opts, + ssecOptions, + }) + } + async directorySize(bucketName, path, continuationToken) { // Note: Listing a bucket does not require SSE-C credentials. return await super.directorySize(bucketName, path, continuationToken) diff --git a/libraries/object-persistor/src/S3Persistor.js b/libraries/object-persistor/src/S3Persistor.js index 65d4358dac..4f438c445c 100644 --- a/libraries/object-persistor/src/S3Persistor.js +++ b/libraries/object-persistor/src/S3Persistor.js @@ -64,6 +64,7 @@ class S3Persistor extends AbstractPersistor { constructor(settings = {}) { super() + settings.storageClass = settings.storageClass || {} this.settings = settings } @@ -100,13 +101,17 @@ class S3Persistor extends AbstractPersistor { // observer will catch errors, clean up and log a warning pipeline(readStream, observer, () => {}) - // if we have an md5 hash, pass this to S3 to verify the upload + /** @type {S3.PutObjectRequest} */ const uploadOptions = { Bucket: bucketName, Key: key, Body: observer, } + if (this.settings.storageClass[bucketName]) { + uploadOptions.StorageClass = this.settings.storageClass[bucketName] + } + if (opts.contentType) { uploadOptions.ContentType = opts.contentType } @@ -336,6 +341,18 @@ class S3Persistor extends AbstractPersistor { return response.ContentLength || 0 } + /** + * @param {string} bucketName + * @param {string} key + * @param {Object} opts + * @param {SSECOptions} [opts.ssecOptions] + * @return {Promise} + */ + async getObjectStorageClass(bucketName, key, opts = {}) { + const response = await this.#headObject(bucketName, key, opts) + return response.StorageClass + } + /** * @param {string} bucketName * @param {string} key diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index ff628d5d1f..b95f601976 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -2,7 +2,7 @@ filestore --data-dirs=uploads,user_files,template_files --dependencies=s3,gcs --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker ---env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files +--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files --env-pass-through= --esmock-loader=False --node-version=20.18.0 diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index 408ec32828..ae8a16377a 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -35,6 +35,7 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" + AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 0cbd391ff3..c38581218b 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -52,6 +52,7 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" + AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 615d423780..30c27fcea2 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -31,7 +31,12 @@ process.on('unhandledRejection', e => { // store settings for multiple backends, so that we can test each one. // fs will always be available - add others if they are configured -const { BackendSettings, s3Config, s3SSECConfig } = require('./TestConfig') +const { + BackendSettings, + s3Config, + s3SSECConfig, + AWS_S3_USER_FILES_STORAGE_CLASS, +} = require('./TestConfig') const { AlreadyWrittenError, NotFoundError, @@ -1257,6 +1262,7 @@ describe('Filestore', function () { }) }) + /** @type {import('aws-sdk/clients/s3')} */ let s3Client before('create s3 client', function () { s3Client = new S3Persistor(s3Config())._getClientForBucket('') @@ -1401,6 +1407,40 @@ describe('Filestore', function () { }) }) + if (backendSettings.backend === 's3SSEC') { + describe('storageClass', function () { + it('should use the default storage class for dek', async function () { + const key = `${projectId}/${new ObjectId()}` + const dekBucket = process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME + await app.persistor.sendStream( + dekBucket, + key, + Stream.Readable.from(['hello']) + ) + expect( + await app.persistor.getObjectStorageClass(dekBucket, key) + ).to.equal(undefined) + }) + + it('should use the custom storage class for user files', async function () { + const key = `${projectId}/${new ObjectId()}` + await app.persistor.sendStream( + Settings.filestore.stores.user_files, + key, + Stream.Readable.from(['hello']) + ) + const sc = AWS_S3_USER_FILES_STORAGE_CLASS + expect(sc).to.exist + expect( + await app.persistor.getObjectStorageClass( + Settings.filestore.stores.user_files, + key + ) + ).to.equal(sc) + }) + }) + } + describe('autoGunzip', function () { let key beforeEach('new key', function () { diff --git a/services/filestore/test/acceptance/js/TestConfig.js b/services/filestore/test/acceptance/js/TestConfig.js index cd0f601c8d..7bd75ba781 100644 --- a/services/filestore/test/acceptance/js/TestConfig.js +++ b/services/filestore/test/acceptance/js/TestConfig.js @@ -5,6 +5,9 @@ const { RootKeyEncryptionKey, } = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor') +const AWS_S3_USER_FILES_STORAGE_CLASS = + process.env.AWS_S3_USER_FILES_STORAGE_CLASS + // use functions to get a fresh copy, not a reference, each time function s3BaseConfig() { return { @@ -43,6 +46,10 @@ function s3SSECConfig() { async getRootKeyEncryptionKeys() { return S3SSECKeys }, + storageClass: { + [process.env.AWS_S3_USER_FILES_BUCKET_NAME]: + AWS_S3_USER_FILES_STORAGE_CLASS, + }, } } @@ -176,6 +183,7 @@ function checkForUnexpectedTestFile() { checkForUnexpectedTestFile() module.exports = { + AWS_S3_USER_FILES_STORAGE_CLASS, BackendSettings, s3Config, s3SSECConfig,