Merge pull request #17237 from overleaf/jpa-s3-iam

[object-persistor] add support for IAM credentials in S3Persistor

GitOrigin-RevId: 4ff34082ca557b17d8a7803f700ee704a3d184c6

libraries/object-persistor/src/S3Persistor.js

@@ -14,12 +14,7 @@ const { pipeline, PassThrough } = require('stream')
 const fs = require('fs')
 const S3 = require('aws-sdk/clients/s3')
 const { URL } = require('url')
-const {
-  WriteError,
-  ReadError,
-  NotFoundError,
-  SettingsError,
-} = require('./Errors')
+const { WriteError, ReadError, NotFoundError } = require('./Errors')
 
 module.exports = class S3Persistor extends AbstractPersistor {
   constructor(settings = {}) {
@@ -336,23 +331,11 @@ module.exports = class S3Persistor extends AbstractPersistor {
   }
 
   _getClientForBucket(bucket, clientOptions) {
-    if (this.settings.bucketCreds && this.settings.bucketCreds[bucket]) {
-      return new S3(
-        this._buildClientOptions(
-          this.settings.bucketCreds[bucket],
-          clientOptions
-        )
+    return new S3(
+      this._buildClientOptions(
+        this.settings.bucketCreds?.[bucket],
+        clientOptions
       )
-    }
-
-    // no specific credentials for the bucket
-    if (this.settings.key) {
-      return new S3(this._buildClientOptions(null, clientOptions))
-    }
-
-    throw new SettingsError(
-      'no bucket-specific or default credentials provided',
-      { bucket }
     )
   }
 
@@ -364,11 +347,14 @@ module.exports = class S3Persistor extends AbstractPersistor {
         accessKeyId: bucketCredentials.auth_key,
         secretAccessKey: bucketCredentials.auth_secret,
       }
-    } else {
+    } else if (this.settings.key) {
       options.credentials = {
         accessKeyId: this.settings.key,
         secretAccessKey: this.settings.secret,
       }
+    } else {
+      // Use the default credentials provider (process.env -> SSP -> ini -> IAM)
+      // Docs: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CredentialProviderChain.html#defaultProviders-property
     }
 
     if (this.settings.endpoint) {

libraries/object-persistor/test/unit/S3PersistorTests.js

@@ -281,14 +281,16 @@ describe('S3PersistorTests', function () {
         expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials)
         expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials)
       })
+    })
 
-      it('throws an error if there are no credentials for the bucket', async function () {
+    describe('without hard-coded credentials', function () {
+      it('uses the default provider chain', async function () {
         delete settings.key
         delete settings.secret
 
-        await expect(
-          S3Persistor.getObjectStream('anotherBucket', key)
-        ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.SettingsError)
+        await S3Persistor.getObjectStream(bucket, key)
+        expect(S3).to.have.been.calledOnce
+        expect(S3.args[0].credentials).to.not.exist
       })
     })
 

services/filestore/config/settings.defaults.js

@@ -62,19 +62,16 @@ const settings = {
       signedUrlExpiryInMs: parseInt(process.env.LINK_EXPIRY_TIMEOUT || 60000),
     },
 
-    s3:
-      process.env.AWS_ACCESS_KEY_ID || process.env.S3_BUCKET_CREDENTIALS
-        ? {
-            key: process.env.AWS_ACCESS_KEY_ID,
-            secret: process.env.AWS_SECRET_ACCESS_KEY,
-            endpoint: process.env.AWS_S3_ENDPOINT,
-            pathStyle: process.env.AWS_S3_PATH_STYLE,
-            partSize: process.env.AWS_S3_PARTSIZE || 100 * 1024 * 1024,
-            bucketCreds: process.env.S3_BUCKET_CREDENTIALS
-              ? JSON.parse(process.env.S3_BUCKET_CREDENTIALS)
-              : undefined,
-          }
+    s3: {
+      key: process.env.AWS_ACCESS_KEY_ID,
+      secret: process.env.AWS_SECRET_ACCESS_KEY,
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      pathStyle: process.env.AWS_S3_PATH_STYLE,
+      partSize: process.env.AWS_S3_PARTSIZE || 100 * 1024 * 1024,
+      bucketCreds: process.env.S3_BUCKET_CREDENTIALS
+        ? JSON.parse(process.env.S3_BUCKET_CREDENTIALS)
         : undefined,
+    },
 
     // GCS should be configured by the service account on the kubernetes pod. See GOOGLE_APPLICATION_CREDENTIALS,
     // which will be picked up automatically.

services/filestore/test/acceptance/js/TestConfig.js

@@ -11,6 +11,14 @@ function s3Config() {
   }
 }
 
+function s3ConfigDefaultProviderCredentials() {
+  return {
+    endpoint: process.env.AWS_S3_ENDPOINT,
+    pathStyle: true,
+    partSize: 100 * 1024 * 1024,
+  }
+}
+
 function s3Stores() {
   return {
     user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
@@ -65,6 +73,11 @@ module.exports = {
     s3: s3Config(),
     stores: s3Stores(),
   },
+  S3PersistorDefaultProviderCredentials: {
+    backend: 's3',
+    s3: s3ConfigDefaultProviderCredentials(),
+    stores: s3Stores(),
+  },
   GcsPersistor: {
     backend: 'gcs',
     gcs: gcsConfig(),