Merge pull request #21361 from overleaf/jpa-filestore-minio

[filestore] migrate to minio as S3 backend for running tests against

GitOrigin-RevId: aa098d8baa4445f5dec7d651b6cf5ed081b0a331

services/filestore/buildscript.txt

@@ -2,7 +2,7 @@ filestore
 --data-dirs=uploads,user_files,template_files
 --dependencies=s3,gcs
 --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,GCS_USER_FILES_BUCKET_NAME=fake_userfiles,GCS_TEMPLATE_FILES_BUCKET_NAME=fake_templatefiles
+--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
 --env-pass-through=
 --esmock-loader=False
 --node-version=18.20.2

services/filestore/docker-compose.ci.yml

@@ -21,10 +21,12 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_ENDPOINT: https://minio:9000
       AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
+      AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
-      AWS_SECRET_ACCESS_KEY: fake
+      AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
       GCS_API_ENDPOINT: http://gcs:9090
       GCS_PROJECT_ID: fake
       STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@@ -33,13 +35,19 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
       ENABLE_CONVERSIONS: "true"
       USE_PROM_METRICS: "true"
-      AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
+      AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
-      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
+      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
-      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+      GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
-      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
+    volumes:
+      - ./test/acceptance/certs:/certs
     depends_on:
-      s3:
+      certs:
-        condition: service_healthy
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+      minio_setup:
+        condition: service_completed_successfully
       gcs:
         condition: service_healthy
     user: node
@@ -53,14 +61,121 @@ services:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
-  s3:
+  certs:
-    image: adobe/s3mock:2.4.14
+    image: node:18.20.2
+    volumes:
+      - ./test/acceptance/certs:/certs
+    working_dir: /certs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        if [ ! -f ./certgen ]; then
+          wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+          chmod +x ./certgen
+        fi
+        if [ ! -f private.key ] || [ ! -f public.crt ]; then
+          ./certgen -host minio
+        fi
+
+  minio:
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+    command: server /data
+    volumes:
+      - ./test/acceptance/certs:/root/.minio/certs
     environment:
-      - initialBuckets=fake_user_files,fake_template_files,bucket
+      MINIO_ROOT_USER: MINIO_ROOT_USER
-    healthcheck:
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
-      test: wget --quiet --output-document=/dev/null http://localhost:9090
+    depends_on:
-      interval: 1s
+      certs:
-      retries: 20
+        condition: service_completed_successfully
+
+  minio_setup:
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+    image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+    volumes:
+      - ./test/acceptance/certs:/root/.mc/certs/CAs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        sleep 1
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+
+        mc mb --ignore-existing s3/fake-user-files
+        mc mb --ignore-existing s3/fake-template-files
+        mc admin user add s3 \
+          OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+          OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+        echo '
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              }
+            ]
+          }' > policy-filestore.json
+
+        mc admin policy create s3 overleaf-filestore policy-filestore.json
+        mc admin policy attach s3 overleaf-filestore \
+          --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
   gcs:
     image: fsouza/fake-gcs-server:1.45.2
     command: ["--port=9090", "--scheme=http"]

services/filestore/docker-compose.yml

@@ -31,15 +31,18 @@ services:
       - .:/overleaf/services/filestore
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
+      - ./test/acceptance/certs:/certs
     working_dir: /overleaf/services/filestore
     environment:
       ELASTIC_SEARCH_DSN: es:9200
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_ENDPOINT: https://minio:9000
       AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
+      AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
-      AWS_SECRET_ACCESS_KEY: fake
+      AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
       GCS_API_ENDPOINT: http://gcs:9090
       GCS_PROJECT_ID: fake
       STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@@ -49,26 +52,137 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
       ENABLE_CONVERSIONS: "true"
       USE_PROM_METRICS: "true"
-      AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
+      AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
-      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
+      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
-      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+      GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
-      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
     user: node
     depends_on:
-      s3:
+      certs:
-        condition: service_healthy
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+      minio_setup:
+        condition: service_completed_successfully
       gcs:
         condition: service_healthy
     command: npm run --silent test:acceptance
 
-  s3:
+  certs:
-    image: adobe/s3mock:2.4.14
+    image: node:18.20.2
+    volumes:
+      - ./test/acceptance/certs:/certs
+    working_dir: /certs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        if [ ! -f ./certgen ]; then
+          wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+          chmod +x ./certgen
+        fi
+        if [ ! -f private.key ] || [ ! -f public.crt ]; then
+          ./certgen -host minio
+        fi
+
+  minio:
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+    command: server /data
+    volumes:
+      - ./test/acceptance/certs:/root/.minio/certs
     environment:
-      - initialBuckets=fake_user_files,fake_template_files,bucket
+      MINIO_ROOT_USER: MINIO_ROOT_USER
-    healthcheck:
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
-      test: wget --quiet --output-document=/dev/null http://localhost:9090
+    depends_on:
-      interval: 1s
+      certs:
-      retries: 20
+        condition: service_completed_successfully
+
+  minio_setup:
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+    image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+    volumes:
+      - ./test/acceptance/certs:/root/.mc/certs/CAs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        sleep 1
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+
+        mc mb --ignore-existing s3/fake-user-files
+        mc mb --ignore-existing s3/fake-template-files
+        mc admin user add s3 \
+          OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+          OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+        echo '
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              }
+            ]
+          }' > policy-filestore.json
+
+        mc admin policy create s3 overleaf-filestore policy-filestore.json
+        mc admin policy attach s3 overleaf-filestore \
+          --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
   gcs:
     image: fsouza/fake-gcs-server:1.45.2
     command: ["--port=9090", "--scheme=http"]

services/filestore/test/acceptance/certs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

services/filestore/test/acceptance/js/FilestoreApp.js

@@ -2,14 +2,11 @@ const logger = require('@overleaf/logger')
 const ObjectPersistor = require('@overleaf/object-persistor')
 const Settings = require('@overleaf/settings')
 const { promisify } = require('util')
-const AWS = require('aws-sdk')
 const App = require('../../../app')
 const FileHandler = require('../../../app/js/FileHandler')
 
 logger.logger.level('info')
 
-const sleep = promisify(setTimeout)
-
 class FilestoreApp {
   async runServer() {
     if (!this.server) {
@@ -27,15 +24,6 @@ class FilestoreApp {
       })
     }
 
-    if (Settings.filestore.backend === 's3') {
-      try {
-        await FilestoreApp.waitForS3()
-      } catch (err) {
-        await this.stop()
-        throw err
-      }
-    }
-
     this.persistor = ObjectPersistor({
       ...Settings.filestore,
       paths: Settings.path,
@@ -52,41 +40,6 @@ class FilestoreApp {
       delete this.server
     }
   }
-
-  static async waitForS3() {
-    let tries = 0
-    if (!Settings.filestore.s3.endpoint) {
-      return
-    }
-
-    const s3 = new AWS.S3({
-      accessKeyId: Settings.filestore.s3.key,
-      secretAccessKey: Settings.filestore.s3.secret,
-      endpoint: Settings.filestore.s3.endpoint,
-      s3ForcePathStyle: true,
-      signatureVersion: 'v4',
-    })
-
-    while (true) {
-      try {
-        return await s3
-          .putObject({
-            Key: 'startup',
-            Body: '42',
-            Bucket: Settings.filestore.stores.user_files,
-          })
-          .promise()
-      } catch (err) {
-        // swallow errors, as we may experience them until fake-s3 is running
-        if (tries === 9) {
-          // throw just before hitting the 10s test timeout
-          throw err
-        }
-        tries++
-        await sleep(1000)
-      }
-    }
-  }
 }
 
 module.exports = FilestoreApp

services/filestore/test/acceptance/js/FilestoreTests.js

@@ -31,7 +31,7 @@ process.on('unhandledRejection', e => {
 
 // store settings for multiple backends, so that we can test each one.
 // fs will always be available - add others if they are configured
-const BackendSettings = require('./TestConfig')
+const { BackendSettings, s3Config } = require('./TestConfig')
 
 describe('Filestore', function () {
   this.timeout(1000 * 10)
@@ -467,17 +467,18 @@ describe('Filestore', function () {
           beforeEach(async function () {
             constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
             fileId = new ObjectId().toString()
-            bucketName = new ObjectId().toString()
+            bucketName = `random-bucket-${new ObjectId().toString()}`
             fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
 
+            const cfg = s3Config()
             const s3ClientSettings = {
               credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+                accessKeyId: process.env.MINIO_ROOT_USER,
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+                secretAccessKey: process.env.MINIO_ROOT_PASSWORD,
               },
-              endpoint: process.env.AWS_S3_ENDPOINT,
+              endpoint: cfg.endpoint,
-              sslEnabled: false,
+              httpOptions: cfg.httpOptions,
-              s3ForcePathStyle: true,
+              s3ForcePathStyle: cfg.pathStyle,
             }
 
             const s3 = new S3(s3ClientSettings)

services/filestore/test/acceptance/js/TestConfig.js

@@ -1,22 +1,33 @@
 const fs = require('fs')
 const Path = require('path')
+const https = require('https')
 
 // use functions to get a fresh copy, not a reference, each time
+function s3BaseConfig() {
+  return {
+    endpoint: process.env.AWS_S3_ENDPOINT,
+    pathStyle: true,
+    partSize: 100 * 1024 * 1024,
+    httpOptions: {
+      agent: new https.Agent({
+        rejectUnauthorized: true,
+        ca: [fs.readFileSync('/certs/public.crt')],
+      }),
+    },
+  }
+}
+
 function s3Config() {
   return {
     key: process.env.AWS_ACCESS_KEY_ID,
     secret: process.env.AWS_SECRET_ACCESS_KEY,
-    endpoint: process.env.AWS_S3_ENDPOINT,
+    ...s3BaseConfig(),
-    pathStyle: true,
-    partSize: 100 * 1024 * 1024,
   }
 }
 
 function s3ConfigDefaultProviderCredentials() {
   return {
-    endpoint: process.env.AWS_S3_ENDPOINT,
+    ...s3BaseConfig(),
-    pathStyle: true,
-    partSize: 100 * 1024 * 1024,
   }
 }
 
@@ -60,7 +71,7 @@ function fallbackStores(primaryConfig, fallbackConfig) {
   }
 }
 
-module.exports = {
+const BackendSettings = {
   SHARD_01_FSPersistor: {
     backend: 'fs',
     stores: fsStores(),
@@ -137,3 +148,8 @@ function checkForUnexpectedTestFile() {
   }
 }
 checkForUnexpectedTestFile()
+
+module.exports = {
+  BackendSettings,
+  s3Config,
+}