Merge pull request #21361 from overleaf/jpa-filestore-minio

[filestore] migrate to minio as S3 backend for running tests against GitOrigin-RevId: aa098d8baa4445f5dec7d651b6cf5ed081b0a331
2024-11-21 20:47:08 -05:00 · 2024-10-31 13:08:26 +01:00 · 2024-10-31 13:08:26 +01:00 · a551a0e9f7
commit a551a0e9f7
parent e1149e80b3
7 changed files with 295 additions and 94 deletions
--- a/services/filestore/buildscript.txt
+++ b/services/filestore/buildscript.txt
@ -2,7 +2,7 @@ filestore
 --data-dirs=uploads,user_files,template_files
 --dependencies=s3,gcs
 --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,GCS_USER_FILES_BUCKET_NAME=fake_userfiles,GCS_TEMPLATE_FILES_BUCKET_NAME=fake_templatefiles
+--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
 --env-pass-through=
 --esmock-loader=False
 --node-version=18.20.2
--- a/services/filestore/docker-compose.ci.yml
+++ b/services/filestore/docker-compose.ci.yml
@ -21,10 +21,12 @@ services:
      ELASTIC_SEARCH_DSN: es:9200
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_ENDPOINT: https://minio:9000
      AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
-      AWS_SECRET_ACCESS_KEY: fake
+      AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@ -33,13 +35,19 @@ services:
      NODE_OPTIONS: "--unhandled-rejections=strict"
      ENABLE_CONVERSIONS: "true"
      USE_PROM_METRICS: "true"
-      AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
-      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
-      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
-      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
+      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
+      GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
+    volumes:
+      - ./test/acceptance/certs:/certs
    depends_on:
-      s3:
-        condition: service_healthy
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+      minio_setup:
+        condition: service_completed_successfully
      gcs:
        condition: service_healthy
    user: node
@ -53,14 +61,121 @@ services:
      - ./:/tmp/build/
    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
    user: root
-  s3:
-    image: adobe/s3mock:2.4.14
+  certs:
+    image: node:18.20.2
+    volumes:
+      - ./test/acceptance/certs:/certs
+    working_dir: /certs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        if [ ! -f ./certgen ]; then
+          wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+          chmod +x ./certgen
+        fi
+        if [ ! -f private.key ] || [ ! -f public.crt ]; then
+          ./certgen -host minio
+        fi
+
+  minio:
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+    command: server /data
+    volumes:
+      - ./test/acceptance/certs:/root/.minio/certs
    environment:
-      - initialBuckets=fake_user_files,fake_template_files,bucket
-    healthcheck:
-      test: wget --quiet --output-document=/dev/null http://localhost:9090
-      interval: 1s
-      retries: 20
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+
+  minio_setup:
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+    image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+    volumes:
+      - ./test/acceptance/certs:/root/.mc/certs/CAs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        sleep 1
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+
+        mc mb --ignore-existing s3/fake-user-files
+        mc mb --ignore-existing s3/fake-template-files
+        mc admin user add s3 \
+          OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+          OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+        echo '
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              }
+            ]
+          }' > policy-filestore.json
+
+        mc admin policy create s3 overleaf-filestore policy-filestore.json
+        mc admin policy attach s3 overleaf-filestore \
+          --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
  gcs:
    image: fsouza/fake-gcs-server:1.45.2
    command: ["--port=9090", "--scheme=http"]
--- a/services/filestore/docker-compose.yml
+++ b/services/filestore/docker-compose.yml
@ -31,15 +31,18 @@ services:
      - .:/overleaf/services/filestore
      - ../../node_modules:/overleaf/node_modules
      - ../../libraries:/overleaf/libraries
+      - ./test/acceptance/certs:/certs
    working_dir: /overleaf/services/filestore
    environment:
      ELASTIC_SEARCH_DSN: es:9200
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_ENDPOINT: https://minio:9000
      AWS_S3_PATH_STYLE: 'true'
-      AWS_ACCESS_KEY_ID: fake
-      AWS_SECRET_ACCESS_KEY: fake
+      AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@ -49,26 +52,137 @@ services:
      NODE_OPTIONS: "--unhandled-rejections=strict"
      ENABLE_CONVERSIONS: "true"
      USE_PROM_METRICS: "true"
-      AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
-      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
-      GCS_USER_FILES_BUCKET_NAME: fake_userfiles
-      GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+      AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
+      AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
+      GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
+      GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
    user: node
    depends_on:
-      s3:
-        condition: service_healthy
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+      minio_setup:
+        condition: service_completed_successfully
      gcs:
        condition: service_healthy
    command: npm run --silent test:acceptance

-  s3:
-    image: adobe/s3mock:2.4.14
+  certs:
+    image: node:18.20.2
+    volumes:
+      - ./test/acceptance/certs:/certs
+    working_dir: /certs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        if [ ! -f ./certgen ]; then
+          wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+          chmod +x ./certgen
+        fi
+        if [ ! -f private.key ] || [ ! -f public.crt ]; then
+          ./certgen -host minio
+        fi
+
+  minio:
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+    command: server /data
+    volumes:
+      - ./test/acceptance/certs:/root/.minio/certs
    environment:
-      - initialBuckets=fake_user_files,fake_template_files,bucket
-    healthcheck:
-      test: wget --quiet --output-document=/dev/null http://localhost:9090
-      interval: 1s
-      retries: 20
+      MINIO_ROOT_USER: MINIO_ROOT_USER
+      MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+
+  minio_setup:
+    depends_on:
+      certs:
+        condition: service_completed_successfully
+      minio:
+        condition: service_started
+    image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+    volumes:
+      - ./test/acceptance/certs:/root/.mc/certs/CAs
+    entrypoint: sh
+    command:
+      - '-cex'
+      - |
+        sleep 1
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+        || sleep 3 && \
+        mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+
+        mc mb --ignore-existing s3/fake-user-files
+        mc mb --ignore-existing s3/fake-template-files
+        mc admin user add s3 \
+          OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+          OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+        echo '
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-user-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::fake-template-files/*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:ListBucket"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              },
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:PutObject",
+                  "s3:GetObject",
+                  "s3:DeleteObject"
+                ],
+                "Resource": "arn:aws:s3:::random-bucket-*"
+              }
+            ]
+          }' > policy-filestore.json
+
+        mc admin policy create s3 overleaf-filestore policy-filestore.json
+        mc admin policy attach s3 overleaf-filestore \
+          --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
  gcs:
    image: fsouza/fake-gcs-server:1.45.2
    command: ["--port=9090", "--scheme=http"]
--- a/services/filestore/test/acceptance/certs/.gitignore
+++ b/services/filestore/test/acceptance/certs/.gitignore
@ -0,0 +1,2 @@
+*
+!.gitignore
--- a/services/filestore/test/acceptance/js/FilestoreApp.js
+++ b/services/filestore/test/acceptance/js/FilestoreApp.js
@ -2,14 +2,11 @@ const logger = require('@overleaf/logger')
 const ObjectPersistor = require('@overleaf/object-persistor')
 const Settings = require('@overleaf/settings')
 const { promisify } = require('util')
-const AWS = require('aws-sdk')
 const App = require('../../../app')
 const FileHandler = require('../../../app/js/FileHandler')

 logger.logger.level('info')

-const sleep = promisify(setTimeout)
-
 class FilestoreApp {
  async runServer() {
    if (!this.server) {
@ -27,15 +24,6 @@ class FilestoreApp {
      })
    }

-    if (Settings.filestore.backend === 's3') {
-      try {
-        await FilestoreApp.waitForS3()
-      } catch (err) {
-        await this.stop()
-        throw err
-      }
-    }
-
    this.persistor = ObjectPersistor({
      ...Settings.filestore,
      paths: Settings.path,
@ -52,41 +40,6 @@ class FilestoreApp {
      delete this.server
    }
  }
-
-  static async waitForS3() {
-    let tries = 0
-    if (!Settings.filestore.s3.endpoint) {
-      return
-    }
-
-    const s3 = new AWS.S3({
-      accessKeyId: Settings.filestore.s3.key,
-      secretAccessKey: Settings.filestore.s3.secret,
-      endpoint: Settings.filestore.s3.endpoint,
-      s3ForcePathStyle: true,
-      signatureVersion: 'v4',
-    })
-
-    while (true) {
-      try {
-        return await s3
-          .putObject({
-            Key: 'startup',
-            Body: '42',
-            Bucket: Settings.filestore.stores.user_files,
-          })
-          .promise()
-      } catch (err) {
-        // swallow errors, as we may experience them until fake-s3 is running
-        if (tries === 9) {
-          // throw just before hitting the 10s test timeout
-          throw err
-        }
-        tries++
-        await sleep(1000)
-      }
-    }
-  }
 }

 module.exports = FilestoreApp
--- a/services/filestore/test/acceptance/js/FilestoreTests.js
+++ b/services/filestore/test/acceptance/js/FilestoreTests.js
@ -31,7 +31,7 @@ process.on('unhandledRejection', e => {

 // store settings for multiple backends, so that we can test each one.
 // fs will always be available - add others if they are configured
-const BackendSettings = require('./TestConfig')
+const { BackendSettings, s3Config } = require('./TestConfig')

 describe('Filestore', function () {
  this.timeout(1000 * 10)
@ -467,17 +467,18 @@ describe('Filestore', function () {
          beforeEach(async function () {
            constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
            fileId = new ObjectId().toString()
-            bucketName = new ObjectId().toString()
+            bucketName = `random-bucket-${new ObjectId().toString()}`
            fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`

+            const cfg = s3Config()
            const s3ClientSettings = {
              credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID,
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+                accessKeyId: process.env.MINIO_ROOT_USER,
+                secretAccessKey: process.env.MINIO_ROOT_PASSWORD,
              },
-              endpoint: process.env.AWS_S3_ENDPOINT,
-              sslEnabled: false,
-              s3ForcePathStyle: true,
+              endpoint: cfg.endpoint,
+              httpOptions: cfg.httpOptions,
+              s3ForcePathStyle: cfg.pathStyle,
            }

            const s3 = new S3(s3ClientSettings)
--- a/services/filestore/test/acceptance/js/TestConfig.js
+++ b/services/filestore/test/acceptance/js/TestConfig.js
@ -1,22 +1,33 @@
 const fs = require('fs')
 const Path = require('path')
+const https = require('https')

 // use functions to get a fresh copy, not a reference, each time
+function s3BaseConfig() {
+  return {
+    endpoint: process.env.AWS_S3_ENDPOINT,
+    pathStyle: true,
+    partSize: 100 * 1024 * 1024,
+    httpOptions: {
+      agent: new https.Agent({
+        rejectUnauthorized: true,
+        ca: [fs.readFileSync('/certs/public.crt')],
+      }),
+    },
+  }
+}
+
 function s3Config() {
  return {
    key: process.env.AWS_ACCESS_KEY_ID,
    secret: process.env.AWS_SECRET_ACCESS_KEY,
-    endpoint: process.env.AWS_S3_ENDPOINT,
-    pathStyle: true,
-    partSize: 100 * 1024 * 1024,
+    ...s3BaseConfig(),
  }
 }

 function s3ConfigDefaultProviderCredentials() {
  return {
-    endpoint: process.env.AWS_S3_ENDPOINT,
-    pathStyle: true,
-    partSize: 100 * 1024 * 1024,
+    ...s3BaseConfig(),
  }
 }

@ -60,7 +71,7 @@ function fallbackStores(primaryConfig, fallbackConfig) {
  }
 }

-module.exports = {
+const BackendSettings = {
  SHARD_01_FSPersistor: {
    backend: 'fs',
    stores: fsStores(),
@ -137,3 +148,8 @@ function checkForUnexpectedTestFile() {
  }
 }
 checkForUnexpectedTestFile()
+
+module.exports = {
+  BackendSettings,
+  s3Config,
+}