From a551a0e9f75bdf22ee29c8639794999dcf9787ea Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 31 Oct 2024 13:08:26 +0100 Subject: [PATCH] Merge pull request #21361 from overleaf/jpa-filestore-minio [filestore] migrate to minio as S3 backend for running tests against GitOrigin-RevId: aa098d8baa4445f5dec7d651b6cf5ed081b0a331 --- services/filestore/buildscript.txt | 2 +- services/filestore/docker-compose.ci.yml | 147 ++++++++++++++++-- services/filestore/docker-compose.yml | 146 +++++++++++++++-- .../test/acceptance/certs/.gitignore | 2 + .../test/acceptance/js/FilestoreApp.js | 47 ------ .../test/acceptance/js/FilestoreTests.js | 15 +- .../test/acceptance/js/TestConfig.js | 30 +++- 7 files changed, 295 insertions(+), 94 deletions(-) create mode 100644 services/filestore/test/acceptance/certs/.gitignore diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index 147e8b4a46..ec315694f1 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -2,7 +2,7 @@ filestore --data-dirs=uploads,user_files,template_files --dependencies=s3,gcs --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker ---env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,GCS_USER_FILES_BUCKET_NAME=fake_userfiles,GCS_TEMPLATE_FILES_BUCKET_NAME=fake_templatefiles +--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files --env-pass-through= --esmock-loader=False --node-version=18.20.2 diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index aaa11bea6b..d1b74c9995 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -21,10 +21,12 @@ services: ELASTIC_SEARCH_DSN: es:9200 MONGO_HOST: mongo POSTGRES_HOST: postgres - AWS_S3_ENDPOINT: http://s3:9090 + AWS_S3_ENDPOINT: https://minio:9000 AWS_S3_PATH_STYLE: 'true' - AWS_ACCESS_KEY_ID: fake - AWS_SECRET_ACCESS_KEY: fake + AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY + MINIO_ROOT_USER: MINIO_ROOT_USER + MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD GCS_API_ENDPOINT: http://gcs:9090 GCS_PROJECT_ID: fake STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1 @@ -33,13 +35,19 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" - AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files - AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files - GCS_USER_FILES_BUCKET_NAME: fake_userfiles - GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles + AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files + AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files + GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files + GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files + volumes: + - ./test/acceptance/certs:/certs depends_on: - s3: - condition: service_healthy + certs: + condition: service_completed_successfully + minio: + condition: service_started + minio_setup: + condition: service_completed_successfully gcs: condition: service_healthy user: node @@ -53,14 +61,121 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - s3: - image: adobe/s3mock:2.4.14 + certs: + image: node:18.20.2 + volumes: + - ./test/acceptance/certs:/certs + working_dir: /certs + entrypoint: sh + command: + - '-cex' + - | + if [ ! -f ./certgen ]; then + wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)" + chmod +x ./certgen + fi + if [ ! -f private.key ] || [ ! -f public.crt ]; then + ./certgen -host minio + fi + + minio: + image: minio/minio:RELEASE.2024-10-13T13-34-11Z + command: server /data + volumes: + - ./test/acceptance/certs:/root/.minio/certs environment: - - initialBuckets=fake_user_files,fake_template_files,bucket - healthcheck: - test: wget --quiet --output-document=/dev/null http://localhost:9090 - interval: 1s - retries: 20 + MINIO_ROOT_USER: MINIO_ROOT_USER + MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD + depends_on: + certs: + condition: service_completed_successfully + + minio_setup: + depends_on: + certs: + condition: service_completed_successfully + minio: + condition: service_started + image: minio/mc:RELEASE.2024-10-08T09-37-26Z + volumes: + - ./test/acceptance/certs:/root/.mc/certs/CAs + entrypoint: sh + command: + - '-cex' + - | + sleep 1 + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD + + mc mb --ignore-existing s3/fake-user-files + mc mb --ignore-existing s3/fake-template-files + mc admin user add s3 \ + OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \ + OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY + + echo ' + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::fake-user-files" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::fake-user-files/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::fake-template-files" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::fake-template-files/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::random-bucket-*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::random-bucket-*" + } + ] + }' > policy-filestore.json + + mc admin policy create s3 overleaf-filestore policy-filestore.json + mc admin policy attach s3 overleaf-filestore \ + --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID gcs: image: fsouza/fake-gcs-server:1.45.2 command: ["--port=9090", "--scheme=http"] diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 427384e452..818acf0a8a 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -31,15 +31,18 @@ services: - .:/overleaf/services/filestore - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ./test/acceptance/certs:/certs working_dir: /overleaf/services/filestore environment: ELASTIC_SEARCH_DSN: es:9200 MONGO_HOST: mongo POSTGRES_HOST: postgres - AWS_S3_ENDPOINT: http://s3:9090 + AWS_S3_ENDPOINT: https://minio:9000 AWS_S3_PATH_STYLE: 'true' - AWS_ACCESS_KEY_ID: fake - AWS_SECRET_ACCESS_KEY: fake + AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY + MINIO_ROOT_USER: MINIO_ROOT_USER + MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD GCS_API_ENDPOINT: http://gcs:9090 GCS_PROJECT_ID: fake STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1 @@ -49,26 +52,137 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" - AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files - AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files - GCS_USER_FILES_BUCKET_NAME: fake_userfiles - GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles + AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files + AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files + GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files + GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files user: node depends_on: - s3: - condition: service_healthy + certs: + condition: service_completed_successfully + minio: + condition: service_started + minio_setup: + condition: service_completed_successfully gcs: condition: service_healthy command: npm run --silent test:acceptance - s3: - image: adobe/s3mock:2.4.14 + certs: + image: node:18.20.2 + volumes: + - ./test/acceptance/certs:/certs + working_dir: /certs + entrypoint: sh + command: + - '-cex' + - | + if [ ! -f ./certgen ]; then + wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)" + chmod +x ./certgen + fi + if [ ! -f private.key ] || [ ! -f public.crt ]; then + ./certgen -host minio + fi + + minio: + image: minio/minio:RELEASE.2024-10-13T13-34-11Z + command: server /data + volumes: + - ./test/acceptance/certs:/root/.minio/certs environment: - - initialBuckets=fake_user_files,fake_template_files,bucket - healthcheck: - test: wget --quiet --output-document=/dev/null http://localhost:9090 - interval: 1s - retries: 20 + MINIO_ROOT_USER: MINIO_ROOT_USER + MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD + depends_on: + certs: + condition: service_completed_successfully + + minio_setup: + depends_on: + certs: + condition: service_completed_successfully + minio: + condition: service_started + image: minio/mc:RELEASE.2024-10-08T09-37-26Z + volumes: + - ./test/acceptance/certs:/root/.mc/certs/CAs + entrypoint: sh + command: + - '-cex' + - | + sleep 1 + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \ + || sleep 3 && \ + mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD + + mc mb --ignore-existing s3/fake-user-files + mc mb --ignore-existing s3/fake-template-files + mc admin user add s3 \ + OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \ + OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY + + echo ' + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::fake-user-files" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::fake-user-files/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::fake-template-files" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::fake-template-files/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::random-bucket-*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::random-bucket-*" + } + ] + }' > policy-filestore.json + + mc admin policy create s3 overleaf-filestore policy-filestore.json + mc admin policy attach s3 overleaf-filestore \ + --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID gcs: image: fsouza/fake-gcs-server:1.45.2 command: ["--port=9090", "--scheme=http"] diff --git a/services/filestore/test/acceptance/certs/.gitignore b/services/filestore/test/acceptance/certs/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/services/filestore/test/acceptance/certs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index dfd0e0ab44..f6df01cc29 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -2,14 +2,11 @@ const logger = require('@overleaf/logger') const ObjectPersistor = require('@overleaf/object-persistor') const Settings = require('@overleaf/settings') const { promisify } = require('util') -const AWS = require('aws-sdk') const App = require('../../../app') const FileHandler = require('../../../app/js/FileHandler') logger.logger.level('info') -const sleep = promisify(setTimeout) - class FilestoreApp { async runServer() { if (!this.server) { @@ -27,15 +24,6 @@ class FilestoreApp { }) } - if (Settings.filestore.backend === 's3') { - try { - await FilestoreApp.waitForS3() - } catch (err) { - await this.stop() - throw err - } - } - this.persistor = ObjectPersistor({ ...Settings.filestore, paths: Settings.path, @@ -52,41 +40,6 @@ class FilestoreApp { delete this.server } } - - static async waitForS3() { - let tries = 0 - if (!Settings.filestore.s3.endpoint) { - return - } - - const s3 = new AWS.S3({ - accessKeyId: Settings.filestore.s3.key, - secretAccessKey: Settings.filestore.s3.secret, - endpoint: Settings.filestore.s3.endpoint, - s3ForcePathStyle: true, - signatureVersion: 'v4', - }) - - while (true) { - try { - return await s3 - .putObject({ - Key: 'startup', - Body: '42', - Bucket: Settings.filestore.stores.user_files, - }) - .promise() - } catch (err) { - // swallow errors, as we may experience them until fake-s3 is running - if (tries === 9) { - // throw just before hitting the 10s test timeout - throw err - } - tries++ - await sleep(1000) - } - } - } } module.exports = FilestoreApp diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 19db3ec919..d55631fef2 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -31,7 +31,7 @@ process.on('unhandledRejection', e => { // store settings for multiple backends, so that we can test each one. // fs will always be available - add others if they are configured -const BackendSettings = require('./TestConfig') +const { BackendSettings, s3Config } = require('./TestConfig') describe('Filestore', function () { this.timeout(1000 * 10) @@ -467,17 +467,18 @@ describe('Filestore', function () { beforeEach(async function () { constantFileContent = `This is a file in a different S3 bucket ${Math.random()}` fileId = new ObjectId().toString() - bucketName = new ObjectId().toString() + bucketName = `random-bucket-${new ObjectId().toString()}` fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}` + const cfg = s3Config() const s3ClientSettings = { credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + accessKeyId: process.env.MINIO_ROOT_USER, + secretAccessKey: process.env.MINIO_ROOT_PASSWORD, }, - endpoint: process.env.AWS_S3_ENDPOINT, - sslEnabled: false, - s3ForcePathStyle: true, + endpoint: cfg.endpoint, + httpOptions: cfg.httpOptions, + s3ForcePathStyle: cfg.pathStyle, } const s3 = new S3(s3ClientSettings) diff --git a/services/filestore/test/acceptance/js/TestConfig.js b/services/filestore/test/acceptance/js/TestConfig.js index 4b72bc971d..4bd97ee0e0 100644 --- a/services/filestore/test/acceptance/js/TestConfig.js +++ b/services/filestore/test/acceptance/js/TestConfig.js @@ -1,22 +1,33 @@ const fs = require('fs') const Path = require('path') +const https = require('https') // use functions to get a fresh copy, not a reference, each time +function s3BaseConfig() { + return { + endpoint: process.env.AWS_S3_ENDPOINT, + pathStyle: true, + partSize: 100 * 1024 * 1024, + httpOptions: { + agent: new https.Agent({ + rejectUnauthorized: true, + ca: [fs.readFileSync('/certs/public.crt')], + }), + }, + } +} + function s3Config() { return { key: process.env.AWS_ACCESS_KEY_ID, secret: process.env.AWS_SECRET_ACCESS_KEY, - endpoint: process.env.AWS_S3_ENDPOINT, - pathStyle: true, - partSize: 100 * 1024 * 1024, + ...s3BaseConfig(), } } function s3ConfigDefaultProviderCredentials() { return { - endpoint: process.env.AWS_S3_ENDPOINT, - pathStyle: true, - partSize: 100 * 1024 * 1024, + ...s3BaseConfig(), } } @@ -60,7 +71,7 @@ function fallbackStores(primaryConfig, fallbackConfig) { } } -module.exports = { +const BackendSettings = { SHARD_01_FSPersistor: { backend: 'fs', stores: fsStores(), @@ -137,3 +148,8 @@ function checkForUnexpectedTestFile() { } } checkForUnexpectedTestFile() + +module.exports = { + BackendSettings, + s3Config, +}