github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #19282 from overleaf/jpa-filestore-sharding

Browse source 
[filestore] add sharding to test_acceptance in CI

GitOrigin-RevId: dd16a0370878d76a934d277838083e08ae0ad8c2
This commit is contained in:
Jakob Ackermann 2024-07-12 11:46:33 +02:00 committed by Copybot
parent 788009330c
commit cbe7a2f5a9
24 changed files with 82 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
services/chat/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/chat/docker-compose.ci.yml
View file

@ -28,7 +28,7 @@ services:
mongo:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/clsi/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/clsi/docker-compose.ci.yml
View file

@ -29,7 +29,7 @@ services:
TEXLIVE_IMAGE:
ENABLE_PDF_CACHING: "true"
PDF_CACHING_ENABLE_WORKER_POOL: "true"
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/contacts/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/contacts/docker-compose.ci.yml
View file

@ -28,7 +28,7 @@ services:
mongo:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/docstore/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/docstore/docker-compose.ci.yml
View file

@ -36,7 +36,7 @@ services:
gcs:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/document-updater/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/document-updater/docker-compose.ci.yml
View file

@ -33,7 +33,7 @@ services:
redis:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/filestore/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

1
services/filestore/buildscript.txt
View file

@ -8,3 +8,4 @@ filestore
--node-version=18.20.2
--public-repo=True
--script-version=4.5.0
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_

2
services/filestore/docker-compose.ci.yml
View file

@ -45,7 +45,7 @@ services:
gcs:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

34
services/filestore/test/acceptance/js/TestConfig.js
View file

@ -1,3 +1,4 @@
const fs = require('fs')
const Path = require('path')
// use functions to get a fresh copy, not a reference, each time
@ -64,26 +65,26 @@ function fallbackStores(primaryConfig, fallbackConfig) {
}
module.exports = {
FSPersistor: {
SHARD_01_FSPersistor: {
backend: 'fs',
stores: fsStores(),
},
S3Persistor: {
SHARD_01_S3Persistor: {
backend: 's3',
s3: s3Config(),
stores: s3Stores(),
},
S3PersistorDefaultProviderCredentials: {
SHARD_01_S3PersistorDefaultProviderCredentials: {
backend: 's3',
s3: s3ConfigDefaultProviderCredentials(),
stores: s3Stores(),
},
GcsPersistor: {
SHARD_01_GcsPersistor: {
backend: 'gcs',
gcs: gcsConfig(),
stores: gcsStores(),
},
FallbackS3ToFSPersistor: {
SHARD_02_FallbackS3ToFSPersistor: {
backend: 's3',
s3: s3Config(),
stores: s3Stores(),
@ -92,7 +93,7 @@ module.exports = {
buckets: fallbackStores(s3Stores(), fsStores()),
},
},
FallbackFSToS3Persistor: {
SHARD_02_FallbackFSToS3Persistor: {
backend: 'fs',
s3: s3Config(),
stores: fsStores(),
@ -101,7 +102,7 @@ module.exports = {
buckets: fallbackStores(fsStores(), s3Stores()),
},
},
FallbackGcsToS3Persistor: {
SHARD_03_FallbackGcsToS3Persistor: {
backend: 'gcs',
gcs: gcsConfig(),
stores: gcsStores(),
@ -111,7 +112,7 @@ module.exports = {
buckets: fallbackStores(gcsStores(), s3Stores()),
},
},
FallbackS3ToGcsPersistor: {
SHARD_03_FallbackS3ToGcsPersistor: {
backend: 's3',
// can use the same bucket names for gcs and s3 (in tests)
stores: s3Stores(),
@ -123,3 +124,20 @@ module.exports = {
},
},
}
function checkForUnexpectedTestFile() {
const awareOfSharding = [
'FilestoreApp.js',
'FilestoreTests.js',
'TestConfig.js',
'TestHelper.js',
]
for (const file of fs.readdirSync(__dirname).sort()) {
if (!awareOfSharding.includes(file)) {
throw new Error(
`Found new test file ${file}: All tests must be aware of the SHARD_ prefix.`
)
}
}
}
checkForUnexpectedTestFile()

6
services/history-v1/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/history-v1/docker-compose.ci.yml
View file

@ -35,7 +35,7 @@ services:
gcs:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/notifications/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/notifications/docker-compose.ci.yml
View file

@ -28,7 +28,7 @@ services:
mongo:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/project-history/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/project-history/docker-compose.ci.yml
View file

@ -33,7 +33,7 @@ services:
redis:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/real-time/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/real-time/docker-compose.ci.yml
View file

@ -31,7 +31,7 @@ services:
redis:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar:

6
services/spelling/Makefile
View file

@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker compose ${DOCKER_COMPOSE_FLAGS}
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)

2
services/spelling/docker-compose.ci.yml
View file

@ -25,7 +25,7 @@ services:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
command: npm run test:acceptance:_run
command: npm run test:acceptance
tar: