mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #19282 from overleaf/jpa-filestore-sharding
[filestore] add sharding to test_acceptance in CI GitOrigin-RevId: dd16a0370878d76a934d277838083e08ae0ad8c2
This commit is contained in:
parent
788009330c
commit
cbe7a2f5a9
24 changed files with 82 additions and 41 deletions
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -28,7 +28,7 @@ services:
|
|||
mongo:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -29,7 +29,7 @@ services:
|
|||
TEXLIVE_IMAGE:
|
||||
ENABLE_PDF_CACHING: "true"
|
||||
PDF_CACHING_ENABLE_WORKER_POOL: "true"
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -28,7 +28,7 @@ services:
|
|||
mongo:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -36,7 +36,7 @@ services:
|
|||
gcs:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -33,7 +33,7 @@ services:
|
|||
redis:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -8,3 +8,4 @@ filestore
|
|||
--node-version=18.20.2
|
||||
--public-repo=True
|
||||
--script-version=4.5.0
|
||||
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
|
||||
|
|
|
@ -45,7 +45,7 @@ services:
|
|||
gcs:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
|
||||
// use functions to get a fresh copy, not a reference, each time
|
||||
|
@ -64,26 +65,26 @@ function fallbackStores(primaryConfig, fallbackConfig) {
|
|||
}
|
||||
|
||||
module.exports = {
|
||||
FSPersistor: {
|
||||
SHARD_01_FSPersistor: {
|
||||
backend: 'fs',
|
||||
stores: fsStores(),
|
||||
},
|
||||
S3Persistor: {
|
||||
SHARD_01_S3Persistor: {
|
||||
backend: 's3',
|
||||
s3: s3Config(),
|
||||
stores: s3Stores(),
|
||||
},
|
||||
S3PersistorDefaultProviderCredentials: {
|
||||
SHARD_01_S3PersistorDefaultProviderCredentials: {
|
||||
backend: 's3',
|
||||
s3: s3ConfigDefaultProviderCredentials(),
|
||||
stores: s3Stores(),
|
||||
},
|
||||
GcsPersistor: {
|
||||
SHARD_01_GcsPersistor: {
|
||||
backend: 'gcs',
|
||||
gcs: gcsConfig(),
|
||||
stores: gcsStores(),
|
||||
},
|
||||
FallbackS3ToFSPersistor: {
|
||||
SHARD_02_FallbackS3ToFSPersistor: {
|
||||
backend: 's3',
|
||||
s3: s3Config(),
|
||||
stores: s3Stores(),
|
||||
|
@ -92,7 +93,7 @@ module.exports = {
|
|||
buckets: fallbackStores(s3Stores(), fsStores()),
|
||||
},
|
||||
},
|
||||
FallbackFSToS3Persistor: {
|
||||
SHARD_02_FallbackFSToS3Persistor: {
|
||||
backend: 'fs',
|
||||
s3: s3Config(),
|
||||
stores: fsStores(),
|
||||
|
@ -101,7 +102,7 @@ module.exports = {
|
|||
buckets: fallbackStores(fsStores(), s3Stores()),
|
||||
},
|
||||
},
|
||||
FallbackGcsToS3Persistor: {
|
||||
SHARD_03_FallbackGcsToS3Persistor: {
|
||||
backend: 'gcs',
|
||||
gcs: gcsConfig(),
|
||||
stores: gcsStores(),
|
||||
|
@ -111,7 +112,7 @@ module.exports = {
|
|||
buckets: fallbackStores(gcsStores(), s3Stores()),
|
||||
},
|
||||
},
|
||||
FallbackS3ToGcsPersistor: {
|
||||
SHARD_03_FallbackS3ToGcsPersistor: {
|
||||
backend: 's3',
|
||||
// can use the same bucket names for gcs and s3 (in tests)
|
||||
stores: s3Stores(),
|
||||
|
@ -123,3 +124,20 @@ module.exports = {
|
|||
},
|
||||
},
|
||||
}
|
||||
|
||||
function checkForUnexpectedTestFile() {
|
||||
const awareOfSharding = [
|
||||
'FilestoreApp.js',
|
||||
'FilestoreTests.js',
|
||||
'TestConfig.js',
|
||||
'TestHelper.js',
|
||||
]
|
||||
for (const file of fs.readdirSync(__dirname).sort()) {
|
||||
if (!awareOfSharding.includes(file)) {
|
||||
throw new Error(
|
||||
`Found new test file ${file}: All tests must be aware of the SHARD_ prefix.`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkForUnexpectedTestFile()
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -35,7 +35,7 @@ services:
|
|||
gcs:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -28,7 +28,7 @@ services:
|
|||
mongo:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -33,7 +33,7 @@ services:
|
|||
redis:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -31,7 +31,7 @@ services:
|
|||
redis:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
|
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
MOCHA_GREP=${MOCHA_GREP} \
|
||||
docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
clean:
|
||||
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
|
|
@ -25,7 +25,7 @@ services:
|
|||
NODE_ENV: test
|
||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||
user: node
|
||||
command: npm run test:acceptance:_run
|
||||
command: npm run test:acceptance
|
||||
|
||||
|
||||
tar:
|
||||
|
|
Loading…
Reference in a new issue