From 1d95137e13cf601832781f990b96d33ec2ffe00e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 17 May 2021 11:37:23 +0200 Subject: [PATCH] Merge pull request #4016 from overleaf/jpa-faster-ci-4288 [perf] faster CI GitOrigin-RevId: 858c0c4947e41961dae35cb20e0fcc2e2f723787 --- services/web/Makefile | 25 +++++++++++++++++++ services/web/Makefile.module | 6 +++++ .../acceptance/config/settings.test.coffee | 3 +++ .../test/acceptance/src/helpers/InitApp.js | 2 ++ .../acceptance/src/helpers/MongoHelper.js | 8 +++++- .../acceptance/src/helpers/RedisHelper.js | 11 ++++++++ .../web/test/acceptance/src/helpers/User.js | 2 +- .../test/acceptance/src/helpers/UserHelper.js | 2 +- 8 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 services/web/test/acceptance/src/helpers/RedisHelper.js diff --git a/services/web/Makefile b/services/web/Makefile index 5fb1649c8e..c13816c487 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -141,6 +141,30 @@ clean_ci: clean_test_acceptance_modules test_acceptance_module: $(MAKE) modules/$(MODULE_NAME)/test_acceptance +TEST_ACCEPTANCE_MODULES_MERGED_INNER = $(MODULE_DIRS:=/test_acceptance_merged_inner) +$(TEST_ACCEPTANCE_MODULES_MERGED_INNER): %/test_acceptance_merged_inner: %/Makefile +test_acceptance_modules_merged_inner: $(TEST_ACCEPTANCE_MODULES_MERGED_INNER) + +test_acceptance_modules_merged: export COMPOSE_PROJECT_NAME = \ + acceptance_test_modules_merged_$(BUILD_DIR_NAME) +test_acceptance_modules_merged: + $(DOCKER_COMPOSE) down -v -t 0 + $(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_app_merged_inner: + npm run --silent test:acceptance:app + +test_acceptance_merged_inner: test_acceptance_app_merged_inner +test_acceptance_merged_inner: test_acceptance_modules_merged_inner + +test_acceptance_merged: export COMPOSE_PROJECT_NAME = \ + acceptance_test_merged_$(BUILD_DIR_NAME) +test_acceptance_merged: + $(DOCKER_COMPOSE) down -v -t 0 + $(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_merged_inner + $(DOCKER_COMPOSE) down -v -t 0 + # # CI tests # @@ -307,6 +331,7 @@ tar: MODULE_TARGETS = \ $(TEST_ACCEPTANCE_MODULES) \ + $(TEST_ACCEPTANCE_MODULES_MERGED_INNER) \ $(CLEAN_TEST_ACCEPTANCE_MODULES) \ $(TEST_UNIT_MODULES) \ diff --git a/services/web/Makefile.module b/services/web/Makefile.module index 5555e94e62..e3d12400ee 100644 --- a/services/web/Makefile.module +++ b/services/web/Makefile.module @@ -33,6 +33,7 @@ endif ifeq (,$(wildcard test/acceptance)) test_acceptance: +test_acceptance_merged_inner: clean_test_acceptance: else @@ -41,6 +42,11 @@ test_acceptance: ${DOCKER_COMPOSE_TEST_ACCEPTANCE} run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/acceptance/src $(MAKE) --no-print-directory clean_test_acceptance +test_acceptance_merged_inner: + cd ../../ && \ + SHARELATEX_CONFIG=/app/$(MODULE_DIR)/test/acceptance/config/settings.test.coffee \ + npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/acceptance/src + clean_test_acceptance: ${DOCKER_COMPOSE_TEST_ACCEPTANCE} down -v -t 0 diff --git a/services/web/test/acceptance/config/settings.test.coffee b/services/web/test/acceptance/config/settings.test.coffee index b7502d9899..a177695c6f 100644 --- a/services/web/test/acceptance/config/settings.test.coffee +++ b/services/web/test/acceptance/config/settings.test.coffee @@ -192,3 +192,6 @@ module.exports = unsupportedBrowsers: ie: '<=11' + + test: + counterInit: 0 diff --git a/services/web/test/acceptance/src/helpers/InitApp.js b/services/web/test/acceptance/src/helpers/InitApp.js index 2c55e38b3b..06275d0d8f 100644 --- a/services/web/test/acceptance/src/helpers/InitApp.js +++ b/services/web/test/acceptance/src/helpers/InitApp.js @@ -1,10 +1,12 @@ const App = require('../../../../app.js') const MongoHelper = require('./MongoHelper') +const RedisHelper = require('./RedisHelper') const { logger } = require('logger-sharelatex') logger.level('error') MongoHelper.initialize() +RedisHelper.initialize() let server diff --git a/services/web/test/acceptance/src/helpers/MongoHelper.js b/services/web/test/acceptance/src/helpers/MongoHelper.js index 0e010e9d41..9c2a0505cc 100644 --- a/services/web/test/acceptance/src/helpers/MongoHelper.js +++ b/services/web/test/acceptance/src/helpers/MongoHelper.js @@ -18,7 +18,13 @@ module.exports = { afterEach(async function () { return Promise.all( - Object.values(db).map(collection => collection.deleteMany({})) + Object.values(db).map(async collection => { + if (collection === db.migrations) { + // Do not clear the collection for tracking migrations. + return + } + return collection.deleteMany({}) + }) ) }) }, diff --git a/services/web/test/acceptance/src/helpers/RedisHelper.js b/services/web/test/acceptance/src/helpers/RedisHelper.js new file mode 100644 index 0000000000..2960577b6f --- /dev/null +++ b/services/web/test/acceptance/src/helpers/RedisHelper.js @@ -0,0 +1,11 @@ +const RateLimiter = require('../../../../app/src/infrastructure/RateLimiter') + +async function clearOverleafLoginRateLimit() { + await RateLimiter.promises.clearRateLimit('overleaf-login', '127.0.0.1') +} + +module.exports = { + initialize() { + before(clearOverleafLoginRateLimit) + }, +} diff --git a/services/web/test/acceptance/src/helpers/User.js b/services/web/test/acceptance/src/helpers/User.js index fa0f4283a4..7811a2de36 100644 --- a/services/web/test/acceptance/src/helpers/User.js +++ b/services/web/test/acceptance/src/helpers/User.js @@ -8,7 +8,7 @@ const { promisify } = require('util') const fs = require('fs') const Path = require('path') -let count = 0 +let count = settings.test.counterInit class User { constructor(options) { diff --git a/services/web/test/acceptance/src/helpers/UserHelper.js b/services/web/test/acceptance/src/helpers/UserHelper.js index a7b3e06f2b..8d2f8b313b 100644 --- a/services/web/test/acceptance/src/helpers/UserHelper.js +++ b/services/web/test/acceptance/src/helpers/UserHelper.js @@ -9,7 +9,7 @@ const request = require('request-promise-native') const { db } = require('../../../../app/src/infrastructure/mongodb') const { ObjectId } = require('mongodb') -let globalUserNum = 1 +let globalUserNum = Settings.test.counterInit class UserHelper { /**