From 411a12cb2db73fa0f68384d870ace365cb186d70 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 20 Jul 2021 11:26:03 +0200 Subject: [PATCH] Merge pull request #4322 from overleaf/jpa-custom-base-config-for-tests [misc] set up custom base config for acceptance tests GitOrigin-RevId: bd49c705360488fd75c6ad311afd13f22f81de61 --- services/web/Makefile | 128 ++++++++++++++++-- services/web/Makefile.module | 23 +++- services/web/app/src/router.js | 6 + services/web/config/settings.defaults.js | 2 + services/web/docker-compose.ci.yml | 3 + services/web/docker-compose.yml | 3 + .../test/acceptance/config/settings.test.js | 6 +- ...settings.test.js => settings.test.saas.js} | 0 .../config/settings.test.server-ce.js | 11 ++ .../config/settings.test.server-pro.js | 7 + .../web/test/acceptance/getModuleTargets.js | 8 ++ 11 files changed, 182 insertions(+), 15 deletions(-) rename services/web/test/acceptance/config/{settings.test.js => settings.test.saas.js} (100%) create mode 100644 services/web/test/acceptance/config/settings.test.server-ce.js create mode 100644 services/web/test/acceptance/config/settings.test.server-pro.js create mode 100644 services/web/test/acceptance/getModuleTargets.js diff --git a/services/web/Makefile b/services/web/Makefile index 1ac205d387..e5baae08c7 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -5,11 +5,17 @@ BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = web BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') +export SHARELATEX_CONFIG ?= /app/test/acceptance/config/settings.test.saas.js +export BASE_CONFIG ?= ${SHARELATEX_CONFIG} + +CFG_SAAS=/app/test/acceptance/config/settings.test.saas.js +CFG_SERVER_CE=/app/test/acceptance/config/settings.test.server-ce.js +CFG_SERVER_PRO=/app/test/acceptance/config/settings.test.server-pro.js + DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ PROJECT_NAME=$(PROJECT_NAME) \ MOCHA_GREP=${MOCHA_GREP} \ - SHARELATEX_CONFIG=/app/test/acceptance/config/settings.test.js \ docker-compose ${DOCKER_COMPOSE_FLAGS} MODULE_DIRS := $(shell find modules -mindepth 1 -maxdepth 1 -type d -not -name '.git' ) @@ -129,29 +135,131 @@ test_acceptance_app: COMPOSE_PROJECT_NAME=acceptance_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --rm test_acceptance COMPOSE_PROJECT_NAME=acceptance_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 +# We are using _make magic_ for turning these file-targets into calls to +# sub-Makefiles in the individual modules. +# These sub-Makefiles need to be kept in sync with the template, hence we +# add a dependency on each modules Makefile and cross-link that to the +# template at the very top of this file. +# Example: `web$ make modules/server-ce-scripts/test_acceptance_server_ce` +# Description: Run the acceptance tests of the server-ce-scripts module in a +# Server CE Environment. +# Break down: +# Target: modules/server-ce-scripts/test_acceptance_server_ce +# -> depends on modules/server-ce-scripts/Makefile +# -> add environment variable BASE_CONFIG=$(CFG_SERVER_CE) +# -> BASE_CONFIG=/app/test/acceptance/config/settings.test.server-ce.js +# -> automatic target: `make -C server-ce-scripts test_acceptance_server_ce` +# -> automatic target: run `make test_acceptance_server_ce` in module +# Target: modules/server-ce-scripts/Makefile +# -> depends on Makefile.module +# -> automatic target: copies the file when changed TEST_ACCEPTANCE_MODULES = $(MODULE_DIRS:=/test_acceptance) $(TEST_ACCEPTANCE_MODULES): %/test_acceptance: %/Makefile -test_acceptance_modules: $(TEST_ACCEPTANCE_MODULES) +$(TEST_ACCEPTANCE_MODULES): modules/%/test_acceptance: + $(MAKE) test_acceptance_module MODULE_NAME=$* + +TEST_ACCEPTANCE_MODULES_SAAS = $(MODULE_DIRS:=/test_acceptance_saas) +$(TEST_ACCEPTANCE_MODULES_SAAS): %/test_acceptance_saas: %/Makefile +$(TEST_ACCEPTANCE_MODULES_SAAS): export BASE_CONFIG = $(CFG_SAAS) + +# This line adds `/test_acceptance_saas` suffix to all items in $(MODULE_DIRS). +TEST_ACCEPTANCE_MODULES_SERVER_CE = $(MODULE_DIRS:=/test_acceptance_server_ce) +# This line adds a dependency on the modules Makefile. +$(TEST_ACCEPTANCE_MODULES_SERVER_CE): %/test_acceptance_server_ce: %/Makefile +# This line adds the environment variable BASE_CONFIG=$(CFG_SERVER_CE) to all +# invocations of `web$ make modules/foo/test_acceptance_server_ce`. +$(TEST_ACCEPTANCE_MODULES_SERVER_CE): export BASE_CONFIG = $(CFG_SERVER_CE) + +TEST_ACCEPTANCE_MODULES_SERVER_PRO = $(MODULE_DIRS:=/test_acceptance_server_pro) +$(TEST_ACCEPTANCE_MODULES_SERVER_PRO): %/test_acceptance_server_pro: %/Makefile +$(TEST_ACCEPTANCE_MODULES_SERVER_PRO): export BASE_CONFIG = $(CFG_SERVER_PRO) CLEAN_TEST_ACCEPTANCE_MODULES = $(MODULE_DIRS:=/clean_test_acceptance) $(CLEAN_TEST_ACCEPTANCE_MODULES): %/clean_test_acceptance: %/Makefile clean_test_acceptance_modules: $(CLEAN_TEST_ACCEPTANCE_MODULES) clean_ci: clean_test_acceptance_modules -test_acceptance_module: - $(MAKE) modules/$(MODULE_NAME)/test_acceptance +test_acceptance_module_noop: + @echo + @echo Module '$(MODULE_NAME)' does not run in ${LABEL}. + @echo +TEST_ACCEPTANCE_MODULE_MAYBE_IN := \ + test_acceptance_module_maybe_in_saas \ + test_acceptance_module_maybe_in_server_ce \ + test_acceptance_module_maybe_in_server_pro \ + +test_acceptance_module: $(TEST_ACCEPTANCE_MODULE_MAYBE_IN) +test_acceptance_module_maybe_in_saas: export BASE_CONFIG=$(CFG_SAAS) +test_acceptance_module_maybe_in_server_ce: export BASE_CONFIG=$(CFG_SERVER_CE) +test_acceptance_module_maybe_in_server_pro: export BASE_CONFIG=$(CFG_SERVER_PRO) + +# We need to figure out whether the module is loaded in a given environment. +# This information is stored in the (base-)settings. +# We get the full list of modules and check for a matching module entry. +# Either the grep will find and emit the module, or exits with code 1, which +# we handle with a fallback to a noop make target. +# Run the node command in a docker-compose container which provides the needed +# npm dependencies (from disk in dev-env or from the CI image in CI). +# Pick the test_unit service which is very light-weight -- the test_acceptance +# service would start mongo/redis. +$(TEST_ACCEPTANCE_MODULE_MAYBE_IN): test_acceptance_module_maybe_in_%: + $(MAKE) $(shell \ + SHARELATEX_CONFIG=$(BASE_CONFIG) \ + $(DOCKER_COMPOSE) run --rm test_unit \ + node test/acceptance/getModuleTargets test_acceptance_$* \ + | grep -e /$(MODULE_NAME)/ || echo test_acceptance_module_noop LABEL=$* \ + ) + +# See docs for test_acceptance_server_ce how this works. +test_acceptance_module_saas: export BASE_CONFIG = $(CFG_SAAS) +test_acceptance_module_saas: + $(MAKE) modules/$(MODULE_NAME)/test_acceptance_saas + +test_acceptance_module_server_ce: export BASE_CONFIG = $(CFG_SERVER_CE) +test_acceptance_module_server_ce: + $(MAKE) modules/$(MODULE_NAME)/test_acceptance_server_ce + +test_acceptance_module_server_pro: export BASE_CONFIG = $(CFG_SERVER_PRO) +test_acceptance_module_server_pro: + $(MAKE) modules/$(MODULE_NAME)/test_acceptance_server_pro + +# See docs for test_acceptance_server_ce how this works. TEST_ACCEPTANCE_MODULES_MERGED_INNER = $(MODULE_DIRS:=/test_acceptance_merged_inner) $(TEST_ACCEPTANCE_MODULES_MERGED_INNER): %/test_acceptance_merged_inner: %/Makefile -test_acceptance_modules_merged_inner: $(TEST_ACCEPTANCE_MODULES_MERGED_INNER) +test_acceptance_modules_merged_inner: + $(MAKE) $(shell \ + SHARELATEX_CONFIG=$(BASE_CONFIG) \ + node test/acceptance/getModuleTargets test_acceptance_merged_inner \ + ) -test_acceptance_modules_merged: export COMPOSE_PROJECT_NAME = \ - acceptance_test_modules_merged_$(BUILD_DIR_NAME) -test_acceptance_modules_merged: +# See docs for test_acceptance_server_ce how this works. +test_acceptance_modules_merged_saas: export COMPOSE_PROJECT_NAME = \ + acceptance_test_modules_merged_saas_$(BUILD_DIR_NAME) +test_acceptance_modules_merged_saas: export BASE_CONFIG = $(CFG_SAAS) + +test_acceptance_modules_merged_server_ce: export COMPOSE_PROJECT_NAME = \ + acceptance_test_modules_merged_server_ce_$(BUILD_DIR_NAME) +test_acceptance_modules_merged_server_ce: export BASE_CONFIG = $(CFG_SERVER_CE) + +test_acceptance_modules_merged_server_pro: export COMPOSE_PROJECT_NAME = \ + acceptance_test_modules_merged_server_pro_$(BUILD_DIR_NAME) +test_acceptance_modules_merged_server_pro: export BASE_CONFIG = $(CFG_SERVER_PRO) + +# All these variants run the same command. +# Each target has a different set of environment defined above. +TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS = \ + test_acceptance_modules_merged_saas \ + test_acceptance_modules_merged_server_ce \ + test_acceptance_modules_merged_server_pro \ + +$(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS): $(DOCKER_COMPOSE) down -v -t 0 $(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner $(DOCKER_COMPOSE) down -v -t 0 +test_acceptance_modules: $(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS) + test_acceptance_app_merged_inner: npm run --silent test:acceptance:app @@ -330,7 +438,9 @@ tar: COMPOSE_PROJECT_NAME=tar_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0 MODULE_TARGETS = \ - $(TEST_ACCEPTANCE_MODULES) \ + $(TEST_ACCEPTANCE_MODULES_SAAS) \ + $(TEST_ACCEPTANCE_MODULES_SERVER_CE) \ + $(TEST_ACCEPTANCE_MODULES_SERVER_PRO) \ $(TEST_ACCEPTANCE_MODULES_MERGED_INNER) \ $(CLEAN_TEST_ACCEPTANCE_MODULES) \ $(TEST_UNIT_MODULES) \ diff --git a/services/web/Makefile.module b/services/web/Makefile.module index f211cbf3fd..37915e9426 100644 --- a/services/web/Makefile.module +++ b/services/web/Makefile.module @@ -3,6 +3,13 @@ MODULE_NAME := $(notdir $(shell pwd)) MODULE_DIR := modules/$(MODULE_NAME) PROJECT_NAME = web +export SHARELATEX_CONFIG = /app/$(MODULE_DIR)/test/acceptance/config/settings.test.js +export BASE_CONFIG ?= /app/test/acceptance/config/settings.test.saas.js + +CFG_SAAS=/app/test/acceptance/config/settings.test.saas.js +CFG_SERVER_CE=/app/test/acceptance/config/settings.test.server-ce.js +CFG_SERVER_PRO=/app/test/acceptance/config/settings.test.server-pro.js + DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml --log-level ERROR DOCKER_COMPOSE := cd ../../ && \ MODULE_DIR=$(MODULE_DIR) \ @@ -10,7 +17,6 @@ DOCKER_COMPOSE := cd ../../ && \ BRANCH_NAME=$(BRANCH_NAME) \ PROJECT_NAME=$(PROJECT_NAME) \ MOCHA_GREP=${MOCHA_GREP} \ - SHARELATEX_CONFIG=/app/$(MODULE_DIR)/test/acceptance/config/settings.test.js \ docker-compose ${DOCKER_COMPOSE_FLAGS} DOCKER_COMPOSE_TEST_ACCEPTANCE := \ @@ -31,20 +37,29 @@ test_unit: endif +ALL_TEST_ACCEPTANCE_VARIANTS := \ + test_acceptance \ + test_acceptance_saas \ + test_acceptance_server_ce \ + test_acceptance_server_pro \ + ifeq (,$(wildcard test/acceptance)) -test_acceptance: +$(ALL_TEST_ACCEPTANCE_VARIANTS): test_acceptance_merged_inner: clean_test_acceptance: else -test_acceptance: +test_acceptance_saas: export BASE_CONFIG = $(CFG_SAAS) +test_acceptance_server_ce: export BASE_CONFIG = $(CFG_SERVER_CE) +test_acceptance_server_pro: export BASE_CONFIG = $(CFG_SERVER_PRO) + +$(ALL_TEST_ACCEPTANCE_VARIANTS): $(MAKE) --no-print-directory clean_test_acceptance ${DOCKER_COMPOSE_TEST_ACCEPTANCE} run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/acceptance/src $(MAKE) --no-print-directory clean_test_acceptance test_acceptance_merged_inner: cd ../../ && \ - SHARELATEX_CONFIG=/app/$(MODULE_DIR)/test/acceptance/config/settings.test.js \ npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/acceptance/src clean_test_acceptance: diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index 5e9fb631df..4bd030e3fd 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -72,6 +72,12 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { webRouter.post('/login', AuthenticationController.passportLogin) + if (Settings.enableLegacyLogin) { + AuthenticationController.addEndpointToLoginWhitelist('/login/legacy') + webRouter.get('/login/legacy', UserPagesController.loginPage) + webRouter.post('/login/legacy', AuthenticationController.passportLogin) + } + webRouter.get( '/read-only/one-time-login', UserPagesController.oneTimeLoginPage diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index eb9c85125c..d5ca69b101 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -72,6 +72,8 @@ const parseTextExtensions = function (extensions) { } module.exports = { + env: 'server-ce', + limits: { httpGlobalAgentMaxSockets: 300, httpsGlobalAgentMaxSockets: 300, diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml index e1babb47d7..18b0b0bc01 100644 --- a/services/web/docker-compose.ci.yml +++ b/services/web/docker-compose.ci.yml @@ -11,6 +11,8 @@ services: user: node command: npm run test:unit:app environment: + BASE_CONFIG: + SHARELATEX_CONFIG: NODE_OPTIONS: "--unhandled-rejections=strict" test_acceptance: @@ -29,6 +31,7 @@ services: LOCK_MANAGER_MAX_LOCK_WAIT_TIME: 30000 NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + BASE_CONFIG: SHARELATEX_CONFIG: COOKIE_DOMAIN: .overleaf.test PUBLIC_URL: 'http://www.overleaf.test:3000' diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index 84437fd663..be20926444 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -13,6 +13,8 @@ services: - .:/app working_dir: /app environment: + BASE_CONFIG: + SHARELATEX_CONFIG: MOCHA_GREP: ${MOCHA_GREP} NODE_OPTIONS: "--unhandled-rejections=strict" command: npm run --silent test:unit:app @@ -39,6 +41,7 @@ services: MONGO_SERVER_SELECTION_TIMEOUT: 600000 MONGO_SOCKET_TIMEOUT: 300000 # SHARELATEX_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING: 'true' + BASE_CONFIG: SHARELATEX_CONFIG: COOKIE_DOMAIN: .overleaf.test PUBLIC_URL: 'http://www.overleaf.test:3000' diff --git a/services/web/modules/server-ce-scripts/test/acceptance/config/settings.test.js b/services/web/modules/server-ce-scripts/test/acceptance/config/settings.test.js index 7f62aa602d..f52786a207 100644 --- a/services/web/modules/server-ce-scripts/test/acceptance/config/settings.test.js +++ b/services/web/modules/server-ce-scripts/test/acceptance/config/settings.test.js @@ -1,5 +1,7 @@ -module.exports = { +const base = require(process.env.BASE_CONFIG) + +module.exports = base.mergeWith({ test: { counterInit: 160000, }, -} +}) diff --git a/services/web/test/acceptance/config/settings.test.js b/services/web/test/acceptance/config/settings.test.saas.js similarity index 100% rename from services/web/test/acceptance/config/settings.test.js rename to services/web/test/acceptance/config/settings.test.saas.js diff --git a/services/web/test/acceptance/config/settings.test.server-ce.js b/services/web/test/acceptance/config/settings.test.server-ce.js new file mode 100644 index 0000000000..9f309ab61a --- /dev/null +++ b/services/web/test/acceptance/config/settings.test.server-ce.js @@ -0,0 +1,11 @@ +const { merge } = require('@overleaf/settings/merge') + +module.exports = { + test: { + counterInit: 0, + }, +} + +module.exports.mergeWith = function (overrides) { + return merge(overrides, module.exports) +} diff --git a/services/web/test/acceptance/config/settings.test.server-pro.js b/services/web/test/acceptance/config/settings.test.server-pro.js new file mode 100644 index 0000000000..9ab298ae2d --- /dev/null +++ b/services/web/test/acceptance/config/settings.test.server-pro.js @@ -0,0 +1,7 @@ +const base = require('../../../config/settings.overrides.server-pro') + +module.exports = base.mergeWith({ + test: { + counterInit: 0, + }, +}) diff --git a/services/web/test/acceptance/getModuleTargets.js b/services/web/test/acceptance/getModuleTargets.js new file mode 100644 index 0000000000..eacbff3f89 --- /dev/null +++ b/services/web/test/acceptance/getModuleTargets.js @@ -0,0 +1,8 @@ +// silence settings module +console.log = function () {} +const Settings = require('@overleaf/settings') + +const MODULES = Settings.moduleImportSequence +const TARGET = process.argv.slice(2).pop() || 'test_acceptance' + +console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))