From fbf8cc2d0303bf46dc664a2ec8f9fd4cf411d4f9 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 23 Nov 2017 16:01:32 +0000 Subject: [PATCH] Run acceptance tests via docker compose --- services/web/Jenkinsfile | 11 ++-- services/web/config/settings.defaults.coffee | 15 +++-- services/web/docker-compose.ci.yml | 12 ++++ services/web/docker-compose.yml | 42 ++++++++++++ services/web/makefile | 64 +++++++++++++++++++ services/web/package.json | 11 ++++ .../coffee/helpers/MockDocUpdaterApi.coffee | 2 +- .../coffee/helpers/MockDocstoreApi.coffee | 2 +- .../acceptance/coffee/helpers/redis.coffee | 1 - .../acceptance/coffee/helpers/request.coffee | 2 +- 10 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 services/web/docker-compose.ci.yml create mode 100644 services/web/docker-compose.yml create mode 100644 services/web/makefile diff --git a/services/web/Jenkinsfile b/services/web/Jenkinsfile index 33aa807fdd..85ecafef58 100644 --- a/services/web/Jenkinsfile +++ b/services/web/Jenkinsfile @@ -117,19 +117,16 @@ pipeline { } } steps { + sh 'make install' + sh 'make test_unit MOCHA_ARGS="--reporter=tap"' sh 'env NODE_ENV=development ./node_modules/.bin/grunt mochaTest:unit --reporter=tap' } } stage('Acceptance Tests') { steps { - // This tagged relase of the acceptance test runner is a temporary fix - // to get the acceptance tests working before we move to a - // docker-compose workflow. See: - // https://github.com/sharelatex/web-sharelatex-internal/pull/148 - - sh 'docker pull sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4' - sh 'docker run --rm -v $(pwd):/app --env SHARELATEX_ALLOW_PUBLIC_ACCESS=true sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4 || (cat forever/app.log && false)' + sh 'make install' + sh "make test_acceptance MOCHA_ARGS="--reporter=tap"' } } diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee index 69bf0a3b7c..fccd197a83 100644 --- a/services/web/config/settings.defaults.coffee +++ b/services/web/config/settings.defaults.coffee @@ -35,12 +35,12 @@ module.exports = settings = # Databases # --------- mongo: - url : 'mongodb://127.0.0.1/sharelatex' + url : "mongodb://#{process.env['MONGO_HOST'] || '127.0.0.1'}/sharelatex" redis: web: - host: "localhost" - port: "6379" + host: process.env['REDIS_HOST'] || "localhost" + port: process.env['REDIS_PORT'] || "6379" password: "" # websessions: @@ -74,8 +74,8 @@ module.exports = settings = # ] api: - host: "localhost" - port: "6379" + host: process.env['REDIS_HOST'] || "localhost" + port: process.env['REDIS_PORT'] || "6379" password: "" # Service locations @@ -87,6 +87,7 @@ module.exports = settings = internal: web: port: webPort = 3000 + host: process.env['LISTEN_ADDRESS'] or 'localhost' documentupdater: port: docUpdaterPort = 3003 @@ -99,7 +100,7 @@ module.exports = settings = user: httpAuthUser pass: httpAuthPass documentupdater: - url : "http://localhost:#{docUpdaterPort}" + url : "http://#{process.env['DOCUPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}" thirdPartyDataStore: url : "http://localhost:3002" emptyProjectFlushDelayMiliseconds: 5 * seconds @@ -113,7 +114,7 @@ module.exports = settings = enabled: false url : "http://localhost:3054" docstore: - url : "http://localhost:3016" + url : "http://#{process.env['DOCSTORE_HOST'] or 'localhost'}:3016" pubUrl: "http://localhost:3016" chat: url: "http://localhost:3010" diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml new file mode 100644 index 0000000000..0bb7af5f74 --- /dev/null +++ b/services/web/docker-compose.ci.yml @@ -0,0 +1,12 @@ +version: "2" + +services: + test_unit: + image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: root + volumes: [] + + test_acceptance: + image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: root + volumes: [] diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml new file mode 100644 index 0000000000..fe30bb8f57 --- /dev/null +++ b/services/web/docker-compose.yml @@ -0,0 +1,42 @@ +version: "2" + +volumes: + node_modules: + +services: + npm: + image: node:6.9.5 + volumes: + - .:/app + - node_modules:/app/node_modules + working_dir: /app + + test_unit: + image: node:6.9.5 + volumes: + - .:/app + - node_modules:/app/node_modules + working_dir: /app + command: npm run test:unit + + test_acceptance: + image: node:6.9.5 + volumes: + - .:/app + - node_modules:/app/node_modules + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true' + LISTEN_ADDRESS: 0.0.0.0 + depends_on: + - redis + - mongo + working_dir: /app + command: npm run start + + redis: + image: redis + + mongo: + image: mongo:3.4.6 diff --git a/services/web/makefile b/services/web/makefile new file mode 100644 index 0000000000..c6e1516598 --- /dev/null +++ b/services/web/makefile @@ -0,0 +1,64 @@ +NPM := docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} run --rm npm npm +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = web + +all: install test + @echo "Run:" + @echo " make install to set up the project dependencies (in docker)" + @echo " make test to run all the tests for the project (in docker)" + @echo " make run to run the app (natively)" + +add: + $(NPM) install --save ${P} + +add_dev: + $(NPM) install --save-dev ${P} + +install: + $(NPM) install + +clean: + rm app.js + rm -r app/js + rm -r test/unit/js + rm -r test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} run --rm test_unit npm run test:unit -- ${MOCHA_ARGS} + +test_acceptance: test_acceptance_start_service test_acceptance_run test_acceptance_stop_service + +test_acceptance_start_service: + docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} up -d test_acceptance + +test_acceptance_stop_service: + docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} stop test_acceptance + +test_acceptance_run: + docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} exec -T test_acceptance npm run test:acceptance -- ${MOCHA_ARGS} + +build: + docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + +ci: + # When we run the tests locally we mount the local directory as a volumne + # and use a persistent node_modules folder (see docker-compose.yml). + # However, on the CI server, we want to run our tests in the image that we + # have built for deployment, which is what the docker-compose.ci.yml + # override does. + PROJECT_NAME=$(PROJECT_NAME) \ + BRANCH_NAME=$(BRANCH_NAME) \ + BUILD_NUMBER=$(BUILD_NUMBER) \ + DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" \ + $(MAKE) build test publish + +.PHONY: + add install update test test_unit test_acceptance \ + test_acceptance_start_service test_acceptance_stop_service \ + test_acceptance_run build publish ci diff --git a/services/web/package.json b/services/web/package.json index e831d929e1..f4ea19a49d 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -9,6 +9,17 @@ "directories": { "public": "./public" }, + "scripts": { + "test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3000/status) do sleep 1; done", + "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js", + "test:acceptance": "npm run compile:acceptance_tests && npm run test:acceptance:wait_for_app && npm run test:acceptance:run -- $@", + "test:unit": "npm run compile:app && npm run compile:unit_tests && mocha --recursive --reporter spec $@ test/unit/js", + "compile:unit_tests": "coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "coffee -o test/acceptance/js -c test/acceptance/coffee", + "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", + "start": "npm run compile:app && node app.js", + "echo": "echo $@" + }, "dependencies": { "archiver": "0.9.0", "async": "0.6.2", diff --git a/services/web/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee b/services/web/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee index aefcd4513a..6f307b0810 100644 --- a/services/web/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee @@ -6,7 +6,7 @@ module.exports = MockDocUpdaterApi = app.post "/project/:project_id/flush", (req, res, next) => res.sendStatus 200 - app.listen 3003, (error) -> + app.listen 3003, '0.0.0.0', (error) -> throw error if error? .on "error", (error) -> console.error "error starting MockDocUpdaterApi:", error.message diff --git a/services/web/test/acceptance/coffee/helpers/MockDocstoreApi.coffee b/services/web/test/acceptance/coffee/helpers/MockDocstoreApi.coffee index 2133d40b9f..7c8f5dbe50 100644 --- a/services/web/test/acceptance/coffee/helpers/MockDocstoreApi.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockDocstoreApi.coffee @@ -22,7 +22,7 @@ module.exports = MockDocStoreApi = docs = (doc for doc_id, doc of @docs[req.params.project_id]) res.send JSON.stringify docs - app.listen 3016, (error) -> + app.listen 3016, '0.0.0.0', (error) -> throw error if error? .on "error", (error) -> console.error "error starting MockDocStoreApi:", error.message diff --git a/services/web/test/acceptance/coffee/helpers/redis.coffee b/services/web/test/acceptance/coffee/helpers/redis.coffee index 9aecf6b387..7c48f97d2e 100644 --- a/services/web/test/acceptance/coffee/helpers/redis.coffee +++ b/services/web/test/acceptance/coffee/helpers/redis.coffee @@ -1,5 +1,4 @@ Settings = require('settings-sharelatex') -redis = require('redis-sharelatex') logger = require("logger-sharelatex") Async = require('async') diff --git a/services/web/test/acceptance/coffee/helpers/request.coffee b/services/web/test/acceptance/coffee/helpers/request.coffee index 879acd843a..1c7120d141 100644 --- a/services/web/test/acceptance/coffee/helpers/request.coffee +++ b/services/web/test/acceptance/coffee/helpers/request.coffee @@ -1,4 +1,4 @@ -BASE_URL = "http://localhost:3000" +BASE_URL = "http://#{process.env["HTTP_TEST_HOST"] or "localhost"}:3000" module.exports = require("request").defaults({ baseUrl: BASE_URL, followRedirect: false