diff --git a/services/web/.gitignore b/services/web/.gitignore index 2a29f414d1..a48481690a 100644 --- a/services/web/.gitignore +++ b/services/web/.gitignore @@ -73,3 +73,4 @@ Gemfile.lock app/views/external /modules/ +docker-shared.yml diff --git a/services/web/Jenkinsfile b/services/web/Jenkinsfile index 33aa807fdd..0650eb4edb 100644 --- a/services/web/Jenkinsfile +++ b/services/web/Jenkinsfile @@ -109,27 +109,16 @@ pipeline { } } - stage('Unit Test') { - agent { - docker { - image 'node:6.9.5' - reuseNode true - } - } + stage('Unit Tests') { steps { - sh 'env NODE_ENV=development ./node_modules/.bin/grunt mochaTest:unit --reporter=tap' + sh 'make install' + sh 'make test_unit MOCHA_ARGS="--reporter=tap"' } } stage('Acceptance Tests') { steps { - // This tagged relase of the acceptance test runner is a temporary fix - // to get the acceptance tests working before we move to a - // docker-compose workflow. See: - // https://github.com/sharelatex/web-sharelatex-internal/pull/148 - - sh 'docker pull sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4' - sh 'docker run --rm -v $(pwd):/app --env SHARELATEX_ALLOW_PUBLIC_ACCESS=true sharelatex/sl-acceptance-test-runner:node-6.9-mongo-3.4 || (cat forever/app.log && false)' + sh 'make test_acceptance MOCHA_ARGS="--reporter=tap"' } } diff --git a/services/web/Makefile b/services/web/Makefile new file mode 100644 index 0000000000..7ab8b89781 --- /dev/null +++ b/services/web/Makefile @@ -0,0 +1,75 @@ +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +NPM := docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm npm npm +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = web + +all: install test + @echo "Run:" + @echo " make install to set up the project dependencies (in docker)" + @echo " make test to run all the tests for the project (in docker)" + +add: docker-shared.yml + $(NPM) install --save ${P} + +add_dev: docker-shared.yml + $(NPM) install --save-dev ${P} + +install: docker-shared.yml + $(NPM) install + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + for dir in modules/*; \ + do \ + rm -f $$dir/index.js; \ + rm -rf $$dir/app/js; \ + rm -rf $$dir/test/unit/js; \ + rm -rf $$dir/test/acceptance/js; \ + done + # Deletes node_modules volume + docker-compose down --volumes + # Regenerate docker-shared.yml - not stictly a 'clean', + # but lets `make clean install` work nicely + bin/generate_volumes_file + +# Need regenerating if you change the web modules you have installed +docker-shared.yml: + bin/generate_volumes_file + +test: test_unit test_acceptance + +test_unit: docker-shared.yml + docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm test_unit npm run test:unit -- ${MOCHA_ARGS} + +test_acceptance: test_acceptance_app test_acceptance_modules + +test_acceptance_app: test_acceptance_app_start_service test_acceptance_app_run test_acceptance_app_stop_service + +test_acceptance_app_start_service: test_acceptance_app_stop_service docker-shared.yml + docker-compose ${DOCKER_COMPOSE_FLAGS} up -d test_acceptance + +test_acceptance_app_stop_service: docker-shared.yml + docker-compose ${DOCKER_COMPOSE_FLAGS} stop test_acceptance redis mongo + +test_acceptance_app_run: docker-shared.yml + docker-compose ${DOCKER_COMPOSE_FLAGS} exec -T test_acceptance npm run test:acceptance -- ${MOCHA_ARGS} + +test_acceptance_modules: docker-shared.yml + for dir in modules/*; \ + do \ + if [ -e $$dir/makefile ]; then \ + (make test_acceptance_module MODULE=$$dir) \ + fi \ + done + +test_acceptance_module: docker-shared.yml + cd $(MODULE) && make test_acceptance + +.PHONY: + all add install update test test_unit test_acceptance \ + test_acceptance_start_service test_acceptance_stop_service \ + test_acceptance_run diff --git a/services/web/README.md b/services/web/README.md index f777e7e5f5..51f73d02ec 100644 --- a/services/web/README.md +++ b/services/web/README.md @@ -17,6 +17,69 @@ web-sharelatex uses [Grunt](http://gruntjs.com/) to build its front-end related Image processing tasks are commented out in the gruntfile and the needed packages aren't presently in the project's `package.json`. If the images need to be processed again (minified and sprited), start by fetching the packages (`npm install grunt-contrib-imagemin grunt-sprity`), then *decomment* the tasks in `Gruntfile.coffee`. After this, the tasks can be called (explicitly, via `grunt imagemin` and `grunt sprity`). +New Docker-based build process +------------------------------ + +Note that the Grunt workflow from above should still work, but we are transitioning to a +Docker based testing workflow, which is documented below: + +### Running the app + +The app runs natively using npm and Node on the local system: + +``` +$ npm install +$ npm run start +``` + +*Ideally the app would run in Docker like the tests below, but with host networking not supported in OS X, we need to run it natively until all services are Dockerised.* + +### Unit Tests + +The test suites run in Docker. + +Unit tests can be run in the `test_unit` container defined in `docker-compose.tests.yml`. + +The makefile contains a short cut to run these: + +``` +make install # Only needs running once, or when npm packages are updated +make unit_test +``` + +During development it is often useful to only run a subset of tests, which can be configured with arguments to the mocha CLI: + +``` +make unit_test MOCHA_ARGS='--grep=AuthorizationManager' +``` + +### Acceptance Tests + +Acceptance tests are run against a live service, which runs in the `acceptance_test` container defined in `docker-compose.tests.yml`. + +To run the tests out-of-the-box, the makefile defines: + +``` +make install # Only needs running once, or when npm packages are updated +make acceptance_test +``` + +However, during development it is often useful to leave the service running for rapid iteration on the acceptance tests. This can be done with: + +``` +make acceptance_test_start_service +make acceptance_test_run # Run as many times as needed during development +make acceptance_test_stop_service +``` + +`make acceptance_test` just runs these three commands in sequence. + +During development it is often useful to only run a subset of tests, which can be configured with arguments to the mocha CLI: + +``` +make acceptance_test_run MOCHA_ARGS='--grep=AuthorizationManager' +``` + Unit test status ---------------- diff --git a/services/web/bin/acceptance_test b/services/web/bin/acceptance_test new file mode 100755 index 0000000000..fd2e5137b5 --- /dev/null +++ b/services/web/bin/acceptance_test @@ -0,0 +1,4 @@ +#!/bin/bash +set -e; +MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000" +$MOCHA "$@" diff --git a/services/web/bin/compile_acceptance_tests b/services/web/bin/compile_acceptance_tests new file mode 100755 index 0000000000..d60ba0cc46 --- /dev/null +++ b/services/web/bin/compile_acceptance_tests @@ -0,0 +1,16 @@ +#!/bin/bash +set -e; + +COFFEE=node_modules/.bin/coffee + +echo Compiling test/acceptance/coffee; +$COFFEE -o test/acceptance/js -c test/acceptance/coffee; + +for dir in modules/*; +do + + if [ -d $dir/test/acceptance ]; then + echo Compiling $dir/test/acceptance/coffee; + $COFFEE -o $dir/test/acceptance/js -c $dir/test/acceptance/coffee; + fi +done \ No newline at end of file diff --git a/services/web/bin/compile_app b/services/web/bin/compile_app new file mode 100755 index 0000000000..b218b01a5a --- /dev/null +++ b/services/web/bin/compile_app @@ -0,0 +1,23 @@ +#!/bin/bash +set -e; + +COFFEE=node_modules/.bin/coffee + +echo Compiling app.coffee; +$COFFEE -c app.coffee; + +echo Compiling app/coffee; +$COFFEE -o app/js -c app/coffee; + +for dir in modules/*; +do + if [ -d $dir/app/coffee ]; then + echo Compiling $dir/app/coffee; + $COFFEE -o $dir/app/js -c $dir/app/coffee; + fi + + if [ -e $dir/index.coffee ]; then + echo Compiling $dir/index.coffee; + $COFFEE -c $dir/index.coffee; + fi +done \ No newline at end of file diff --git a/services/web/bin/compile_unit_tests b/services/web/bin/compile_unit_tests new file mode 100755 index 0000000000..780a189bda --- /dev/null +++ b/services/web/bin/compile_unit_tests @@ -0,0 +1,15 @@ +#!/bin/bash +set -e; + +COFFEE=node_modules/.bin/coffee + +echo Compiling test/unit/coffee; +$COFFEE -o test/unit/js -c test/unit/coffee; + +for dir in modules/*; +do + if [ -d $dir/test/unit ]; then + echo Compiling $dir/test/unit/coffee; + $COFFEE -o $dir/test/unit/js -c $dir/test/unit/coffee; + fi +done \ No newline at end of file diff --git a/services/web/bin/generate_volumes_file b/services/web/bin/generate_volumes_file new file mode 100755 index 0000000000..d70ac11c3d --- /dev/null +++ b/services/web/bin/generate_volumes_file @@ -0,0 +1,26 @@ +#!/usr/bin/env python2 + +from os import listdir +from os.path import isfile, isdir, join + +volumes = [] + +for module in listdir("modules/"): + if module[0] != '.': + if isfile(join("modules", module, 'index.coffee')): + volumes.append(join("modules", module, 'index.coffee')) + for directory in ['app/coffee', 'app/views', 'public/coffee', 'test/unit/coffee', 'test/acceptance/coffee', 'test/acceptance/config']: + if isdir(join("modules", module, directory)): + volumes.append(join("modules", module, directory)) + +volumes_string = map(lambda vol: "- ./" + vol + ":/app/" + vol + ":ro", volumes) +volumes_string = "\n ".join(volumes_string) + +with open("docker-shared.template.yml", "r") as f: + docker_shared_file = f.read() + +docker_shared_file = docker_shared_file.replace("MODULE_VOLUMES", volumes_string) + +with open("docker-shared.yml", "w") as f: + f.write(docker_shared_file) + diff --git a/services/web/bin/unit_test b/services/web/bin/unit_test new file mode 100755 index 0000000000..da13a441fa --- /dev/null +++ b/services/web/bin/unit_test @@ -0,0 +1,14 @@ +#!/bin/bash +set -e; + +MOCHA="node_modules/.bin/mocha --recursive --reporter spec" + +$MOCHA "$@" test/unit/js + +for dir in modules/*; +do + if [ -d $dir/test/unit/js ]; then + $MOCHA "$@" $dir/test/unit/js + fi +done + diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee index 69bf0a3b7c..98d1c9e031 100644 --- a/services/web/config/settings.defaults.coffee +++ b/services/web/config/settings.defaults.coffee @@ -35,12 +35,12 @@ module.exports = settings = # Databases # --------- mongo: - url : 'mongodb://127.0.0.1/sharelatex' + url : process.env['MONGO_URL'] || "mongodb://127.0.0.1/sharelatex" redis: web: - host: "localhost" - port: "6379" + host: process.env['REDIS_HOST'] || "localhost" + port: process.env['REDIS_PORT'] || "6379" password: "" # websessions: @@ -74,8 +74,8 @@ module.exports = settings = # ] api: - host: "localhost" - port: "6379" + host: process.env['REDIS_HOST'] || "localhost" + port: process.env['REDIS_PORT'] || "6379" password: "" # Service locations @@ -87,6 +87,7 @@ module.exports = settings = internal: web: port: webPort = 3000 + host: process.env['LISTEN_ADDRESS'] or 'localhost' documentupdater: port: docUpdaterPort = 3003 @@ -99,7 +100,7 @@ module.exports = settings = user: httpAuthUser pass: httpAuthPass documentupdater: - url : "http://localhost:#{docUpdaterPort}" + url : "http://#{process.env['DOCUPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}" thirdPartyDataStore: url : "http://localhost:3002" emptyProjectFlushDelayMiliseconds: 5 * seconds @@ -113,7 +114,7 @@ module.exports = settings = enabled: false url : "http://localhost:3054" docstore: - url : "http://localhost:3016" + url : "http://#{process.env['DOCSTORE_HOST'] or 'localhost'}:3016" pubUrl: "http://localhost:3016" chat: url: "http://localhost:3010" diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml new file mode 100644 index 0000000000..aaa8666a16 --- /dev/null +++ b/services/web/docker-compose.yml @@ -0,0 +1,36 @@ +version: "2" + +volumes: + node_modules: + +services: + npm: + extends: + file: docker-shared.yml + service: app + command: npm install + + test_unit: + extends: + file: docker-shared.yml + service: app + command: npm run test:unit + + test_acceptance: + extends: + file: docker-shared.yml + service: app + environment: + REDIS_HOST: redis + MONGO_URL: "mongodb://mongo/sharelatex" + SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true' + depends_on: + - redis + - mongo + command: npm run start + + redis: + image: redis + + mongo: + image: mongo:3.4.6 diff --git a/services/web/docker-shared.template.yml b/services/web/docker-shared.template.yml new file mode 100644 index 0000000000..48adb7a036 --- /dev/null +++ b/services/web/docker-shared.template.yml @@ -0,0 +1,30 @@ +version: "2" + +# We mount all the directories explicitly so that we are only mounting +# the coffee directories, so that the compiled js is only written inside +# the container, and not back to the local filesystem, where it would be +# root owned, and conflict with working outside of the container. + +services: + app: + image: node:6.9.5 + volumes: + - ./package.json:/app/package.json + - ./npm-shrinkwrap.json:/app/npm-shrinkwrap.json + - node_modules:/app/node_modules + - ./bin:/app/bin + # Copying the whole public dir is fine for now, and needed for + # some unit tests to pass, but we will want to isolate the coffee + # and vendor js files, so that the compiled js files are not written + # back to the local filesystem. + - ./public:/app/public + - ./app.coffee:/app/app.coffee:ro + - ./app/coffee:/app/app/coffee:ro + - ./app/templates:/app/app/templates:ro + - ./app/views:/app/app/views:ro + - ./config:/app/config + - ./test/unit/coffee:/app/test/unit/coffee:ro + - ./test/acceptance/coffee:/app/test/acceptance/coffee:ro + - ./test/smoke/coffee:/app/test/smoke/coffee:ro + MODULE_VOLUMES + working_dir: /app \ No newline at end of file diff --git a/services/web/package.json b/services/web/package.json index e831d929e1..c221dc1052 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -9,6 +9,17 @@ "directories": { "public": "./public" }, + "scripts": { + "test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3000/status) do sleep 1; done", + "test:acceptance:run": "bin/acceptance_test $@", + "test:acceptance:dir": "npm run compile:acceptance_tests && npm run test:acceptance:wait_for_app && npm run test:acceptance:run -- $@", + "test:acceptance": "npm run test:acceptance:dir -- $@ test/acceptance/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && bin/unit_test $@", + "compile:unit_tests": "bin/compile_unit_tests", + "compile:acceptance_tests": "bin/compile_acceptance_tests", + "compile:app": "bin/compile_app", + "start": "npm run compile:app && node app.js" + }, "dependencies": { "archiver": "0.9.0", "async": "0.6.2", diff --git a/services/web/test/acceptance/coffee/helpers/redis.coffee b/services/web/test/acceptance/coffee/helpers/redis.coffee index 9aecf6b387..7c48f97d2e 100644 --- a/services/web/test/acceptance/coffee/helpers/redis.coffee +++ b/services/web/test/acceptance/coffee/helpers/redis.coffee @@ -1,5 +1,4 @@ Settings = require('settings-sharelatex') -redis = require('redis-sharelatex') logger = require("logger-sharelatex") Async = require('async') diff --git a/services/web/test/acceptance/coffee/helpers/request.coffee b/services/web/test/acceptance/coffee/helpers/request.coffee index 879acd843a..1c7120d141 100644 --- a/services/web/test/acceptance/coffee/helpers/request.coffee +++ b/services/web/test/acceptance/coffee/helpers/request.coffee @@ -1,4 +1,4 @@ -BASE_URL = "http://localhost:3000" +BASE_URL = "http://#{process.env["HTTP_TEST_HOST"] or "localhost"}:3000" module.exports = require("request").defaults({ baseUrl: BASE_URL, followRedirect: false