From a59dc9e14917f0cd4c603c499142ed140ad3146d Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 29 Jan 2018 12:01:26 +0000 Subject: [PATCH 1/3] Move acceptance tests to single process pattern --- services/chat/app.coffee | 12 ++++++----- .../coffee/DeletingAMessageTests.coffee | 4 +++- .../coffee/DeletingAThreadTests.coffee | 4 +++- .../coffee/EditingAMessageTests.coffee | 4 +++- .../coffee/GettingMessagesTests.coffee | 4 +++- .../coffee/ResolvingAThreadTests.coffee | 5 ++++- .../coffee/SendingAMessageTests.coffee | 4 +++- .../acceptance/coffee/helpers/ChatApp.coffee | 20 +++++++++++++++++++ 8 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 services/chat/test/acceptance/coffee/helpers/ChatApp.coffee diff --git a/services/chat/app.coffee b/services/chat/app.coffee index 8f5cc18dc0..3fad2e610e 100644 --- a/services/chat/app.coffee +++ b/services/chat/app.coffee @@ -3,9 +3,11 @@ settings = require 'settings-sharelatex' Server = require "./app/js/server" -port = settings.internal?.chat?.port or 3010 -host = settings.internal?.chat?.host or "localhost" +if !module.parent # Called directly + port = settings.internal?.chat?.port or 3010 + host = settings.internal?.chat?.host or "localhost" + Server.server.listen port, host, (error) -> + throw error if error? + logger.info "Chat starting up, listening on #{host}:#{port}" -Server.server.listen port, host, (error) -> - throw error if error? - logger.info "Chat starting up, listening on #{host}:#{port}" +module.exports = Server.server diff --git a/services/chat/test/acceptance/coffee/DeletingAMessageTests.coffee b/services/chat/test/acceptance/coffee/DeletingAMessageTests.coffee index 65c183d0f0..7d6ca8b00b 100644 --- a/services/chat/test/acceptance/coffee/DeletingAMessageTests.coffee +++ b/services/chat/test/acceptance/coffee/DeletingAMessageTests.coffee @@ -2,12 +2,14 @@ expect = require("chai").expect ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Deleting a message", -> - before -> + before (done) -> @project_id = ObjectId().toString() @user_id = ObjectId().toString() @thread_id = ObjectId().toString() + ChatApp.ensureRunning done describe "in a thread", -> before (done) -> diff --git a/services/chat/test/acceptance/coffee/DeletingAThreadTests.coffee b/services/chat/test/acceptance/coffee/DeletingAThreadTests.coffee index 850bcf0ef0..a803e730c7 100644 --- a/services/chat/test/acceptance/coffee/DeletingAThreadTests.coffee +++ b/services/chat/test/acceptance/coffee/DeletingAThreadTests.coffee @@ -3,11 +3,13 @@ expect = require("chai").expect crypto = require "crypto" ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Deleting a thread", -> - before -> + before (done) -> @project_id = ObjectId().toString() @user_id = ObjectId().toString() + ChatApp.ensureRunning done describe "with a thread that is deleted", -> before (done) -> diff --git a/services/chat/test/acceptance/coffee/EditingAMessageTests.coffee b/services/chat/test/acceptance/coffee/EditingAMessageTests.coffee index 51e14cfedd..f507e4b205 100644 --- a/services/chat/test/acceptance/coffee/EditingAMessageTests.coffee +++ b/services/chat/test/acceptance/coffee/EditingAMessageTests.coffee @@ -2,12 +2,14 @@ expect = require("chai").expect ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Editing a message", -> - before -> + before (done) -> @project_id = ObjectId().toString() @user_id = ObjectId().toString() @thread_id = ObjectId().toString() + ChatApp.ensureRunning done describe "in a thread", -> before (done) -> diff --git a/services/chat/test/acceptance/coffee/GettingMessagesTests.coffee b/services/chat/test/acceptance/coffee/GettingMessagesTests.coffee index 6d622b409a..b0c872e5cd 100644 --- a/services/chat/test/acceptance/coffee/GettingMessagesTests.coffee +++ b/services/chat/test/acceptance/coffee/GettingMessagesTests.coffee @@ -4,13 +4,15 @@ async = require "async" crypto = require "crypto" ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Getting messages", -> - before -> + before (done) -> @user_id1 = ObjectId().toString() @user_id2 = ObjectId().toString() @content1 = "foo bar" @content2 = "hello world" + ChatApp.ensureRunning done describe "globally", -> before (done) -> diff --git a/services/chat/test/acceptance/coffee/ResolvingAThreadTests.coffee b/services/chat/test/acceptance/coffee/ResolvingAThreadTests.coffee index 4f666666ac..d162f881f6 100644 --- a/services/chat/test/acceptance/coffee/ResolvingAThreadTests.coffee +++ b/services/chat/test/acceptance/coffee/ResolvingAThreadTests.coffee @@ -3,11 +3,14 @@ expect = require("chai").expect crypto = require "crypto" ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Resolving a thread", -> - before -> + before (done) -> @project_id = ObjectId().toString() @user_id = ObjectId().toString() + ChatApp.ensureRunning done + describe "with a resolved thread", -> before (done) -> @thread_id = ObjectId().toString() diff --git a/services/chat/test/acceptance/coffee/SendingAMessageTests.coffee b/services/chat/test/acceptance/coffee/SendingAMessageTests.coffee index 77af23bda7..64a2d5dbe5 100644 --- a/services/chat/test/acceptance/coffee/SendingAMessageTests.coffee +++ b/services/chat/test/acceptance/coffee/SendingAMessageTests.coffee @@ -2,9 +2,11 @@ expect = require("chai").expect ChatClient = require "./helpers/ChatClient" +ChatApp = require "./helpers/ChatApp" describe "Sending a message", -> - before -> + before (done) -> + ChatApp.ensureRunning done describe "globally", -> before (done) -> diff --git a/services/chat/test/acceptance/coffee/helpers/ChatApp.coffee b/services/chat/test/acceptance/coffee/helpers/ChatApp.coffee new file mode 100644 index 0000000000..8973541003 --- /dev/null +++ b/services/chat/test/acceptance/coffee/helpers/ChatApp.coffee @@ -0,0 +1,20 @@ +app = require('../../../../app') +require("logger-sharelatex").logger.level("error") + +module.exports = + running: false + initing: false + callbacks: [] + ensureRunning: (callback = (error) ->) -> + if @running + return callback() + else if @initing + @callbacks.push callback + else + @initing = true + @callbacks.push callback + app.listen 3010, "localhost", (error) => + throw error if error? + @running = true + for callback in @callbacks + callback() \ No newline at end of file From bd581d069bd32dbc320867a9c83ebdd3bec4f72a Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 29 Jan 2018 12:07:35 +0000 Subject: [PATCH 2/3] Update with build scripts version 1.0.0 --- services/chat/.nvmrc | 2 +- services/chat/Jenkinsfile | 52 +++++++++++++++++++---------- services/chat/Makefile | 29 ++++++++++++++++ services/chat/docker-compose.ci.yml | 33 ++++++++++++++++++ services/chat/docker-compose.yml | 33 ++++++++++++++++++ services/chat/nodemon.json | 15 +++++++++ services/chat/package.json | 12 ++++++- 7 files changed, 156 insertions(+), 20 deletions(-) create mode 100644 services/chat/Makefile create mode 100644 services/chat/docker-compose.ci.yml create mode 100644 services/chat/docker-compose.yml create mode 100644 services/chat/nodemon.json diff --git a/services/chat/.nvmrc b/services/chat/.nvmrc index d87edbfc10..fae6e3d04b 100644 --- a/services/chat/.nvmrc +++ b/services/chat/.nvmrc @@ -1 +1 @@ -4.2.1 \ No newline at end of file +4.2.1 diff --git a/services/chat/Jenkinsfile b/services/chat/Jenkinsfile index 67b26694ab..c9f4a0840f 100644 --- a/services/chat/Jenkinsfile +++ b/services/chat/Jenkinsfile @@ -1,10 +1,11 @@ -pipeline { +String cron_string = BRANCH_NAME == "master" ? "@daily" : "" +pipeline { agent any - + triggers { pollSCM('* * * * *') - cron('@daily') + cron(cron_string) } stages { @@ -17,15 +18,17 @@ pipeline { } } steps { - // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id - // which does not exist in the container's /etc/passwd file, causing the clone to fail. + // we need to disable logallrefupdates, else git clones + // during the npm install will require git to lookup the + // user id which does not exist in the container's + // /etc/passwd file, causing the clone to fail. sh 'git config --global core.logallrefupdates false' - sh 'rm -fr node_modules' + sh 'rm -rf node_modules' sh 'npm install && npm rebuild' - sh 'npm install --quiet grunt-cli' } } - stage('Compile and Test') { + + stage('Compile') { agent { docker { image 'node:4.2.1' @@ -33,28 +36,37 @@ pipeline { } } steps { - sh 'node_modules/.bin/grunt coffee' - sh 'node_modules/.bin/grunt compile:acceptance_tests' - sh 'NODE_ENV=development node_modules/.bin/grunt test:unit' + sh 'npm run compile:all' } } + + stage('Unit Tests') { + steps { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' + } + } + stage('Acceptance Tests') { steps { - sh 'docker pull sharelatex/acceptance-test-runner' - sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' } } - stage('Package') { + + stage('Package and publish build') { steps { sh 'echo ${BUILD_NUMBER} > build_number.txt' sh 'touch build.tar.gz' // Avoid tar warning about files changing during read sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' - } - } - stage('Publish') { - steps { withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + } + } + } + + stage('Publish build number') { + steps { + sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' + withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } @@ -63,6 +75,10 @@ pipeline { } post { + always { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + } + failure { mail(from: "${EMAIL_ALERT_FROM}", to: "${EMAIL_ALERT_TO}", diff --git a/services/chat/Makefile b/services/chat/Makefile new file mode 100644 index 0000000000..865d63a9fc --- /dev/null +++ b/services/chat/Makefile @@ -0,0 +1,29 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.0.0 + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = chat +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := docker-compose ${DOCKER_COMPOSE_FLAGS} + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + @[ -d test/unit ] && $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS} || echo "chat has no unit tests" + +test_acceptance: test_clean # clear the database before each acceptance test run + @[ -d test/acceptance ] && $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS} || echo "chat has no acceptance tests" + +test_clean: + $(DOCKER_COMPOSE) down + +.PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml new file mode 100644 index 0000000000..bc5398bd83 --- /dev/null +++ b/services/chat/docker-compose.ci.yml @@ -0,0 +1,33 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.0.0 + +version: "2" + +services: + test_unit: + image: node:4.2.1 + volumes: + - .:/app + working_dir: /app + entrypoint: npm run test:unit:run + + test_acceptance: + image: node:4.2.1 + volumes: + - .:/app + working_dir: /app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + depends_on: + - redis + - mongo + entrypoint: npm run test:acceptance:run + + redis: + image: redis + + mongo: + image: mongo:3.4 diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml new file mode 100644 index 0000000000..4a76b6d3d8 --- /dev/null +++ b/services/chat/docker-compose.yml @@ -0,0 +1,33 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.0.0 + +version: "2" + +services: + test_unit: + image: node:4.2.1 + volumes: + - .:/app + working_dir: /app + entrypoint: npm run test:unit + + test_acceptance: + image: node:4.2.1 + volumes: + - .:/app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + depends_on: + - redis + - mongo + working_dir: /app + entrypoint: npm run test:acceptance + + redis: + image: redis + + mongo: + image: mongo:3.4 diff --git a/services/chat/nodemon.json b/services/chat/nodemon.json new file mode 100644 index 0000000000..9044f921c6 --- /dev/null +++ b/services/chat/nodemon.json @@ -0,0 +1,15 @@ +{ + "ignore": [ + ".git", + "node_modules/" + ], + "verbose": true, + "execMap": { + "js": "npm run start" + }, + "watch": [ + "app/coffee/", + "app.coffee" + ], + "ext": "coffee" +} diff --git a/services/chat/package.json b/services/chat/package.json index 16ea704bee..f7db817ad8 100644 --- a/services/chat/package.json +++ b/services/chat/package.json @@ -8,7 +8,15 @@ }, "scripts": { "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js" + "start": "npm run compile:app && node app.js", + "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:run -- $@", + "test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:run -- $@", + "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'", + "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "nodemon": "nodemon --config nodemon.json" }, "dependencies": { "async": "0.2.9", @@ -37,6 +45,8 @@ "grunt-nodemon": "~0.1.2", "grunt-notify": "~0.2.16", "grunt-plato": "~0.2.1", + "mocha": "^4.1.0", + "nodemon": "^1.14.11", "sandboxed-module": "", "sinon": "", "timekeeper": "" From 02918189ca68b06b7e531634570092b1dbd3ad7a Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 29 Jan 2018 14:21:22 +0000 Subject: [PATCH 3/3] Make :run npm commands 'private' --- services/chat/docker-compose.ci.yml | 4 +-- services/chat/package.json | 10 ++++---- .../chat/test/acceptance/scripts/full-test.sh | 25 ------------------- 3 files changed, 7 insertions(+), 32 deletions(-) delete mode 100755 services/chat/test/acceptance/scripts/full-test.sh diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index bc5398bd83..0ba28ed4e3 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -11,7 +11,7 @@ services: volumes: - .:/app working_dir: /app - entrypoint: npm run test:unit:run + entrypoint: npm run test:unit:_run test_acceptance: image: node:4.2.1 @@ -24,7 +24,7 @@ services: depends_on: - redis - mongo - entrypoint: npm run test:acceptance:run + entrypoint: npm run test:acceptance:_run redis: image: redis diff --git a/services/chat/package.json b/services/chat/package.json index f7db817ad8..2f0eac6f9f 100644 --- a/services/chat/package.json +++ b/services/chat/package.json @@ -9,14 +9,14 @@ "scripts": { "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", "start": "npm run compile:app && node app.js", - "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:run -- $@", - "test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:run -- $@", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@", "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'", "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'", "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", - "nodemon": "nodemon --config nodemon.json" + "nodemon": "nodemon --config nodemon.json", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js" }, "dependencies": { "async": "0.2.9", diff --git a/services/chat/test/acceptance/scripts/full-test.sh b/services/chat/test/acceptance/scripts/full-test.sh deleted file mode 100755 index 13620ba41e..0000000000 --- a/services/chat/test/acceptance/scripts/full-test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env bash - -# npm rebuild - -sleep 60 - -echo ">> Starting server..." - -grunt --no-color forever:app:start - -echo ">> Server started" - -sleep 5 - -echo ">> Running acceptance tests..." -grunt --no-color mochaTest:acceptance -_test_exit_code=$? - -echo ">> Killing server" - -grunt --no-color forever:app:stop - -echo ">> Done" - -exit $_test_exit_code