From aada9417ea6d10f44e7f6a44d97dc5c53db4d396 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 May 2018 11:40:15 +0100 Subject: [PATCH] dockerised for k8, update build scripts to 1.1.3 --- services/spelling/.dockerignore | 9 ++++ services/spelling/Dockerfile | 22 ++++++++- services/spelling/Jenkinsfile | 66 ++++++++++--------------- services/spelling/Makefile | 42 ++++++++++++++++ services/spelling/docker-compose.ci.yml | 32 ++++++++++++ services/spelling/docker-compose.yml | 39 +++++++++++++++ services/spelling/install_deps.sh | 2 + services/spelling/nodemon.json | 19 +++++++ services/spelling/package-lock.json | 0 services/spelling/package.json | 12 ++++- 10 files changed, 200 insertions(+), 43 deletions(-) create mode 100644 services/spelling/.dockerignore create mode 100644 services/spelling/Makefile create mode 100644 services/spelling/docker-compose.ci.yml create mode 100644 services/spelling/docker-compose.yml create mode 100644 services/spelling/install_deps.sh create mode 100644 services/spelling/nodemon.json create mode 100644 services/spelling/package-lock.json diff --git a/services/spelling/.dockerignore b/services/spelling/.dockerignore new file mode 100644 index 0000000000..386f26df30 --- /dev/null +++ b/services/spelling/.dockerignore @@ -0,0 +1,9 @@ +node_modules/* +gitrev +.git +.gitignore +.npm +.nvmrc +nodemon.json +app.js +**/js/* diff --git a/services/spelling/Dockerfile b/services/spelling/Dockerfile index afda623b72..f359ac3d6c 100644 --- a/services/spelling/Dockerfile +++ b/services/spelling/Dockerfile @@ -1,4 +1,22 @@ +FROM node:6.9.5 as app + +WORKDIR /app + +#wildcard as some files may not be in all repos +COPY package*.json npm-shrink*.json /app/ + +RUN npm install --quiet + +COPY . /app + +RUN npm run compile:all + FROM node:6.9.5 -RUN apt-get update -RUN apt-get install -y aspell aspell-en aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-alt aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-or aspell-pa aspell-pl aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-sv aspell-ta aspell-te aspell-tl aspell-uk aspell-uz \ No newline at end of file +COPY --from=app /app /app + +WORKDIR /app +RUN chmod 0755 ./install_deps.sh && ./install_deps.sh +USER node + +CMD ["node","app.js"] diff --git a/services/spelling/Jenkinsfile b/services/spelling/Jenkinsfile index 5d0dccdb74..bc9ba0142f 100644 --- a/services/spelling/Jenkinsfile +++ b/services/spelling/Jenkinsfile @@ -1,79 +1,67 @@ -pipeline { - - agent { - docker { - image 'sharelatex/node-aspell:6.9.5' - args "-v /var/lib/jenkins/.npm:/tmp/.npm" - } - } +String cron_string = BRANCH_NAME == "master" ? "@daily" : "" - environment { - HOME = "/tmp" - } +pipeline { + agent any triggers { pollSCM('* * * * *') - cron('@daily') + cron(cron_string) } stages { - stage('Set up') { + stage('Build') { steps { - // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id - // which does not exist in the container's /etc/passwd file, causing the clone to fail. - sh 'git config --global core.logallrefupdates false' + sh 'make build' } } - stage('Install') { + + stage('Unit Tests') { steps { - sh 'rm -fr node_modules' - sh 'npm install' - sh 'npm rebuild' - sh 'npm install --quiet grunt-cli' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' } } - stage('Compile') { + + stage('Acceptance Tests') { steps { - sh 'node_modules/.bin/grunt install' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' } } - stage('Test') { + + stage('Package and publish build') { steps { - sh 'node_modules/.bin/grunt test:unit --timeout 10000' + sh 'make publish' } } - stage('Package') { - steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' - } - } - stage('Publish') { + + stage('Publish build number') { steps { + sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } } } } - + post { + always { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + } + failure { - mail(from: "${EMAIL_ALERT_FROM}", - to: "${EMAIL_ALERT_TO}", + mail(from: "${EMAIL_ALERT_FROM}", + to: "${EMAIL_ALERT_TO}", subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", body: "Build: ${BUILD_URL}") } } - + // The options directive is for configuration that applies to the whole job. options { // we'd like to make sure remove old builds, so we don't fill up our storage! buildDiscarder(logRotator(numToKeepStr:'50')) - + // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: timeout(time: 30, unit: 'MINUTES') } diff --git a/services/spelling/Makefile b/services/spelling/Makefile new file mode 100644 index 0000000000..9733a06d96 --- /dev/null +++ b/services/spelling/Makefile @@ -0,0 +1,42 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = spelling +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + docker-compose ${DOCKER_COMPOSE_FLAGS} + + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + @[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit + +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run + @[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance + +test_clean: + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_pre_run: + @[ ! -f test/acceptance/scripts/pre-run ] && echo "spelling has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run +build: + docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + +.PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/spelling/docker-compose.ci.yml b/services/spelling/docker-compose.ci.yml new file mode 100644 index 0000000000..95c8c3041a --- /dev/null +++ b/services/spelling/docker-compose.ci.yml @@ -0,0 +1,32 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run + + test_acceptance: + build: . + image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + depends_on: + - mongo + - redis + user: node + command: npm run test:acceptance:_run + + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/spelling/docker-compose.yml b/services/spelling/docker-compose.yml new file mode 100644 index 0000000000..f24caa8883 --- /dev/null +++ b/services/spelling/docker-compose.yml @@ -0,0 +1,39 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + build: . + volumes: + - .:/app + working_dir: /app + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit + user: node + + test_acceptance: + build: . + volumes: + - .:/app + working_dir: /app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + user: node + depends_on: + - mongo + - redis + command: npm run test:acceptance + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/spelling/install_deps.sh b/services/spelling/install_deps.sh new file mode 100644 index 0000000000..6cda31ec9f --- /dev/null +++ b/services/spelling/install_deps.sh @@ -0,0 +1,2 @@ +apt-get update +apt-get install -y aspell aspell-en aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-alt aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-or aspell-pa aspell-pl aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-sv aspell-ta aspell-te aspell-tl aspell-uk aspell-uz \ No newline at end of file diff --git a/services/spelling/nodemon.json b/services/spelling/nodemon.json new file mode 100644 index 0000000000..98db38d71b --- /dev/null +++ b/services/spelling/nodemon.json @@ -0,0 +1,19 @@ +{ + "ignore": [ + ".git", + "node_modules/" + ], + "verbose": true, + "legacyWatch": true, + "execMap": { + "js": "npm run start" + }, + + "watch": [ + "app/coffee/", + "app.coffee", + "config/" + ], + "ext": "coffee" + +} diff --git a/services/spelling/package-lock.json b/services/spelling/package-lock.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/spelling/package.json b/services/spelling/package.json index 7f12b103a8..156ccb7499 100644 --- a/services/spelling/package.json +++ b/services/spelling/package.json @@ -7,8 +7,16 @@ "url": "https://github.com/sharelatex/spelling-sharelatex.git" }, "scripts": { - "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js" + "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "nodemon": "nodemon --config nodemon.json" }, "version": "0.1.4", "dependencies": {