From 581963a4af43a543f9939cb95555f595f0a8b1cc Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 23 May 2018 13:52:20 +0100 Subject: [PATCH] Dockerised env. 1.1.3 build scripts --- services/document-updater/.dockerignore | 60 +++---------------- services/document-updater/Dockerfile | 22 +++++++ services/document-updater/Jenkinsfile | 36 +---------- services/document-updater/Makefile | 25 ++++++-- .../document-updater/docker-compose.ci.yml | 23 ++++--- services/document-updater/docker-compose.yml | 22 ++++--- services/document-updater/nodemon.json | 5 +- services/document-updater/package.json | 12 ++-- 8 files changed, 87 insertions(+), 118 deletions(-) create mode 100644 services/document-updater/Dockerfile diff --git a/services/document-updater/.dockerignore b/services/document-updater/.dockerignore index ad21f261b4..386f26df30 100644 --- a/services/document-updater/.dockerignore +++ b/services/document-updater/.dockerignore @@ -1,53 +1,9 @@ -compileFolder - -Compiled source # -################### -*.com -*.class -*.dll -*.exe -*.o -*.so - -# Packages # -############ -# it's better to unpack these files and commit the raw source -# git has its own built in compression methods -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip - -# Logs and databases # -###################### -*.log -*.sql -*.sqlite - -# OS generated files # -###################### -.DS_Store? -ehthumbs.db -Icon? -Thumbs.db - -/node_modules/* - +node_modules/* +gitrev +.git +.gitignore +.npm +.nvmrc +nodemon.json app.js -app/js/* - -test/unit/js/* -test/acceptance/js/* - -forever/ - -**.swp - -# Redis cluster -**/appendonly.aof -**/dump.rdb -**/nodes.conf +**/js/* diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile new file mode 100644 index 0000000000..aabf01ad91 --- /dev/null +++ b/services/document-updater/Dockerfile @@ -0,0 +1,22 @@ +FROM node:6.9.5 as app + +WORKDIR /app + +#wildcard as some files may not be in all repos +COPY package*.json npm-shrink*.json /app/ + +RUN npm install --quiet + +COPY . /app + + +RUN npm run compile:all + +FROM node:6.9.5 + +COPY --from=app /app /app + +WORKDIR /app +USER node + +CMD ["node","app.js"] diff --git a/services/document-updater/Jenkinsfile b/services/document-updater/Jenkinsfile index ab90aaae29..bc9ba0142f 100644 --- a/services/document-updater/Jenkinsfile +++ b/services/document-updater/Jenkinsfile @@ -9,34 +9,9 @@ pipeline { } stages { - stage('Install') { - agent { - docker { - image 'node:6.9.5' - args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" - reuseNode true - } - } + stage('Build') { steps { - // we need to disable logallrefupdates, else git clones - // during the npm install will require git to lookup the - // user id which does not exist in the container's - // /etc/passwd file, causing the clone to fail. - sh 'git config --global core.logallrefupdates false' - sh 'rm -rf node_modules' - sh 'npm install && npm rebuild' - } - } - - stage('Compile') { - agent { - docker { - image 'node:6.9.5' - reuseNode true - } - } - steps { - sh 'npm run compile:all' + sh 'make build' } } @@ -54,12 +29,7 @@ pipeline { stage('Package and publish build') { steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' - withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") - } + sh 'make publish' } } diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index 10adbe3866..215b56a9e9 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -1,13 +1,18 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.0.1 +# Version: 1.1.3 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = document-updater DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml -DOCKER_COMPOSE := docker-compose ${DOCKER_COMPOSE_FLAGS} +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + docker-compose ${DOCKER_COMPOSE_FLAGS} + clean: rm -f app.js @@ -18,12 +23,20 @@ clean: test: test_unit test_acceptance test_unit: - @[ -d test/unit ] && $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS} || echo "document-updater has no unit tests" + @[ ! -d test/unit ] && echo "document-updater has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -test_acceptance: test_clean # clear the database before each acceptance test run - @[ -d test/acceptance ] && $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS} || echo "document-updater has no acceptance tests" +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run + @[ ! -d test/acceptance ] && echo "document-updater has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: - $(DOCKER_COMPOSE) down -t 0 + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_pre_run: + @[ ! -f test/acceptance/scripts/pre-run ] && echo "document-updater has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run +build: + docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index 9f40ba879d..21c006641e 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -1,33 +1,32 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.0.1 +# Version: 1.1.3 version: "2" services: test_unit: - image: node:6.9.5 - volumes: - - .:/app - working_dir: /app - entrypoint: npm run test:unit:_run + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run test_acceptance: - image: node:6.9.5 - volumes: - - .:/app - working_dir: /app + build: . + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: REDIS_HOST: redis MONGO_HOST: mongo + POSTGRES_HOST: postgres depends_on: - - redis - mongo - entrypoint: npm run test:acceptance:_run + - redis + user: node + command: npm run test:acceptance:_run redis: image: redis mongo: image: mongo:3.4 + diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index d55ca56580..f24caa8883 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -1,33 +1,39 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.0.1 +# Version: 1.1.3 version: "2" services: test_unit: - image: node:6.9.5 + build: . volumes: - .:/app working_dir: /app - entrypoint: npm run test:unit + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit + user: node test_acceptance: - image: node:6.9.5 + build: . volumes: - .:/app + working_dir: /app environment: REDIS_HOST: redis MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + user: node depends_on: - - redis - mongo - working_dir: /app - entrypoint: npm run test:acceptance - + - redis + command: npm run test:acceptance redis: image: redis mongo: image: mongo:3.4 + diff --git a/services/document-updater/nodemon.json b/services/document-updater/nodemon.json index 9a3be8d966..98db38d71b 100644 --- a/services/document-updater/nodemon.json +++ b/services/document-updater/nodemon.json @@ -8,9 +8,12 @@ "execMap": { "js": "npm run start" }, + "watch": [ "app/coffee/", - "app.coffee" + "app.coffee", + "config/" ], "ext": "coffee" + } diff --git a/services/document-updater/package.json b/services/document-updater/package.json index ce5aa9d18c..a23ae73427 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -7,14 +7,14 @@ "url": "https://github.com/sharelatex/document-updater-sharelatex.git" }, "scripts": { - "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js", + "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@", - "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'", - "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", "nodemon": "nodemon --config nodemon.json" },