From 15aae19d2d3e8e50d5fc69b3eb830eaf0050c0c8 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 12 Jun 2018 10:15:17 +0100 Subject: [PATCH 1/7] Use metadata to determine Google Cloud project dynamically. Fixes: #601 --- services/clsi/bin/install_texlive_gce.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/clsi/bin/install_texlive_gce.sh b/services/clsi/bin/install_texlive_gce.sh index 85ab709694..4f98fed7e6 100755 --- a/services/clsi/bin/install_texlive_gce.sh +++ b/services/clsi/bin/install_texlive_gce.sh @@ -1,13 +1,19 @@ #!/bin/sh METADATA=http://metadata.google.internal./computeMetadata/v1 SVC_ACCT=$METADATA/instance/service-accounts/default +PROJECT_URL=$METADATA/project/project-id ACCESS_TOKEN=$(curl -s -H 'Metadata-Flavor: Google' $SVC_ACCT/token | cut -d'"' -f 4) -if [ -z "$ACCESS_TOKEN" ]; then - echo "No acccess token to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." +#if [ -z "$ACCESS_TOKEN" ]; then +# echo "No acccess token to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." +# exit 0 +#fi +PROJECT=$(curl -s -H 'Metadata-Flavor: Google' $PROJECT_URL) +if [ -z "$PROJECT" ]; then + echo "No project name to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." exit 0 fi docker login -u '_token' -p $ACCESS_TOKEN https://gcr.io -docker pull --all-tags gcr.io/henry-terraform-admin/texlive-full #TODO NEED TO MAKE THIS AN ENV VAR +docker pull --all-tags gcr.io/$PROJECT/texlive-full cp /app/bin/synctex /app/bin/synctex-mount/synctex echo "Finished downloading texlive-full images" From 824745dfbc0537e89a26f448acc4cb5175df0569 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 12 Jun 2018 10:22:30 +0100 Subject: [PATCH 2/7] Update build scripts from 1.1.3 to 1.1.6 --- services/clsi/Jenkinsfile | 9 ++++++++- services/clsi/Makefile | 11 ++++++++--- services/clsi/docker-compose.ci.yml | 9 +++++---- services/clsi/docker-compose.yml | 4 +++- services/clsi/package.json | 4 ++-- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/services/clsi/Jenkinsfile b/services/clsi/Jenkinsfile index bc9ba0142f..eddce50dba 100644 --- a/services/clsi/Jenkinsfile +++ b/services/clsi/Jenkinsfile @@ -29,7 +29,13 @@ pipeline { stage('Package and publish build') { steps { - sh 'make publish' + + withCredentials([file(credentialsId: 'csh-gcdm-test', variable: 'DOCKER_REPO_KEY_PATH')]) { + sh 'docker login -u _json_key --password-stdin https://csh-gcdm-test < ${DOCKER_REPO_KEY_PATH}' + } + sh 'DOCKER_REPO=csh-gcdm-test make publish' + sh 'docker logout https://csh-gcdm-test' + } } @@ -47,6 +53,7 @@ pipeline { post { always { sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + sh 'make clean' } failure { diff --git a/services/clsi/Makefile b/services/clsi/Makefile index 63ff8c02fc..ae7cf16eb8 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.6 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -15,6 +15,8 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ clean: + docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker rmi csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) rm -f app.js rm -rf app/js rm -rf test/unit/js @@ -34,9 +36,12 @@ test_clean: test_acceptance_pre_run: @[ ! -f test/acceptance/scripts/pre-run ] && echo "clsi has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run build: - docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --tag csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + . publish: - docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + + docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 0e14b0f0dc..651b4546d4 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -1,25 +1,27 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.6 version: "2" services: test_unit: - image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER command: npm run test:unit:_run test_acceptance: build: . - image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER extends: file: docker-compose-config.yml service: ci environment: + ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} depends_on: - mongo - redis @@ -30,4 +32,3 @@ services: mongo: image: mongo:3.4 - diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 2283746133..a0b426195b 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.6 version: "2" @@ -24,6 +24,7 @@ services: file: docker-compose-config.yml service: dev environment: + ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres @@ -32,6 +33,7 @@ services: - mongo - redis command: npm run test:acceptance + redis: image: redis diff --git a/services/clsi/package.json b/services/clsi/package.json index 49a259a956..8bcdba29c1 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -10,9 +10,9 @@ "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", From 2155657651c6565b3d9f6705b8583e7039f25003 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 12 Jun 2018 11:17:26 +0100 Subject: [PATCH 3/7] Accidently left warning message commented out :( --- services/clsi/bin/install_texlive_gce.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/clsi/bin/install_texlive_gce.sh b/services/clsi/bin/install_texlive_gce.sh index 4f98fed7e6..ee6efba471 100755 --- a/services/clsi/bin/install_texlive_gce.sh +++ b/services/clsi/bin/install_texlive_gce.sh @@ -3,10 +3,10 @@ METADATA=http://metadata.google.internal./computeMetadata/v1 SVC_ACCT=$METADATA/instance/service-accounts/default PROJECT_URL=$METADATA/project/project-id ACCESS_TOKEN=$(curl -s -H 'Metadata-Flavor: Google' $SVC_ACCT/token | cut -d'"' -f 4) -#if [ -z "$ACCESS_TOKEN" ]; then -# echo "No acccess token to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." -# exit 0 -#fi +if [ -z "$ACCESS_TOKEN" ]; then + echo "No acccess token to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." + exit 0 +fi PROJECT=$(curl -s -H 'Metadata-Flavor: Google' $PROJECT_URL) if [ -z "$PROJECT" ]; then echo "No project name to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment." From cfa1b8ef649a6aaadc01986994486cc1db3cb603 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 12 Jun 2018 15:04:14 +0100 Subject: [PATCH 4/7] Increase acceptance test timeout. --- services/clsi/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/clsi/package.json b/services/clsi/package.json index 8bcdba29c1..92265ebf63 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -10,7 +10,7 @@ "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", From 8ff001ad54d6d2cbfc9e920b87d7662b80fa1d62 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 12 Jun 2018 15:26:10 +0100 Subject: [PATCH 5/7] Specify repo correctly --- services/clsi/Jenkinsfile | 8 ++++---- services/clsi/Makefile | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/clsi/Jenkinsfile b/services/clsi/Jenkinsfile index eddce50dba..b88e3e4412 100644 --- a/services/clsi/Jenkinsfile +++ b/services/clsi/Jenkinsfile @@ -30,11 +30,11 @@ pipeline { stage('Package and publish build') { steps { - withCredentials([file(credentialsId: 'csh-gcdm-test', variable: 'DOCKER_REPO_KEY_PATH')]) { - sh 'docker login -u _json_key --password-stdin https://csh-gcdm-test < ${DOCKER_REPO_KEY_PATH}' + withCredentials([file(credentialsId: 'gcr.io_csh-gcdm-test', variable: 'DOCKER_REPO_KEY_PATH')]) { + sh 'docker login -u _json_key --password-stdin https://gcr.io/csh-gcdm-test < ${DOCKER_REPO_KEY_PATH}' } - sh 'DOCKER_REPO=csh-gcdm-test make publish' - sh 'docker logout https://csh-gcdm-test' + sh 'DOCKER_REPO=gcr.io/csh-gcdm-test make publish' + sh 'docker logout https://gcr.io/csh-gcdm-test' } } diff --git a/services/clsi/Makefile b/services/clsi/Makefile index ae7cf16eb8..a1cefa4cbf 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -16,7 +16,7 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - docker rmi csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker rmi gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) rm -f app.js rm -rf app/js rm -rf test/unit/js @@ -37,7 +37,7 @@ test_acceptance_pre_run: @[ ! -f test/acceptance/scripts/pre-run ] && echo "clsi has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ - --tag csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ . publish: From 62a33701e972984a5f8ba758bc561167efd11438 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 12 Jun 2018 17:44:13 +0100 Subject: [PATCH 6/7] update build scripts so smoke tests are compiled --- services/clsi/Makefile | 2 +- services/clsi/docker-compose.ci.yml | 2 +- services/clsi/docker-compose.yml | 2 +- services/clsi/package.json | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/clsi/Makefile b/services/clsi/Makefile index a1cefa4cbf..ca126e8e48 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.6 +# Version: 1.1.7 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 651b4546d4..9f3d54d1f4 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.6 +# Version: 1.1.7 version: "2" diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index a0b426195b..4c1cfc73f2 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.6 +# Version: 1.1.7 version: "2" diff --git a/services/clsi/package.json b/services/clsi/package.json index 92265ebf63..54625fd273 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -16,9 +16,9 @@ "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", - "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", "nodemon": "nodemon --config nodemon.json", - "compile:smoke_tests": "[ -e test/smoke ] && coffee -o test/smoke/js -c test/smoke/coffee || echo 'No smoke tests to compile, skipping'" + "compile:smoke_tests": "[ ! -e test/smoke/coffee] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" }, "author": "James Allen ", "dependencies": { From 2c10ac06f249af9bd3c8d21d175f9b3b69cea376 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 12 Jun 2018 17:48:23 +0100 Subject: [PATCH 7/7] remove the compile npm command, it isn't needed --- services/clsi/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/services/clsi/package.json b/services/clsi/package.json index 54625fd273..79f9c806d5 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -8,7 +8,6 @@ }, "scripts": { "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", - "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",