Merge pull request #20 from sharelatex/ho-docker

Move to docker based builds

services/chat/.dockerignore

@@ -0,0 +1,7 @@
+node_modules/*
+gitrev
+.git
+.gitignore
+.npm
+.nvmrc
+nodemon.json

services/chat/.gitignore

@@ -5,3 +5,5 @@ public/build/
 node_modules/
 
 plato/
+
+**/*.map

services/chat/Dockerfile

@@ -0,0 +1,21 @@
+FROM node:6.14.1 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package*.json npm-shrink*.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
+
+
+
+FROM node:6.14.1
+
+COPY --from=app /app /app
+
+WORKDIR /app
+USER node
+
+CMD ["node", "--expose-gc", "app.js"]

services/chat/Gruntfile.js

@@ -1,129 +0,0 @@
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-module.exports = function(grunt) {
-  // Project configuration.
-  grunt.initConfig({
-    forever: {
-      app: {
-        options: {
-          index: 'app.js'
-        }
-      }
-    },
-
-    execute: {
-      app: {
-        src: 'app.js'
-      }
-    },
-
-    coffee: {
-      server: {
-        expand: true,
-        flatten: false,
-        cwd: 'app/coffee',
-        src: ['**/*.coffee'],
-        dest: 'app/js/',
-        ext: '.js'
-      },
-
-      app_server: {
-        expand: true,
-        flatten: false,
-        src: ['app.coffee'],
-        dest: './',
-        ext: '.js'
-      },
-
-      unit_tests: {
-        expand: true,
-        flatten: false,
-        cwd: 'test/unit/coffee',
-        src: ['**/*.coffee'],
-        dest: 'test/unit/js/',
-        ext: '.js'
-      },
-
-      acceptance_tests: {
-        expand: true,
-        flatten: false,
-        cwd: 'test/acceptance/coffee',
-        src: ['**/*.coffee'],
-        dest: 'test/acceptance/js/',
-        ext: '.js'
-      }
-    },
-
-    watch: {
-      server_coffee: {
-        files: ['app/**/*.coffee', 'test/unit/**/*.coffee'],
-        tasks: ['compile:server', 'compile:unit_tests', 'mochaTest']
-      }
-    },
-
-    clean: ['app/js', 'test/unit/js'],
-
-    nodemon: {
-      dev: {
-        options: {
-          file: 'app.js'
-        }
-      }
-    },
-
-    concurrent: {
-      dev: {
-        tasks: ['nodemon', 'watch'],
-        options: {
-          logConcurrentOutput: true
-        }
-      }
-    },
-
-    mochaTest: {
-      unit: {
-        options: {
-          reporter: process.env.MOCHA_RUNNER || 'spec',
-          grep: grunt.option('grep')
-        },
-        src: ['test/unit/**/*.js']
-      },
-      acceptance: {
-        options: {
-          reporter: process.env.MOCHA_RUNNER || 'spec',
-          grep: grunt.option('grep')
-        },
-        src: ['test/acceptance/**/*.js']
-      }
-    },
-
-    plato: {
-      your_task: {
-        files: { plato: ['app/js/**/*.js'] }
-      }
-    }
-  })
-
-  grunt.loadNpmTasks('grunt-contrib-coffee')
-  grunt.loadNpmTasks('grunt-contrib-watch')
-  grunt.loadNpmTasks('grunt-nodemon')
-  grunt.loadNpmTasks('grunt-contrib-clean')
-  grunt.loadNpmTasks('grunt-concurrent')
-  grunt.loadNpmTasks('grunt-mocha-test')
-  grunt.loadNpmTasks('grunt-plato')
-  grunt.loadNpmTasks('grunt-execute')
-  grunt.loadNpmTasks('grunt-bunyan')
-  grunt.loadNpmTasks('grunt-forever')
-
-  grunt.registerTask('compile', ['clean', 'coffee'])
-  grunt.registerTask('install', ['compile'])
-  grunt.registerTask('default', ['compile', 'bunyan', 'execute'])
-  grunt.registerTask('test:unit', ['compile', 'mochaTest:unit'])
-  return grunt.registerTask('test:acceptance', [
-    'compile:acceptance_tests',
-    'mochaTest:acceptance'
-  ])
-}

services/chat/Jenkinsfile

@@ -17,13 +17,6 @@ pipeline {
 
   stages {
     stage('Install') {
-      agent {
-        docker {
-          image 'node:6.14.1'
-          args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
-          reuseNode true
-        }
-      }
       steps {
         withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
           sh "curl $GIT_API_URL \
@@ -34,13 +27,12 @@ pipeline {
             \"context\": \"ci/jenkins\" }' \
             -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
         }
-        // we need to disable logallrefupdates, else git clones
+      }
-        // during the npm install will require git to lookup the
+    }
-        // user id which does not exist in the container's
+
-        // /etc/passwd file, causing the clone to fail.
+    stage('Build') {
-        sh 'git config --global core.logallrefupdates false'
+      steps {
-        sh 'rm -rf node_modules'
+        sh 'make build'
-        sh 'npm install && npm rebuild'
       }
     }
 
@@ -56,20 +48,27 @@ pipeline {
       }
     }
 
-    stage('Package and publish build') {
+    stage('Package and docker push') {
       steps {
         sh 'echo ${BUILD_NUMBER} > build_number.txt'
         sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
-        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
+        
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
+        withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
+          sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
         }
+        sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
+        sh 'docker logout https://gcr.io/overleaf-ops'
+        
       }
     }
 
-    stage('Publish build number') {
+    stage('Publish to s3') {
       steps {
         sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
+        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
+            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
+        }
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
             // The deployment process uses this file to figure out the latest build
             s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")

services/chat/Makefile

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.10
+# Version: 1.1.12
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -13,8 +13,9 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker-compose ${DOCKER_COMPOSE_FLAGS}
 
-
 clean:
+	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 lint:
 	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
 
@@ -31,5 +32,16 @@ test_clean:
 
 test_acceptance_pre_run:
 	@[ ! -f test/acceptance/scripts/pre-run ] && echo "chat has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
+build:
+	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		.
+
+tar:
+	$(DOCKER_COMPOSE) up tar
+
+publish:
+
+	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/chat/app/js/server.js

@@ -8,10 +8,10 @@
  * DS102: Remove unnecessary code created because of implicit returns
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
-const logger = require('logger-sharelatex')
-logger.initialize('chat-sharelatex')
 const metrics = require('metrics-sharelatex')
 metrics.initialize('chat')
+const logger = require('logger-sharelatex')
+logger.initialize('chat')
 const Path = require('path')
 const express = require('express')
 const app = express()
@@ -20,6 +20,7 @@ const Router = require('./router')
 
 app.use(express.bodyParser())
 app.use(metrics.http.monitor(logger))
+metrics.injectMetricsRoute(app)
 
 if (app.get('env') === 'development') {
   console.log('Development Enviroment')

services/chat/buildscript.txt

@@ -1,9 +1,8 @@
 chat
---node-version=6.14.1
---script-version=1.1.10
---build-target=native
---kube=false
---dependencies=redis,mongo
 --language=es
---docker-repos=quay.io/sharelatex
+--node-version=6.14.1
 --acceptance-creds=None
+--dependencies=mongo,redis
+--docker-repos=gcr.io/overleaf-ops
+--build-target=docker
+--script-version=1.1.12

services/chat/config/settings.defaults.js

@@ -17,14 +17,14 @@ module.exports = {
   },
 
   mongo: {
-    url: `mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex`
+    url: process.env['MONGO_CONNECTION_STRING'] || `mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex`
   },
 
   redis: {
     web: {
       host: process.env['REDIS_HOST'] || 'localhost',
       port: '6379',
-      password: ''
+      password: process.env['REDIS_PASSWORD'] || ''
     }
   }
 }

services/chat/docker-compose.ci.yml

@@ -1,24 +1,20 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.10
+# Version: 1.1.12
 
 version: "2"
 
 services:
   test_unit:
-    image: node:6.14.1
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - .:/app
-    working_dir: /app
     user: node
     command: npm run test:unit:_run
 
+
   test_acceptance:
-    image: node:6.14.1
+    build: .
-    volumes:
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-      - .:/app
-    working_dir: /app
     environment:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
@@ -31,6 +27,16 @@ services:
     user: node
     command: npm run test:acceptance:_run
 
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
   redis:
     image: redis
 

services/chat/docker-compose.yml

@@ -1,13 +1,13 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.10
+# Version: 1.1.12
 
 version: "2"
 
 services:
   test_unit:
-    image: node:6.14.1
+    build: .
     volumes:
       - .:/app
     working_dir: /app
@@ -17,7 +17,7 @@ services:
     user: node
 
   test_acceptance:
-    image: node:6.14.1
+    build: .
     volumes:
       - .:/app
     working_dir: /app
@@ -33,6 +33,16 @@ services:
       - redis
     command: npm run test:acceptance
 
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
   redis:
     image: redis
 

services/chat/package.json

@@ -7,13 +7,9 @@
     "url": "https://github.com/sharelatex/chat-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
     "start": "node $NODE_APP_OPTIONS app.js",
     "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
     "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
-    "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
-    "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
     "nodemon": "nodemon --config nodemon.json",
     "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
     "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
@@ -21,20 +17,20 @@
   },
   "dependencies": {
     "async": "0.2.9",
-    "coffee-script": "~1.7.1",
     "express": "3.3.1",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master",
+    "logger-sharelatex": "^1.6.0",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.7.1",
+    "metrics-sharelatex": "^2.1.1",
     "mongojs": "2.4.0",
     "redis": "~0.10.1",
     "request": "^2.79.0",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0",
+    "settings-sharelatex": "^1.1.0",
     "v8-profiler": "^5.6.5"
   },
   "devDependencies": {
+    "ajv": "^5.5.2",
     "bunyan": "^1.0.0",
     "chai": "",
-    "eslint": "^5.10.0",
+    "eslint": "^5.11.1",
     "eslint-config-prettier": "^3.3.0",
     "eslint-config-standard": "^12.0.0",
     "eslint-config-standard-jsx": "^6.0.2",
@@ -45,22 +41,10 @@
     "eslint-plugin-jsx-a11y": "^6.1.2",
     "eslint-plugin-mocha": "^5.2.0",
     "eslint-plugin-node": "^8.0.0",
-    "eslint-plugin-prettier": "^3.0.0",
+    "eslint-plugin-prettier": "^3.0.1",
     "eslint-plugin-promise": "^4.0.1",
-    "eslint-plugin-react": "^7.11.1",
+    "eslint-plugin-react": "^7.12.2",
     "eslint-plugin-standard": "^4.0.0",
-    "grunt": "~0.4.1",
-    "grunt-bunyan": "^0.5.0",
-    "grunt-concurrent": "~0.4.2",
-    "grunt-contrib-clean": "~0.5.0",
-    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-watch": "~0.5.3",
-    "grunt-execute": "^0.2.2",
-    "grunt-forever": "^0.4.7",
-    "grunt-mocha-test": "~0.8.0",
-    "grunt-nodemon": "~0.1.2",
-    "grunt-notify": "~0.2.16",
-    "grunt-plato": "~0.2.1",
     "mocha": "^4.1.0",
     "nodemon": "^1.14.11",
     "prettier-eslint-cli": "^4.7.1",