Merge pull request #1471 from sharelatex/ho-docker2

Move to docker image based builds

GitOrigin-RevId: 992aeaf69bfea4d1847a07c457575a63985d4407

services/web/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:6.15.1 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package.json npm-shrinkwrap.json /app/
+
+RUN npm install --quiet
+
+
+COPY . /app
+
+RUN make compile_full
+
+FROM node:6.15.1
+
+COPY --from=app /app /app
+
+WORKDIR /app
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+USER node
+
+CMD ["node", "--expose-gc", "app.js"]

services/web/Makefile

@@ -1,11 +1,20 @@
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
+
+
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-GIT_SHA ?= $(shell git rev-parse HEAD)
 PROJECT_NAME = web
 
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS}
+
 MODULE_DIRS := $(shell find modules -mindepth 1 -maxdepth 1 -type d -not -name '.git' )
 MODULE_MAKEFILES := $(MODULE_DIRS:=/Makefile)
+
+COFFEE := node_modules/.bin/coffee -m $(COFFEE_OPTIONS)
 COFFEE := node_modules/.bin/coffee $(COFFEE_OPTIONS)
 BABEL := node_modules/.bin/babel
 GRUNT := node_modules/.bin/grunt
@@ -37,8 +46,6 @@ CSS_OL_IEEE_FILE := public/stylesheets/ieee-style.css
 
 CSS_FILES := $(CSS_SL_FILE) $(CSS_OL_FILE) $(CSS_OL_LIGHT_FILE) $(CSS_OL_IEEE_FILE)
 
-SENTRY_TEMPLATE := app/views/sentry.pug
-
 # The automatic variable $(@D) is the target directory name
 app.js: app.coffee
 	$(COFFEE) --compile -o $(@D) $< 
@@ -119,7 +126,9 @@ minify_css: $(CSS_FILES)
 minify_es:
 	npm -q run webpack:production
 
-compile: $(JS_FILES) $(OUTPUT_SRC_FILES) css public/js/main.js public/js/ide.js
+compile: compile_app $(OUTPUT_SRC_FILES) css public/js/main.js public/js/ide.js
+
+compile_app: $(JS_FILES)
 	@$(MAKE) compile_modules
 
 compile_full:
@@ -198,19 +207,20 @@ clean_css:
 	rm -f public/stylesheets/*.css*
 
 clean_ci:
-	docker-compose down -v -t 0
+	$(DOCKER_COMPOSE) down -v -t 0
 
 test: test_unit test_frontend test_acceptance
 
 test_unit:
-	npm -q run test:unit -- ${MOCHA_ARGS}
+	@[ ! -d test/unit ] && echo "web has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
 
 test_unit_app:
 	npm -q run test:unit:app -- ${MOCHA_ARGS}
 
-test_frontend: test_clean # stop service
+test_frontend: test_clean compile test_frontend_run
-	$(MAKE) compile
+
-	docker-compose ${DOCKER_COMPOSE_FLAGS} up --exit-code-from test_frontend --abort-on-container-exit test_frontend
+test_frontend_run:
+	$(DOCKER_COMPOSE) up test_frontend
 
 test_acceptance: compile test_acceptance_app_run test_acceptance_modules_run
 
@@ -218,9 +228,10 @@ test_acceptance_app: compile test_acceptance_app_run
 
 test_acceptance_module: compile test_acceptance_module_run
 
+test_acceptance_run: test_acceptance_app_run test_acceptance_modules_run
+
 test_acceptance_app_run: test_clean
-	@set -e; \
+	$(DOCKER_COMPOSE) run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} test/acceptance/js
-	docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} test/acceptance/js
 
 test_acceptance_modules_run:
 	@set -e; \
@@ -237,7 +248,7 @@ test_acceptance_module_run: $(MODULE_MAKEFILES) test_clean
 	fi
 
 test_clean:
-	docker-compose ${DOCKER_COMPOSE_FLAGS} down -v -t 0
+	$(DOCKER_COMPOSE) down -v -t 0
 
 ci:
 	MOCHA_ARGS="--reporter tap" \
@@ -252,10 +263,17 @@ format_fix:
 lint:
 	npm -q run lint
 
-version:
+	
-	sed -i.original -e "s/@@COMMIT@@/${GIT_SHA}/g" $(SENTRY_TEMPLATE)
+build:
-	sed -i.original -e "s/@@RELEASE@@/${BUILD_NUMBER}/g" $(SENTRY_TEMPLATE)
+	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
-	rm $(SENTRY_TEMPLATE).original
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		.
+
+publish:
+	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+
+tar:
+	$(DOCKER_COMPOSE) up tar
 
 .PHONY:
 	all add install update test test_unit test_frontend test_acceptance \

services/web/Makefile.module

@@ -1,5 +1,6 @@
 MODULE_NAME := $(notdir $(shell pwd))
 MODULE_DIR := modules/$(MODULE_NAME)
+PROJECT_NAME = web
 COFFEE := ../../node_modules/.bin/coffee
 BABEL := ../../node_modules/.bin/babel
 
@@ -17,8 +18,15 @@ IDE_TEST_OUTPUT_FILES := $(subst test/unit_frontend/src/ide,../../test/unit_fron
 MAIN_SRC_FILES := $(shell [ -e public/src/main ] && find public/src/main -name '*.js')
 MAIN_OUTPUT_FILES := $(subst public/src/main,../../public/js/main/$(MODULE_NAME),$(MAIN_SRC_FILES))
 
-DOCKER_COMPOSE_FLAGS := -f $(MODULE_DIR)/docker-compose.yml
+DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := cd ../../ && MODULE_DIR=$(MODULE_DIR) docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS}
+DOCKER_COMPOSE_MODULE_FLAGS := ${DOCKER_COMPOSE_FLAGS} -f $(MODULE_DIR)/docker-compose.yml
+DOCKER_COMPOSE := cd ../../ && \
+	MODULE_DIR=$(MODULE_DIR) \
+	BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_MODULE_FLAGS}
 
 app/js/%.js: app/coffee/%.coffee
 	@mkdir -p $(dir $@)

services/web/app.coffee

@@ -1,6 +1,8 @@
+metrics = require("metrics-sharelatex")
+metrics.initialize(process.env['METRICS_APP_NAME'] or "web")
 Settings = require('settings-sharelatex')
 logger = require 'logger-sharelatex'
-logger.initialize("web-sharelatex")
+logger.initialize("web")
 logger.logger.serializers.user = require("./app/js/infrastructure/LoggerSerializers").user
 logger.logger.serializers.docs = require("./app/js/infrastructure/LoggerSerializers").docs
 logger.logger.serializers.files = require("./app/js/infrastructure/LoggerSerializers").files
@@ -8,8 +10,6 @@ logger.logger.serializers.project = require("./app/js/infrastructure/LoggerSeria
 if Settings.sentry?.dsn?
 	logger.initializeErrorReporting(Settings.sentry.dsn)
 
-metrics = require("metrics-sharelatex")
-metrics.initialize("web")
 metrics.memory.monitor(logger)
 Server = require("./app/js/infrastructure/Server")
 

services/web/app/coffee/Features/Uploads/UploadsRouter.coffee

@@ -5,10 +5,17 @@ RateLimiterMiddlewear = require('../Security/RateLimiterMiddlewear')
 Settings = require('settings-sharelatex')
 multer = require('multer')
 
+try 
 	upload = multer(
 		dest: Settings.path.uploadFolder
 		limits: fileSize: Settings.maxUploadSize
 	)
+catch err
+	if err.message == "EEXIST"
+		logger.log uploadFolder:Settings.path.uploadFolder, "dir already exists, continuing"
+	else
+		logger.err err:err, "caught error from multer in uploads router"
+
 
 module.exports =
 	apply: (webRouter, apiRouter) ->

services/web/app/coffee/infrastructure/Server.coffee

@@ -7,7 +7,6 @@ crawlerLogger = require('./CrawlerLogger')
 expressLocals = require('./ExpressLocals')
 Router = require('../router')
 helmet = require "helmet"
-metrics.inc("startup")
 UserSessionsRedis = require('../Features/User/UserSessionsRedis')
 Csrf = require('./Csrf')
 
@@ -197,6 +196,8 @@ if enableWebRouter or notDefined(enableWebRouter)
 	app.use(webRouter)
 	app.use(ErrorController.handleError)
 
+metrics.injectMetricsRoute(webRouter)
+
 router = new Router(webRouter, privateApiRouter, publicApiRouter)
 
 module.exports =

services/web/config/settings.defaults.coffee

@@ -8,17 +8,17 @@ minutes = 60 * seconds
 
 # These credentials are used for authenticating api requests
 # between services that may need to go over public channels
-httpAuthUser = "sharelatex"
+httpAuthUser = process.env['WEB_API_USER'] or "sharelatex"
-httpAuthPass = "password"
+httpAuthPass = process.env['WEB_API_PASSWORD'] or "password"
 httpAuthUsers = {}
 httpAuthUsers[httpAuthUser] = httpAuthPass
 
-sessionSecret = "secret-please-change"
+sessionSecret = process.env['SESSION_SECRET'] or "secret-please-change"
 
 v1Api =
-	url: "http://#{process.env['V1_HOST'] or 'localhost'}:5000"
+	url: process.env['V1_API_URL'] or "http://#{process.env['V1_HOST'] or 'localhost'}:5000"
-	user: 'overleaf'
+	user: process.env['V1_API_USER'] or 'overleaf'
-	pass: 'password'
+	pass: process.env['V1_API_PASSWORD'] or 'password'
 
 module.exports = settings =
 
@@ -40,13 +40,13 @@ module.exports = settings =
 	# Databases
 	# ---------
 	mongo:
-		url : process.env['MONGO_URL'] || "mongodb://#{process.env['MONGO_HOST'] or '127.0.0.1'}/sharelatex"
+		url : process.env['MONGO_CONNECTION_STRING'] || process.env['MONGO_URL'] || "mongodb://#{process.env['MONGO_HOST'] or '127.0.0.1'}/sharelatex"
 
 	redis:
 		web:
 			host: process.env['REDIS_HOST'] || "localhost"
 			port: process.env['REDIS_PORT'] || "6379"
-			password: ""
+			password: process.env["REDIS_PASSWORD"] or ""
 
 		# websessions:
 		# 	cluster: [
@@ -81,7 +81,7 @@ module.exports = settings =
 		api:
 			host: process.env['REDIS_HOST'] || "localhost"
 			port: process.env['REDIS_PORT'] || "6379"
-			password: ""
+			password: process.env["REDIS_PASSWORD"] or ""
 
 	# Service locations
 	# -----------------
@@ -103,14 +103,15 @@ module.exports = settings =
 	# options incase you want to run some services on remote hosts.
 	apis:
 		web:
-			url: "http://#{process.env['WEB_HOST'] or 'localhost'}:#{webPort}"
+			url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}"
 			user: httpAuthUser
 			pass: httpAuthPass
 		documentupdater:
-			url : "http://#{process.env['DOCUPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}"
+			url : "http://#{process.env['DOCUPDATER_HOST'] or process.env['DOCUMENT_UPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}"
 		thirdPartyDataStore:
 			url : "http://#{process.env['TPDS_HOST'] or 'localhost'}:3002"
 			emptyProjectFlushDelayMiliseconds: 5 * seconds
+			dropboxApp: process.env['TPDS_DROPBOX_APP']
 		tags:
 			url :"http://#{process.env['TAGS_HOST'] or 'localhost'}:3012"
 		spelling:
@@ -144,11 +145,12 @@ module.exports = settings =
 		githubSync:
 			url: "http://#{process.env['GITHUB_SYNC_HOST'] or 'localhost'}:3022"
 		recurly:
-			privateKey: ""
+			privateKey: process.env['RECURLY_PRIVATE_KEY'] or ''
-			apiKey: ""
+			apiKey: process.env['RECURLY_API_KEY'] or ''
-			subdomain: ""
+			subdomain: process.env['RECURLY_SUBDOMAIN'] or ''
+			publicKey: process.env['RECURLY_PUBLIC_KEY'] or ''
 		geoIpLookup:
-			url: "http://#{process.env['GEOIP_HOST'] or 'localhost'}:8080/json/"
+			url: "http://#{process.env['GEOIP_HOST'] or process.env['FREEGEOIP_HOST'] or  'localhost'}:8080/json/"
 		realTime:
 			url: "http://#{process.env['REALTIME_HOST'] or 'localhost'}:3026"
 		contacts:
@@ -188,14 +190,18 @@ module.exports = settings =
 	# that are sent out, generated links, etc.
 	siteUrl : siteUrl = process.env['PUBLIC_URL'] or 'http://localhost:3000'
 
+
+	# Used to close the editor off to users
+	editorIsOpen: process.env['EDITOR_IS_OPEN'] or true
+	
 	# Optional separate location for websocket connections, if unset defaults to siteUrl.
 	wsUrl: process.env['WEBSOCKET_URL']
 
 	# cookie domain
 	# use full domain for cookies to only be accessible from that domain,
 	# replace subdomain with dot to have them accessible on all subdomains
-	# cookieDomain: ".sharelatex.dev"
+	cookieDomain: process.env['COOKIE_DOMAIN']
-	cookieName: "sharelatex.sid"
+	cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid"
 
 	# this is only used if cookies are used for clsi backend
 	#clsiCookieKey: "clsiserver"
@@ -350,6 +356,8 @@ module.exports = settings =
 	# public projects, /learn, /templates, about pages, etc.
 	allowPublicAccess: if process.env["SHARELATEX_ALLOW_PUBLIC_ACCESS"] == 'true' then true else false
 
+	enableHomepage: process.env["HOME_PAGE_ENABLED"] == 'true'
+
 	# editor should be open by default
 	editorIsOpen: if process.env["EDITOR_OPEN"] == 'false' then false else true
 
@@ -361,7 +369,7 @@ module.exports = settings =
 	# disablePerUserCompiles: true
 
 	# Domain the client (pdfjs) should download the compiled pdf from
-	# pdfDownloadDomain: "http://clsi-lb:3014"
+	pdfDownloadDomain: process.env["PDF_DOWNLOAD_DOMAIN"]  #"http://clsi-lb:3014"
 
 	# Maximum size of text documents in the real-time editing system.
 	max_doc_length: 2 * 1024 * 1024 # 2mb
@@ -372,8 +380,8 @@ module.exports = settings =
 		# If we ever need to write something to disk (e.g. incoming requests
 		# that need processing but may be too big for memory, then write
 		# them to disk here).
-		dumpFolder: Path.resolve __dirname + "/../data/dumpFolder"
+		dumpFolder: "/data/dumpFolder"
-		uploadFolder: Path.resolve __dirname + "/../data/uploads"
+		uploadFolder: "/data/uploads"
 
 	# Automatic Snapshots
 	# -------------------
@@ -398,6 +406,7 @@ module.exports = settings =
 		rateLimitSubject: process.env['SMOKE_TEST_RATE_LIMIT_SUBJECT'] or "127.0.0.1"
 
 	appName: process.env['APP_NAME'] or "ShareLaTeX (Community Edition)"
+
 	adminEmail: process.env['ADMIN_EMAIL'] or "placeholder@example.com"
 
 	brandPrefix: process.env['BRAND_PREFIX'] or "sl-" # Set to 'ol-' for overleaf styles

services/web/docker-compose.ci.yml

@@ -0,0 +1,55 @@
+version: "2"
+
+volumes:
+  data:
+
+services:
+
+  test_unit:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    user: node
+    command: npm run test:unit
+
+  test_acceptance:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    working_dir: /app
+    environment:
+      REDIS_HOST: redis
+      MONGO_URL: "mongodb://mongo/sharelatex"
+      SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
+      PROJECT_HISTORY_ENABLED: 'true'
+      ENABLED_LINKED_FILE_TYPES: 'url,project_file,project_output_file,mendeley,zotero'
+      LINKED_URL_PROXY: 'http://localhost:6543'
+      SHARELATEX_CONFIG: /app/test/acceptance/config/settings.test.coffee
+      NODE_ENV: production
+    user: root
+    depends_on:
+      - redis
+      - mongo
+    command: npm run test:acceptance:app
+
+  test_frontend:
+    build:
+      context: .
+      dockerfile: Dockerfile.frontend
+    volumes:
+      - .:/app
+    working_dir: /app
+    command: npm run test:frontend -- --single-run
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4.6
+    container_name: mongo-ci-$BUILD_NUMBER

services/web/docker-compose.yml

@@ -4,20 +4,31 @@ volumes:
   data:
 
 services:
-  test_acceptance:
+
+  test_unit:
     image: node:6.15.1
     volumes:
       - .:/app
     working_dir: /app
+    environment:
+      MOCHA_GREP: ${MOCHA_GREP}
+    command: npm run test:unit
+    user: node
+
+  test_acceptance:
+    image: node:6.15.1
+    volumes:
+      - .:/app:ro
+    working_dir: /app
     environment:
       REDIS_HOST: redis
       MONGO_URL: "mongodb://mongo/sharelatex"
       SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
       PROJECT_HISTORY_ENABLED: 'true'
-      ENABLED_LINKED_FILE_TYPES: 'url'
       LINKED_URL_PROXY: 'http://localhost:6543'
       ENABLED_LINKED_FILE_TYPES: 'url,project_file,project_output_file,mendeley,zotero'
       SHARELATEX_CONFIG: /app/test/acceptance/config/settings.test.coffee
+      MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: production
     depends_on:
       - redis

services/web/install_deps.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+WEBPACK_ENV=production make minify &
+make --no-print-directory format & 
+make --no-print-directory lint & 
+npm install git+https://github.com/sharelatex/translations-sharelatex.git#master &
+wait
+
+chmod -R 0755 /app/public
+chown -R node:node /app/public
+
+set -B
+
+rm -rf /app/data
+mkdir -p /app/data/{dumpFolder,logs,pdf,uploads,zippedProjects}
+chmod -R 0755 /app/data/
+chown -R node:node /app/data/

services/web/package.json

@@ -10,11 +10,12 @@
     "public": "./public"
   },
   "scripts": {
-    "test:acceptance:run_dir": "mocha --recursive --reporter spec --timeout 15000 --exit $@",
+    "test:acceptance:run_dir": "mocha --recursive --reporter spec --timeout 15000 --exit --grep=$MOCHA_GREP $@",
-    "test:unit": "npm -q run compile && bin/unit_test $@",
+    "test:unit": "npm -q run compile:app && bin/unit_test --grep=$MOCHA_GREP $@",
-    "test:unit:app": "npm -q run compile && bin/unit_test_app $@",
+    "test:unit:app": "npm -q run compile:app && bin/unit_test_app $@",
     "test:frontend": "karma start",
     "compile": "make compile",
+    "compile:app": "make compile_app",
     "start": "npm -q run compile && node $NODE_APP_OPTIONS app.js",
     "nodemon": "nodemon --config nodemon.json",
     "nodemon:frontend": "nodemon --config nodemon.frontend.json",
@@ -58,12 +59,12 @@
     "jsonwebtoken": "^8.0.1",
     "ldapjs": "^0.7.1",
     "lodash": "^4.13.1",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master",
+    "logger-sharelatex": "^1.6.0",
     "lynx": "0.1.1",
     "mailchimp-api-v3": "^1.12.0",
     "marked": "^0.3.5",
     "method-override": "^2.3.3",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.8.0",
+    "metrics-sharelatex": "^2.1.1",
     "minimist": "1.2.0",
     "mmmagic": "^0.5.2",
     "mocha": "^5.0.1",
@@ -90,7 +91,7 @@
     "pug": "^2.0.0-beta6",
     "react": "^15.4.2",
     "react-dom": "^15.4.2",
-    "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
+    "redis-sharelatex": "^1.0.5",
     "request": "^2.69.0",
     "requestretry": "^1.13.0",
     "requests": "^0.1.7",
@@ -98,7 +99,7 @@
     "rolling-rate-limiter": "git+https://github.com/ShaneKilkelly/rolling-rate-limiter.git#master",
     "sanitizer": "0.1.1",
     "sequelize": "^3.2.0",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
+    "settings-sharelatex": "^1.1.0",
     "sixpack-client": "^1.0.0",
     "temp": "^0.8.3",
     "underscore": "1.6.0",