Merge pull request #1471 from sharelatex/ho-docker2

Move to docker image based builds

GitOrigin-RevId: 992aeaf69bfea4d1847a07c457575a63985d4407
This commit is contained in:
Henry Oswald 2019-02-19 14:51:34 +00:00 committed by James Allen
parent 57450bb0fb
commit d55b38cc05
12 changed files with 1097 additions and 454 deletions

23
services/web/Dockerfile Normal file
View file

@ -0,0 +1,23 @@
FROM node:6.15.1 as app
WORKDIR /app
#wildcard as some files may not be in all repos
COPY package.json npm-shrinkwrap.json /app/
RUN npm install --quiet
COPY . /app
RUN make compile_full
FROM node:6.15.1
COPY --from=app /app /app
WORKDIR /app
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
USER node
CMD ["node", "--expose-gc", "app.js"]

View file

@ -1,11 +1,20 @@
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
GIT_SHA ?= $(shell git rev-parse HEAD)
PROJECT_NAME = web
DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
MODULE_DIRS := $(shell find modules -mindepth 1 -maxdepth 1 -type d -not -name '.git' )
MODULE_MAKEFILES := $(MODULE_DIRS:=/Makefile)
COFFEE := node_modules/.bin/coffee -m $(COFFEE_OPTIONS)
COFFEE := node_modules/.bin/coffee $(COFFEE_OPTIONS)
BABEL := node_modules/.bin/babel
GRUNT := node_modules/.bin/grunt
@ -37,8 +46,6 @@ CSS_OL_IEEE_FILE := public/stylesheets/ieee-style.css
CSS_FILES := $(CSS_SL_FILE) $(CSS_OL_FILE) $(CSS_OL_LIGHT_FILE) $(CSS_OL_IEEE_FILE)
SENTRY_TEMPLATE := app/views/sentry.pug
# The automatic variable $(@D) is the target directory name
app.js: app.coffee
$(COFFEE) --compile -o $(@D) $<
@ -119,7 +126,9 @@ minify_css: $(CSS_FILES)
minify_es:
npm -q run webpack:production
compile: $(JS_FILES) $(OUTPUT_SRC_FILES) css public/js/main.js public/js/ide.js
compile: compile_app $(OUTPUT_SRC_FILES) css public/js/main.js public/js/ide.js
compile_app: $(JS_FILES)
@$(MAKE) compile_modules
compile_full:
@ -198,19 +207,20 @@ clean_css:
rm -f public/stylesheets/*.css*
clean_ci:
docker-compose down -v -t 0
$(DOCKER_COMPOSE) down -v -t 0
test: test_unit test_frontend test_acceptance
test_unit:
npm -q run test:unit -- ${MOCHA_ARGS}
@[ ! -d test/unit ] && echo "web has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
test_unit_app:
npm -q run test:unit:app -- ${MOCHA_ARGS}
test_frontend: test_clean # stop service
$(MAKE) compile
docker-compose ${DOCKER_COMPOSE_FLAGS} up --exit-code-from test_frontend --abort-on-container-exit test_frontend
test_frontend: test_clean compile test_frontend_run
test_frontend_run:
$(DOCKER_COMPOSE) up test_frontend
test_acceptance: compile test_acceptance_app_run test_acceptance_modules_run
@ -218,9 +228,10 @@ test_acceptance_app: compile test_acceptance_app_run
test_acceptance_module: compile test_acceptance_module_run
test_acceptance_run: test_acceptance_app_run test_acceptance_modules_run
test_acceptance_app_run: test_clean
@set -e; \
docker-compose ${DOCKER_COMPOSE_FLAGS} run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} test/acceptance/js
$(DOCKER_COMPOSE) run --rm test_acceptance npm -q run test:acceptance:run_dir -- ${MOCHA_ARGS} test/acceptance/js
test_acceptance_modules_run:
@set -e; \
@ -237,7 +248,7 @@ test_acceptance_module_run: $(MODULE_MAKEFILES) test_clean
fi
test_clean:
docker-compose ${DOCKER_COMPOSE_FLAGS} down -v -t 0
$(DOCKER_COMPOSE) down -v -t 0
ci:
MOCHA_ARGS="--reporter tap" \
@ -252,10 +263,17 @@ format_fix:
lint:
npm -q run lint
version:
sed -i.original -e "s/@@COMMIT@@/${GIT_SHA}/g" $(SENTRY_TEMPLATE)
sed -i.original -e "s/@@RELEASE@@/${BUILD_NUMBER}/g" $(SENTRY_TEMPLATE)
rm $(SENTRY_TEMPLATE).original
build:
docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
.
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
tar:
$(DOCKER_COMPOSE) up tar
.PHONY:
all add install update test test_unit test_frontend test_acceptance \

View file

@ -1,5 +1,6 @@
MODULE_NAME := $(notdir $(shell pwd))
MODULE_DIR := modules/$(MODULE_NAME)
PROJECT_NAME = web
COFFEE := ../../node_modules/.bin/coffee
BABEL := ../../node_modules/.bin/babel
@ -17,8 +18,15 @@ IDE_TEST_OUTPUT_FILES := $(subst test/unit_frontend/src/ide,../../test/unit_fron
MAIN_SRC_FILES := $(shell [ -e public/src/main ] && find public/src/main -name '*.js')
MAIN_OUTPUT_FILES := $(subst public/src/main,../../public/js/main/$(MODULE_NAME),$(MAIN_SRC_FILES))
DOCKER_COMPOSE_FLAGS := -f $(MODULE_DIR)/docker-compose.yml
DOCKER_COMPOSE := cd ../../ && MODULE_DIR=$(MODULE_DIR) docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS}
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
DOCKER_COMPOSE_MODULE_FLAGS := ${DOCKER_COMPOSE_FLAGS} -f $(MODULE_DIR)/docker-compose.yml
DOCKER_COMPOSE := cd ../../ && \
MODULE_DIR=$(MODULE_DIR) \
BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
docker-compose ${DOCKER_COMPOSE_MODULE_FLAGS}
app/js/%.js: app/coffee/%.coffee
@mkdir -p $(dir $@)

View file

@ -1,6 +1,8 @@
metrics = require("metrics-sharelatex")
metrics.initialize(process.env['METRICS_APP_NAME'] or "web")
Settings = require('settings-sharelatex')
logger = require 'logger-sharelatex'
logger.initialize("web-sharelatex")
logger.initialize("web")
logger.logger.serializers.user = require("./app/js/infrastructure/LoggerSerializers").user
logger.logger.serializers.docs = require("./app/js/infrastructure/LoggerSerializers").docs
logger.logger.serializers.files = require("./app/js/infrastructure/LoggerSerializers").files
@ -8,8 +10,6 @@ logger.logger.serializers.project = require("./app/js/infrastructure/LoggerSeria
if Settings.sentry?.dsn?
logger.initializeErrorReporting(Settings.sentry.dsn)
metrics = require("metrics-sharelatex")
metrics.initialize("web")
metrics.memory.monitor(logger)
Server = require("./app/js/infrastructure/Server")

View file

@ -5,10 +5,17 @@ RateLimiterMiddlewear = require('../Security/RateLimiterMiddlewear')
Settings = require('settings-sharelatex')
multer = require('multer')
upload = multer(
dest: Settings.path.uploadFolder
limits: fileSize: Settings.maxUploadSize
)
try
upload = multer(
dest: Settings.path.uploadFolder
limits: fileSize: Settings.maxUploadSize
)
catch err
if err.message == "EEXIST"
logger.log uploadFolder:Settings.path.uploadFolder, "dir already exists, continuing"
else
logger.err err:err, "caught error from multer in uploads router"
module.exports =
apply: (webRouter, apiRouter) ->

View file

@ -7,7 +7,6 @@ crawlerLogger = require('./CrawlerLogger')
expressLocals = require('./ExpressLocals')
Router = require('../router')
helmet = require "helmet"
metrics.inc("startup")
UserSessionsRedis = require('../Features/User/UserSessionsRedis')
Csrf = require('./Csrf')
@ -197,6 +196,8 @@ if enableWebRouter or notDefined(enableWebRouter)
app.use(webRouter)
app.use(ErrorController.handleError)
metrics.injectMetricsRoute(webRouter)
router = new Router(webRouter, privateApiRouter, publicApiRouter)
module.exports =

View file

@ -8,17 +8,17 @@ minutes = 60 * seconds
# These credentials are used for authenticating api requests
# between services that may need to go over public channels
httpAuthUser = "sharelatex"
httpAuthPass = "password"
httpAuthUser = process.env['WEB_API_USER'] or "sharelatex"
httpAuthPass = process.env['WEB_API_PASSWORD'] or "password"
httpAuthUsers = {}
httpAuthUsers[httpAuthUser] = httpAuthPass
sessionSecret = "secret-please-change"
sessionSecret = process.env['SESSION_SECRET'] or "secret-please-change"
v1Api =
url: "http://#{process.env['V1_HOST'] or 'localhost'}:5000"
user: 'overleaf'
pass: 'password'
url: process.env['V1_API_URL'] or "http://#{process.env['V1_HOST'] or 'localhost'}:5000"
user: process.env['V1_API_USER'] or 'overleaf'
pass: process.env['V1_API_PASSWORD'] or 'password'
module.exports = settings =
@ -40,13 +40,13 @@ module.exports = settings =
# Databases
# ---------
mongo:
url : process.env['MONGO_URL'] || "mongodb://#{process.env['MONGO_HOST'] or '127.0.0.1'}/sharelatex"
url : process.env['MONGO_CONNECTION_STRING'] || process.env['MONGO_URL'] || "mongodb://#{process.env['MONGO_HOST'] or '127.0.0.1'}/sharelatex"
redis:
web:
host: process.env['REDIS_HOST'] || "localhost"
port: process.env['REDIS_PORT'] || "6379"
password: ""
password: process.env["REDIS_PASSWORD"] or ""
# websessions:
# cluster: [
@ -81,7 +81,7 @@ module.exports = settings =
api:
host: process.env['REDIS_HOST'] || "localhost"
port: process.env['REDIS_PORT'] || "6379"
password: ""
password: process.env["REDIS_PASSWORD"] or ""
# Service locations
# -----------------
@ -103,14 +103,15 @@ module.exports = settings =
# options incase you want to run some services on remote hosts.
apis:
web:
url: "http://#{process.env['WEB_HOST'] or 'localhost'}:#{webPort}"
url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}"
user: httpAuthUser
pass: httpAuthPass
documentupdater:
url : "http://#{process.env['DOCUPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}"
url : "http://#{process.env['DOCUPDATER_HOST'] or process.env['DOCUMENT_UPDATER_HOST'] or 'localhost'}:#{docUpdaterPort}"
thirdPartyDataStore:
url : "http://#{process.env['TPDS_HOST'] or 'localhost'}:3002"
emptyProjectFlushDelayMiliseconds: 5 * seconds
dropboxApp: process.env['TPDS_DROPBOX_APP']
tags:
url :"http://#{process.env['TAGS_HOST'] or 'localhost'}:3012"
spelling:
@ -144,11 +145,12 @@ module.exports = settings =
githubSync:
url: "http://#{process.env['GITHUB_SYNC_HOST'] or 'localhost'}:3022"
recurly:
privateKey: ""
apiKey: ""
subdomain: ""
privateKey: process.env['RECURLY_PRIVATE_KEY'] or ''
apiKey: process.env['RECURLY_API_KEY'] or ''
subdomain: process.env['RECURLY_SUBDOMAIN'] or ''
publicKey: process.env['RECURLY_PUBLIC_KEY'] or ''
geoIpLookup:
url: "http://#{process.env['GEOIP_HOST'] or 'localhost'}:8080/json/"
url: "http://#{process.env['GEOIP_HOST'] or process.env['FREEGEOIP_HOST'] or 'localhost'}:8080/json/"
realTime:
url: "http://#{process.env['REALTIME_HOST'] or 'localhost'}:3026"
contacts:
@ -188,14 +190,18 @@ module.exports = settings =
# that are sent out, generated links, etc.
siteUrl : siteUrl = process.env['PUBLIC_URL'] or 'http://localhost:3000'
# Used to close the editor off to users
editorIsOpen: process.env['EDITOR_IS_OPEN'] or true
# Optional separate location for websocket connections, if unset defaults to siteUrl.
wsUrl: process.env['WEBSOCKET_URL']
# cookie domain
# use full domain for cookies to only be accessible from that domain,
# replace subdomain with dot to have them accessible on all subdomains
# cookieDomain: ".sharelatex.dev"
cookieName: "sharelatex.sid"
cookieDomain: process.env['COOKIE_DOMAIN']
cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid"
# this is only used if cookies are used for clsi backend
#clsiCookieKey: "clsiserver"
@ -350,6 +356,8 @@ module.exports = settings =
# public projects, /learn, /templates, about pages, etc.
allowPublicAccess: if process.env["SHARELATEX_ALLOW_PUBLIC_ACCESS"] == 'true' then true else false
enableHomepage: process.env["HOME_PAGE_ENABLED"] == 'true'
# editor should be open by default
editorIsOpen: if process.env["EDITOR_OPEN"] == 'false' then false else true
@ -361,7 +369,7 @@ module.exports = settings =
# disablePerUserCompiles: true
# Domain the client (pdfjs) should download the compiled pdf from
# pdfDownloadDomain: "http://clsi-lb:3014"
pdfDownloadDomain: process.env["PDF_DOWNLOAD_DOMAIN"] #"http://clsi-lb:3014"
# Maximum size of text documents in the real-time editing system.
max_doc_length: 2 * 1024 * 1024 # 2mb
@ -372,8 +380,8 @@ module.exports = settings =
# If we ever need to write something to disk (e.g. incoming requests
# that need processing but may be too big for memory, then write
# them to disk here).
dumpFolder: Path.resolve __dirname + "/../data/dumpFolder"
uploadFolder: Path.resolve __dirname + "/../data/uploads"
dumpFolder: "/data/dumpFolder"
uploadFolder: "/data/uploads"
# Automatic Snapshots
# -------------------
@ -398,6 +406,7 @@ module.exports = settings =
rateLimitSubject: process.env['SMOKE_TEST_RATE_LIMIT_SUBJECT'] or "127.0.0.1"
appName: process.env['APP_NAME'] or "ShareLaTeX (Community Edition)"
adminEmail: process.env['ADMIN_EMAIL'] or "placeholder@example.com"
brandPrefix: process.env['BRAND_PREFIX'] or "sl-" # Set to 'ol-' for overleaf styles
@ -536,4 +545,4 @@ module.exports = settings =
'table': [ 'border', 'class', 'id', 'style' ]
'td': [ 'colspan', 'rowspan', 'headers' ]
'th': [ 'abbr', 'headers', 'colspan', 'rowspan', 'scope', 'sorted' ]
'video': [ 'alt', 'class', 'controls', 'height', 'width' ]
'video': [ 'alt', 'class', 'controls', 'height', 'width' ]

View file

@ -0,0 +1,55 @@
version: "2"
volumes:
data:
services:
test_unit:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
command: npm run test:unit
test_acceptance:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
working_dir: /app
environment:
REDIS_HOST: redis
MONGO_URL: "mongodb://mongo/sharelatex"
SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
PROJECT_HISTORY_ENABLED: 'true'
ENABLED_LINKED_FILE_TYPES: 'url,project_file,project_output_file,mendeley,zotero'
LINKED_URL_PROXY: 'http://localhost:6543'
SHARELATEX_CONFIG: /app/test/acceptance/config/settings.test.coffee
NODE_ENV: production
user: root
depends_on:
- redis
- mongo
command: npm run test:acceptance:app
test_frontend:
build:
context: .
dockerfile: Dockerfile.frontend
volumes:
- .:/app
working_dir: /app
command: npm run test:frontend -- --single-run
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
mongo:
image: mongo:3.4.6
container_name: mongo-ci-$BUILD_NUMBER

View file

@ -4,20 +4,31 @@ volumes:
data:
services:
test_acceptance:
test_unit:
image: node:6.15.1
volumes:
- .:/app
working_dir: /app
environment:
MOCHA_GREP: ${MOCHA_GREP}
command: npm run test:unit
user: node
test_acceptance:
image: node:6.15.1
volumes:
- .:/app:ro
working_dir: /app
environment:
REDIS_HOST: redis
MONGO_URL: "mongodb://mongo/sharelatex"
SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
PROJECT_HISTORY_ENABLED: 'true'
ENABLED_LINKED_FILE_TYPES: 'url'
LINKED_URL_PROXY: 'http://localhost:6543'
ENABLED_LINKED_FILE_TYPES: 'url,project_file,project_output_file,mendeley,zotero'
SHARELATEX_CONFIG: /app/test/acceptance/config/settings.test.coffee
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: production
depends_on:
- redis

View file

@ -0,0 +1,17 @@
#!/bin/bash
WEBPACK_ENV=production make minify &
make --no-print-directory format &
make --no-print-directory lint &
npm install git+https://github.com/sharelatex/translations-sharelatex.git#master &
wait
chmod -R 0755 /app/public
chown -R node:node /app/public
set -B
rm -rf /app/data
mkdir -p /app/data/{dumpFolder,logs,pdf,uploads,zippedProjects}
chmod -R 0755 /app/data/
chown -R node:node /app/data/

File diff suppressed because it is too large Load diff

View file

@ -10,11 +10,12 @@
"public": "./public"
},
"scripts": {
"test:acceptance:run_dir": "mocha --recursive --reporter spec --timeout 15000 --exit $@",
"test:unit": "npm -q run compile && bin/unit_test $@",
"test:unit:app": "npm -q run compile && bin/unit_test_app $@",
"test:acceptance:run_dir": "mocha --recursive --reporter spec --timeout 15000 --exit --grep=$MOCHA_GREP $@",
"test:unit": "npm -q run compile:app && bin/unit_test --grep=$MOCHA_GREP $@",
"test:unit:app": "npm -q run compile:app && bin/unit_test_app $@",
"test:frontend": "karma start",
"compile": "make compile",
"compile:app": "make compile_app",
"start": "npm -q run compile && node $NODE_APP_OPTIONS app.js",
"nodemon": "nodemon --config nodemon.json",
"nodemon:frontend": "nodemon --config nodemon.frontend.json",
@ -58,12 +59,12 @@
"jsonwebtoken": "^8.0.1",
"ldapjs": "^0.7.1",
"lodash": "^4.13.1",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master",
"logger-sharelatex": "^1.6.0",
"lynx": "0.1.1",
"mailchimp-api-v3": "^1.12.0",
"marked": "^0.3.5",
"method-override": "^2.3.3",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.8.0",
"metrics-sharelatex": "^2.1.1",
"minimist": "1.2.0",
"mmmagic": "^0.5.2",
"mocha": "^5.0.1",
@ -90,7 +91,7 @@
"pug": "^2.0.0-beta6",
"react": "^15.4.2",
"react-dom": "^15.4.2",
"redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
"redis-sharelatex": "^1.0.5",
"request": "^2.69.0",
"requestretry": "^1.13.0",
"requests": "^0.1.7",
@ -98,7 +99,7 @@
"rolling-rate-limiter": "git+https://github.com/ShaneKilkelly/rolling-rate-limiter.git#master",
"sanitizer": "0.1.1",
"sequelize": "^3.2.0",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
"settings-sharelatex": "^1.1.0",
"sixpack-client": "^1.0.0",
"temp": "^0.8.3",
"underscore": "1.6.0",