update build script and add load balancer agent

This commit is contained in:
Henry Oswald 2018-03-29 12:12:29 +01:00
parent 41e39458d1
commit 551e8d36b4
13 changed files with 184 additions and 3628 deletions

35
services/clsi/.viminfo Normal file
View file

@ -0,0 +1,35 @@
# This viminfo file was generated by Vim 7.4.
# You may edit it if you're careful!
# Value of 'encoding' when this file was written
*encoding=latin1
# hlsearch on (H) or off (h):
~h
# Command Line History (newest to oldest):
:x
# Search String History (newest to oldest):
# Expression History (newest to oldest):
# Input Line History (newest to oldest):
# Input Line History (newest to oldest):
# Registers:
# File marks:
'0 1 0 ~/hello
# Jumplist (newest first):
-' 1 0 ~/hello
# History of marks within files (newest to oldest):
> ~/hello
" 1 0
^ 1 1
. 1 0
+ 1 0

View file

@ -14,10 +14,6 @@ FROM node:6.13.0
COPY --from=app /app /app
WORKDIR /app
# All app and node_modules will be owned by root.
# The app will run as the 'node' user, and so not have write permissions
# on any files it doesn't need.
RUN ./install_deps.sh
ENTRYPOINT ["/bin/sh", "entrypoint.sh"]

View file

@ -21,13 +21,13 @@ clean:
test: test_unit test_acceptance
test_unit:
@[ -d test/unit ] && $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS} || echo "clsi has no unit tests"
@[ ! -d test/unit ] && echo "clsi has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS}
test_acceptance: test_clean # clear the database before each acceptance test run
@[ -d test/acceptance ] && $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS} || echo "clsi has no acceptance tests"
@[ ! -d test/acceptance ] && echo "clsi has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS}
test_clean:
$(DOCKER_COMPOSE) down -t 0
$(DOCKER_COMPOSE) down -t -v 0
build:
docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .

View file

@ -169,13 +169,54 @@ app.use (error, req, res, next) ->
logger.error {err: error, url: req.url}, "server error"
res.sendStatus(error?.statusCode || 500)
net = require "net"
os = require "os"
STATE = "up"
server = net.createServer (socket) ->
socket.on "error", (err)->
if err.code == "ECONNRESET"
# this always comes up, we don't know why
return
logger.err err:err, "error with socket on load check"
socket.destroy()
if STATE == "up" and settings.load_balancer_agent.report_load
currentLoad = os.loadavg()[0]
# staging clis's have 1 cpu core only
if os.cpus().length == 1
availableWorkingCpus = 1
else
availableWorkingCpus = os.cpus().length - 1
freeLoad = availableWorkingCpus - currentLoad
freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
if freeLoadPercentage <= 0
freeLoadPercentage = 1 # when its 0 the server is set to drain and will move projects to different servers
socket.write("up, #{freeLoadPercentage}%\n", "ASCII")
socket.end()
else
socket.write("#{STATE}\n", "ASCII")
socket.end()
port = (Settings.internal?.clsi?.port or 3013)
host = (Settings.internal?.clsi?.host or "localhost")
load_port = settings.internal.clsi.load_port or 3048
if !module.parent # Called directly
app.listen port, host, (error) ->
logger.info "CLSI starting up, listening on #{host}:#{port}"
server.listen load_port, host, (error) ->
throw error if error?
logger.info "Load agent listening on load port #{load_port}"
module.exports = app

5
services/clsi/debug Executable file
View file

@ -0,0 +1,5 @@
#!/bin/bash
echo "hello world"
sleep 3
echo "awake"
/opt/synctex pdf /compile/output.pdf 1 100 200

View file

@ -8,7 +8,8 @@ version: "2"
services:
test_unit:
image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
command: npm run test:unit:_run
user: node
entrypoint: npm run test:unit:_run
test_acceptance:
build: .
@ -22,7 +23,8 @@ services:
depends_on:
- redis
- mongo
command: npm run test:acceptance:_run
user: node
entrypoint: npm run test:acceptance:_run
redis:
image: redis

View file

@ -11,7 +11,8 @@ services:
volumes:
- .:/app
working_dir: /app
command: npm run test:unit
user: node
entrypoint: npm run test:unit
test_acceptance:
build: .
@ -24,10 +25,11 @@ services:
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
user: node
depends_on:
- redis
- mongo
command: npm run test:acceptance
entrypoint: npm run test:acceptance
redis:
image: redis

View file

@ -11,4 +11,6 @@ mkdir -p /app/compiles
chown -R node:node /app/compiles
./bin/install_texlive_gce.sh
echo "HELOOOo"
echo "$@"
exec runuser -u node "$@"

View file

@ -10,7 +10,8 @@
},
"watch": [
"app/coffee/",
"app.coffee"
"app.coffee",
"config/"
],
"ext": "coffee"
}

File diff suppressed because it is too large Load diff

View file

@ -14,9 +14,9 @@
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
"test:unit:_run": "mocha --recursive --exit --reporter spec $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile, skipping'",
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile, skipping'",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
"nodemon": "nodemon --config nodemon.json",
"compile:smoke_tests": "[ -e test/smoke ] && coffee -o test/smoke/js -c test/smoke/coffee || echo 'No smoke tests to compile, skipping'"
},

View file

@ -0,0 +1,3 @@
FROM quay.io/sharelatex/texlive-full:2017.1
# RUN usermod -u 1001 tex

View file

@ -0,0 +1,34 @@
include /etc/firejail/disable-common.inc
include /etc/firejail/disable-devel.inc
# include /etc/firejail/disable-mgmt.inc ## removed in 0.9.40
# include /etc/firejail/disable-secret.inc ## removed in 0.9.40
read-only /bin
blacklist /boot
blacklist /dev
read-only /etc
blacklist /home # blacklisted for synctex
read-only /lib
read-only /lib64
blacklist /media
blacklist /mnt
blacklist /opt
blacklist /root
read-only /run
blacklist /sbin
blacklist /selinux
blacklist /src
blacklist /sys
read-only /usr
caps.drop all
noroot
nogroups
net none
private-tmp
private-dev
shell none
seccomp
nonewprivs