github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-01-23 17:32:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

update build scripts, acceptence tests now pass. includes dockerfile 1.1.3

Browse source
This commit is contained in:
Henry Oswald 2018-05-23 15:00:46 +01:00
parent 49ce3e6917
commit 327fa79f1c
17 changed files with 239 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/real-time/.dockerignore Normal file
View file

@ -0,0 +1,9 @@
node_modules/*
gitrev
.git
.gitignore
.npm
.nvmrc
nodemon.json
app.js
**/js/*

2
services/real-time/.nvmrc
View file

@ -1 +1 @@
6.9.5
6.9.5

22
services/real-time/Dockerfile Normal file
View file

@ -0,0 +1,22 @@
FROM node:6.9.5 as app
WORKDIR /app
#wildcard as some files may not be in all repos
COPY package*.json npm-shrink*.json /app/
RUN npm install --quiet
COPY . /app
RUN npm run compile:all
FROM node:6.9.5
COPY --from=app /app /app
WORKDIR /app
USER node
CMD ["node","app.js"]

72
services/real-time/Jenkinsfile vendored
View file

@ -1,87 +1,67 @@
pipeline {
agent any
String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
environment {
HOME = "/tmp"
}
pipeline {
agent any
triggers {
pollSCM('* * * * *')
cron('@daily')
cron(cron_string)
}
stages {
stage('Install') {
agent {
docker {
image 'node:6.9.5'
args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
reuseNode true
}
}
stage('Build') {
steps {
// we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
// which does not exist in the container's /etc/passwd file, causing the clone to fail.
sh 'git config --global core.logallrefupdates false'
sh 'rm -fr node_modules'
sh 'npm install'
sh 'npm rebuild'
sh 'npm install --quiet grunt-cli'
sh 'make build'
}
}
stage('Compile and Test') {
agent {
docker {
image 'node:6.9.5'
args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
reuseNode true
}
}
stage('Unit Tests') {
steps {
sh 'node_modules/.bin/grunt install'
sh 'node_modules/.bin/grunt compile:acceptance_tests'
sh 'node_modules/.bin/grunt test:unit'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
}
}
stage('Acceptance Tests') {
steps {
sh 'docker pull sharelatex/acceptance-test-runner'
sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
}
stage('Package') {
stage('Package and publish build') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
sh 'make publish'
}
}
stage('Publish') {
stage('Publish build number') {
steps {
sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
}
}
}
}
post {
always {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
}
failure {
mail(from: "${EMAIL_ALERT_FROM}",
to: "${EMAIL_ALERT_TO}",
mail(from: "${EMAIL_ALERT_FROM}",
to: "${EMAIL_ALERT_TO}",
subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
body: "Build: ${BUILD_URL}")
}
}
// The options directive is for configuration that applies to the whole job.
options {
// we'd like to make sure remove old builds, so we don't fill up our storage!
buildDiscarder(logRotator(numToKeepStr:'50'))
// And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
timeout(time: 30, unit: 'MINUTES')
}

42
services/real-time/Makefile Normal file
View file

@ -0,0 +1,42 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.3
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
PROJECT_NAME = real-time
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
clean:
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
rm -rf test/acceptance/js
test: test_unit test_acceptance
test_unit:
@[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
@[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
test_clean:
$(DOCKER_COMPOSE) down -v -t 0
test_acceptance_pre_run:
@[ ! -f test/acceptance/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
build:
docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
publish:
docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish

2
services/real-time/app.coffee
View file

@ -8,6 +8,8 @@ Settings = require "settings-sharelatex"
if Settings.sentry?.dsn?
logger.initializeErrorReporting(Settings.sentry.dsn)
console.log "dasdsadasdsadsadsadsad"
console.log Settings.redis.websessions
sessionRedisClient = redis.createClient(Settings.redis.websessions)
RedisStore = require('connect-redis')(session)

2
services/real-time/app/coffee/ConnectedUsersManager.coffee
View file

@ -5,6 +5,8 @@ redis = require("redis-sharelatex")
rclient = redis.createClient(Settings.redis.realtime)
Keys = Settings.redis.realtime.key_schema
console.log Settings.redis.realtime, "REALTIME"
ONE_HOUR_IN_S = 60 * 60
ONE_DAY_IN_S = ONE_HOUR_IN_S * 24
FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4

2
services/real-time/app/coffee/DocumentUpdaterController.coffee
View file

@ -4,6 +4,8 @@ redis = require("redis-sharelatex")
rclient = redis.createClient(settings.redis.documentupdater)
SafeJsonParse = require "./SafeJsonParse"
console.log "REDIS", settings.redis
MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb
module.exports = DocumentUpdaterController =

6
services/real-time/config/settings.defaults.coffee
View file

@ -1,7 +1,7 @@
module.exports =
redis:
realtime:
host: process.env['REDIS_HOST'] or "localhost"
host: process.env['REDIS_HOST'] or "localhostssss"
port: "6379"
password: ""
key_schema:
@ -9,14 +9,14 @@ module.exports =
connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}"
documentupdater:
host: process.env['REDIS_HOST'] or "localhost"
host: process.env['REDIS_HOST'] or "localhostssss"
port: "6379"
password: ""
key_schema:
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
websessions:
host: process.env['REDIS_HOST'] or "localhost"
host: process.env['REDIS_HOST'] or "localhostssss"
port: "6379"
password: ""

32
services/real-time/docker-compose.ci.yml Normal file
View file

@ -0,0 +1,32 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.3
version: "2"
services:
test_unit:
image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
command: npm run test:unit:_run
test_acceptance:
build: .
image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
depends_on:
- mongo
- redis
user: node
command: npm run test:acceptance:_run
redis:
image: redis
mongo:
image: mongo:3.4

41
services/real-time/docker-compose.yml Normal file
View file

@ -0,0 +1,41 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.3
version: "2"
services:
test_unit:
build: .
volumes:
- .:/app
working_dir: /app
environment:
MOCHA_GREP: ${MOCHA_GREP}
command: npm run test:unit
user: node
test_acceptance:
build: .
volumes:
- .:/app
working_dir: /app
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
user: node
depends_on:
- mongo
- redis
command: npm run test:acceptance
redis:
image: redis
mongo:
image: mongo:3.4

19
services/real-time/nodemon.json Normal file
View file

@ -0,0 +1,19 @@
{
"ignore": [
".git",
"node_modules/"
],
"verbose": true,
"legacyWatch": true,
"execMap": {
"js": "npm run start"
},
"watch": [
"app/coffee/",
"app.coffee",
"config/"
],
"ext": "coffee"
}

13
services/real-time/package.json
View file

@ -8,8 +8,16 @@
"url": "https://github.com/sharelatex/real-time-sharelatex.git"
},
"scripts": {
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
"start": "npm run compile:app && node app.js"
"compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
"compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
"compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
"nodemon": "nodemon --config nodemon.json"
},
"dependencies": {
"async": "^0.9.0",
@ -42,6 +50,7 @@
"grunt-shell": "~0.7.0",
"sandboxed-module": "~0.3.0",
"sinon": "~1.5.2",
"mocha": "^4.0.1",
"uid-safe": "^1.0.1",
"timekeeper": "0.0.4"
}

2
services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee
View file

@ -8,7 +8,7 @@ FixturesManager = require "./helpers/FixturesManager"
settings = require "settings-sharelatex"
redis = require "redis-sharelatex"
rclient = redis.createClient(settings.redis.web)
rclient = redis.createClient(settings.redis.websessions)
describe "applyOtUpdate", ->
before ->

2
services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee
View file

@ -10,7 +10,7 @@ async = require "async"
settings = require "settings-sharelatex"
redis = require "redis-sharelatex"
rclient = redis.createClient(settings.redis.web)
rclient = redis.createClient(settings.redis.websessions)
describe "receiveUpdate", ->
before (done) ->

2
services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee
View file

@ -4,7 +4,7 @@ io = require("socket.io-client")
request = require "request"
Settings = require "settings-sharelatex"
redis = require "redis-sharelatex"
rclient = redis.createClient(Settings.redis.web)
rclient = redis.createClient(Settings.redis.websessions)
uid = require('uid-safe').sync
signature = require("cookie-signature")

24
services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee Normal file
View file

@ -0,0 +1,24 @@
app = require('../../../../app')
require("logger-sharelatex").logger.level("info")
logger = require("logger-sharelatex")
Settings = require("settings-sharelatex")
module.exports =
running: false
initing: false
callbacks: []
ensureRunning: (callback = (error) ->) ->
if @running
return callback()
else if @initing
@callbacks.push callback
else
@initing = true
@callbacks.push callback
app.listen Settings.internal?.realtime?.port, "localhost", (error) =>
throw error if error?
@running = true
logger.log("clsi running in dev mode")
for callback in @callbacks
callback()