github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-06 09:51:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #54 from sharelatex/ho-docker

Browse source 
Update docker build process
This commit is contained in:
Henry Oswald 2019-02-06 13:42:03 +00:00 • committed by GitHub
commit 01d3f4bff4
25 changed files with 2581 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/track-changes/.dockerignore Normal file
View file

@ -0,0 +1,9 @@
node_modules/*
gitrev
.git
.gitignore
.npm
.nvmrc
nodemon.json
app.js
**/js/*

38
services/track-changes/.github/ISSUE_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,38 @@
<!-- BUG REPORT TEMPLATE -->
## Steps to Reproduce
<!-- Describe the steps leading up to when / where you found the bug. -->
<!-- Screenshots may be helpful here. -->
1.
2.
3.
## Expected Behaviour
<!-- What should have happened when you completed the steps above? -->
## Observed Behaviour
<!-- What actually happened when you completed the steps above? -->
<!-- Screenshots may be helpful here. -->
## Context
<!-- How has this issue affected you? What were you trying to accomplish? -->
## Technical Info
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
* URL:
* Browser Name and version:
* Operating System and version (desktop or mobile):
* Signed in as:
* Project and/or file:
## Analysis
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
## Who Needs to Know?
<!-- If you want to bring this to the attention of particular people, @-mention them below. -->
<!-- If a user reported this bug and should be notified when it is fixed, provide the Front conversation link. -->
-
-

45
services/track-changes/.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,45 @@
<!-- Please review https://github.com/overleaf/write_latex/blob/master/.github/CONTRIBUTING.md for guidance on what is expected in each section. -->
### Description
#### Screenshots
#### Related Issues / PRs
### Review
#### Potential Impact
#### Manual Testing Performed
- [ ]
- [ ]
#### Accessibility
### Deployment
#### Deployment Checklist
- [ ] Update documentation not included in the PR (if any)
- [ ]
#### Metrics and Monitoring
#### Who Needs to Know?

1
services/track-changes/.gitignore vendored
View file

@ -2,6 +2,7 @@
node_modules/
app/js
app.js
**/*.map
test/unit/js
test/acceptance/js
forever/

22
services/track-changes/Dockerfile Normal file
View file

@ -0,0 +1,22 @@
FROM node:6.11.2 as app
WORKDIR /app
#wildcard as some files may not be in all repos
COPY package*.json npm-shrink*.json /app/
RUN npm install --quiet
COPY . /app
RUN npm run compile:all
FROM node:6.11.2
COPY --from=app /app /app
WORKDIR /app
USER node
CMD ["node", "--expose-gc", "app.js"]

117
services/track-changes/Gruntfile.coffee
View file

@ -1,117 +0,0 @@
module.exports = (grunt) ->
grunt.loadNpmTasks 'grunt-contrib-coffee'
grunt.loadNpmTasks 'grunt-contrib-clean'
grunt.loadNpmTasks 'grunt-mocha-test'
grunt.loadNpmTasks 'grunt-available-tasks'
grunt.loadNpmTasks 'grunt-execute'
grunt.loadNpmTasks 'grunt-bunyan'
grunt.loadNpmTasks 'grunt-forever'
grunt.initConfig
forever:
app:
options:
index: "app.js"
execute:
app:
src: "app.js"
bunyan:
strict: false
coffee:
app_dir:
expand: true,
flatten: false,
cwd: 'app/coffee',
src: ['**/*.coffee'],
dest: 'app/js/',
ext: '.js'
app:
src: 'app.coffee'
dest: 'app.js'
acceptance_tests:
expand: true,
flatten: false,
cwd: 'test/acceptance/coffee',
src: ['**/*.coffee'],
dest: 'test/acceptance/js/',
ext: '.js'
unit_tests:
expand: true,
flatten: false,
cwd: 'test/unit/coffee',
src: ['**/*.coffee'],
dest: 'test/unit/js/',
ext: '.js'
clean:
app: ["app/js"]
acceptance_tests: ["test/acceptance/js"]
mochaTest:
unit:
src: ["test/unit/js/#{grunt.option("feature") or "**"}/*.js"]
options:
reporter: grunt.option('reporter') or 'spec'
grep: grunt.option("grep")
acceptance:
src: ["test/acceptance/js/**/*.js"]
options:
reporter: grunt.option('reporter') or 'spec'
grep: grunt.option("grep")
timeout: 10000
availabletasks:
tasks:
options:
filter: 'exclude',
tasks: [
'coffee'
'clean'
'mochaTest'
'availabletasks'
'execute'
'bunyan'
]
groups:
"Compile tasks": [
"compile:server"
"compile:tests"
"compile"
"compile:unit_tests"
"compile:acceptance_tests"
"install"
]
"Test tasks": [
"test:unit"
"test:acceptance"
]
"Run tasks": [
"run"
"default"
]
"Misc": [
"help"
]
grunt.registerTask 'help', 'Display this help list', 'availabletasks'
grunt.registerTask 'compile:server', 'Compile the server side coffee script', ['clean:app', 'coffee:app', 'coffee:app_dir']
grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['coffee:unit_tests']
grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
grunt.registerTask 'compile:tests', 'Compile all the tests', ['compile:acceptance_tests', 'compile:unit_tests']
grunt.registerTask 'compile', 'Compiles everything need to run track-changes-sharelatex', ['compile:server']
grunt.registerTask 'install', "Compile everything when installing as an npm module", ['compile']
grunt.registerTask 'test:unit', 'Run the unit tests (use --grep=<regex> for individual tests)', ['compile:server', 'compile:unit_tests', 'mochaTest:unit']
grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep=<regex> for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance']
grunt.registerTask 'run', "Compile and run the track-changes-sharelatex server", ['compile', 'bunyan', 'execute']
grunt.registerTask 'default', 'run'

100
services/track-changes/Jenkinsfile vendored
View file

@ -1,57 +1,77 @@
String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
pipeline {
agent any
agent {
docker {
image 'node:6.11.2'
args "-v /var/lib/jenkins/.npm:/tmp/.npm"
}
}
environment {
HOME = "/tmp"
environment {
GIT_PROJECT = "track-changes-sharelatex"
JENKINS_WORKFLOW = "track-changes-sharelatex"
TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT"
}
triggers {
pollSCM('* * * * *')
cron('@daily')
cron(cron_string)
}
stages {
stage('Set up') {
steps {
// we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
// which does not exist in the container's /etc/passwd file, causing the clone to fail.
sh 'git config --global core.logallrefupdates false'
}
}
stage('Install') {
steps {
sh 'rm -fr node_modules'
sh 'npm install'
sh 'npm install --quiet grunt-cli'
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"pending\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build is underway\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
}
stage('Compile') {
stage('Build') {
steps {
sh 'node_modules/.bin/grunt compile'
sh 'make build'
}
}
stage('Test') {
stage('Unit Tests') {
steps {
sh 'NODE_ENV=development node_modules/.bin/grunt test:unit'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
}
}
stage('Package') {
stage('Acceptance Tests') {
steps {
withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) {
sh 'AWS_BUCKET="sl-acceptance-tests" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
}
}
stage('Package and docker push') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
}
sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
sh 'docker logout https://gcr.io/overleaf-ops'
}
}
stage('Publish') {
stage('Publish to s3') {
steps {
sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
}
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
}
@ -60,11 +80,37 @@ pipeline {
}
post {
always {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
sh 'make clean'
}
success {
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"success\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build succeeded!\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
failure {
mail(from: "${EMAIL_ALERT_FROM}",
to: "${EMAIL_ALERT_TO}",
subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
body: "Build: ${BUILD_URL}")
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"failure\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build failed\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
}

52
services/track-changes/Makefile Normal file
View file

@ -0,0 +1,52 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.12
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
PROJECT_NAME = track-changes
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
AWS_BUCKET=${AWS_BUCKET} \
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
clean:
docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
rm -rf test/acceptance/js
test: test_unit test_acceptance
test_unit:
@[ ! -d test/unit ] && echo "track-changes has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
test_clean:
$(DOCKER_COMPOSE) down -v -t 0
test_acceptance_pre_run:
@[ ! -f test/acceptance/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
build:
docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
.
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish

9
services/track-changes/README.md
View file

@ -5,6 +5,15 @@ An API for converting raw editor updates into a compressed and browseable histor
[![Build Status](https://travis-ci.org/sharelatex/track-changes-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/track-changes-sharelatex)
Acceptance tests can be run with the command
```
AWS_BUCKET=<bucket-name> AWS_ACCESS_KEY_ID=<aws-access-key> AWS_SECRET_ACCESS_KEY=<aws-secret-access-key> make test
```
where `bucket-name`, `aws-access-key` and `aws-secret-access-key` are the credentials for an AWS S3 bucket.
License
-------

21
services/track-changes/app.coffee
View file

@ -1,3 +1,5 @@
Metrics = require "metrics-sharelatex"
Metrics.initialize("track-changes")
Settings = require "settings-sharelatex"
logger = require "logger-sharelatex"
TrackChangesLogger = logger.initialize("track-changes").logger
@ -23,8 +25,7 @@ TrackChangesLogger.addSerializers {
}
Path = require "path"
Metrics = require "metrics-sharelatex"
Metrics.initialize("track-changes")
Metrics.memory.monitor(logger)
child_process = require "child_process"
@ -35,6 +36,8 @@ app = express()
app.use Metrics.http.monitor(logger)
Metrics.injectMetricsRoute(app)
app.post "/project/:project_id/doc/:doc_id/flush", HttpController.flushDoc
app.get "/project/:project_id/doc/:doc_id/diff", HttpController.getDiff
@ -92,9 +95,13 @@ app.use (error, req, res, next) ->
port = Settings.internal?.trackchanges?.port or 3015
host = Settings.internal?.trackchanges?.host or "localhost"
app.listen port, host, (error) ->
if error?
logger.error err: error, "could not start track-changes server"
else
logger.info "trackchanges starting up, listening on #{host}:#{port}"
if !module.parent # Called directly
app.listen port, host, (error) ->
if error?
logger.error err: error, "could not start track-changes server"
else
logger.info "trackchanges starting up, listening on #{host}:#{port}"
module.exports = app

8
services/track-changes/buildscript.txt Normal file
View file

@ -0,0 +1,8 @@
track-changes
--language=coffeescript
--node-version=6.11.2
--acceptance-creds=aws
--dependencies=mongo,redis
--docker-repos=gcr.io/overleaf-ops
--build-target=docker
--script-version=1.1.12

32
services/track-changes/config/settings.defaults.coffee
View file

@ -1,16 +1,17 @@
Path = require('path')
TMP_DIR = Path.resolve(Path.join(__dirname, "../../", "tmp"))
TMP_DIR = process.env["TMP_PATH"] or Path.resolve(Path.join(__dirname, "../../", "tmp"))
module.exports =
mongo:
url: "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
internal:
trackchanges:
port: 3015
host: process.env["LISTEN_ADDRESS"] or "localhost"
apis:
documentupdater:
url: "http://#{process.env["DOCUPDATER_HOST"] or "localhost"}:3003"
url: "http://#{process.env["DOCUMENT_UPDATER_HOST"] or process.env["DOCUPDATER_HOST"] or "localhost"}:3003"
docstore:
url: "http://#{process.env["DOCSTORE_HOST"] or "localhost"}:3016"
web:
@ -20,26 +21,25 @@ module.exports =
redis:
lock:
host: process.env["REDIS_HOST"] or "localhost"
port: 6379
pass: ""
port: process.env['REDIS_PORT'] or 6379
password: process.env["REDIS_PASSWORD"] or ""
key_schema:
historyLock: ({doc_id}) -> "HistoryLock:#{doc_id}"
historyIndexLock: ({project_id}) -> "HistoryIndexLock:#{project_id}"
historyLock: ({doc_id}) -> "HistoryLock:{#{doc_id}}"
historyIndexLock: ({project_id}) -> "HistoryIndexLock:{#{project_id}}"
history:
port: "6379"
host: process.env["REDIS_HOST"] or "localhost"
password:""
port: process.env['REDIS_PORT'] or 6379
password: process.env["REDIS_PASSWORD"] or ""
key_schema:
uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:#{doc_id}"
docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:#{project_id}"
uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:{#{doc_id}}"
docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}"
trackchanges:
s3:
key: ""
secret: ""
key: process.env['AWS_ACCESS_KEY_ID']
secret: process.env['AWS_SECRET_ACCESS_KEY']
stores:
doc_history: ""
doc_history: process.env['AWS_BUCKET']
path:
dumpFolder: Path.join(TMP_DIR, "dumpFolder")

47
services/track-changes/docker-compose.ci.yml Normal file
View file

@ -0,0 +1,47 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.12
version: "2"
services:
test_unit:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
command: npm run test:unit:_run
test_acceptance:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_BUCKET: ${AWS_BUCKET}
MOCHA_GREP: ${MOCHA_GREP}
depends_on:
- mongo
- redis
user: node
command: npm run test:acceptance:_run
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
mongo:
image: mongo:3.4

54
services/track-changes/docker-compose.yml Normal file
View file

@ -0,0 +1,54 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.12
version: "2"
services:
test_unit:
build: .
volumes:
- .:/app
working_dir: /app
environment:
MOCHA_GREP: ${MOCHA_GREP}
command: npm run test:unit
user: node
test_acceptance:
build: .
volumes:
- .:/app
working_dir: /app
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_BUCKET: ${AWS_BUCKET}
MOCHA_GREP: ${MOCHA_GREP}
user: node
depends_on:
- mongo
- redis
command: npm run test:acceptance
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
mongo:
image: mongo:3.4

19
services/track-changes/nodemon.json Normal file
View file

@ -0,0 +1,19 @@
{
"ignore": [
".git",
"node_modules/"
],
"verbose": true,
"legacyWatch": true,
"execMap": {
"js": "npm run start"
},
"watch": [
"app/coffee/",
"app.coffee",
"config/"
],
"ext": "coffee"
}

2079
services/track-changes/npm-shrinkwrap.json generated Normal file
View file

File diff suppressed because it is too large Load diff

42
services/track-changes/package.json
View file

@ -7,8 +7,17 @@
"url": "https://github.com/sharelatex/track-changes-sharelatex.git"
},
"scripts": {
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
"start": "npm run compile:app && node app.js"
"compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
"compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
"compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
"nodemon": "nodemon --config nodemon.json",
"compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
},
"dependencies": {
"JSONStream": "^1.0.4",
@ -20,34 +29,27 @@
"express": "3.3.5",
"heap": "^0.2.6",
"line-reader": "^0.2.4",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.7.1",
"logger-sharelatex": "^1.6.0",
"metrics-sharelatex": "^2.1.1",
"mongo-uri": "^0.1.2",
"mongojs": "2.4.0",
"redis": "~0.10.1",
"redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
"redis-sharelatex": "^1.0.5",
"request": "~2.33.0",
"requestretry": "^1.12.0",
"s3-streams": "^0.3.0",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
"settings-sharelatex": "^1.1.0",
"underscore": "~1.7.0",
"v8-profiler": "^5.6.5"
},
"devDependencies": {
"chai": "~4.1.1",
"coffee-script": "^1.7.1",
"sinon": "~3.2.1",
"sandboxed-module": "~0.3.0",
"grunt-execute": "~0.1.5",
"grunt-contrib-clean": "~0.5.0",
"grunt-mocha-test": "~0.9.3",
"grunt": "~0.4.2",
"grunt-available-tasks": "~0.4.2",
"grunt-contrib-coffee": "~0.10.1",
"bunyan": "~2.0.2",
"grunt-bunyan": "~0.5.0",
"grunt-forever": "~0.4.2",
"timekeeper": "0.0.4",
"memorystream": "0.3.1"
"chai": "~4.1.1",
"memorystream": "0.3.1",
"mocha": "^4.0.1",
"coffee-script": "^1.7.1",
"sandboxed-module": "~0.3.0",
"sinon": "~3.2.1",
"timekeeper": "0.0.4"
}
}

6
services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee
View file

@ -6,12 +6,16 @@ mongojs = require "../../../app/js/mongojs"
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
request = require "request"
rclient = require("redis").createClient() # Only works locally for now
rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockWebApi = require "./helpers/MockWebApi"
describe "Appending doc ops to the history", ->
before (done)->
TrackChangesApp.ensureRunning done
describe "when the history does not exist yet", ->
before (done) ->
@project_id = ObjectId().toString()

17
services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee
View file

@ -7,14 +7,19 @@ db = mongojs.db
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
request = require "request"
rclient = require("redis").createClient() # Only works locally for now
rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockDocStoreApi = require "./helpers/MockDocStoreApi"
MockWebApi = require "./helpers/MockWebApi"
describe "Archiving updates", ->
before (done) ->
if Settings?.trackchanges?.s3?.key.length < 1
message = "s3 keys not setup, this test setup will fail"
return done(message)
@now = Date.now()
@to = @now
@user_id = ObjectId().toString()
@ -53,12 +58,12 @@ describe "Archiving updates", ->
meta: { ts: @now + (i-2048) * @hours + 10*@minutes, user_id: @user_id }
v: 2 * i + 2
}
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
TrackChangesApp.ensureRunning =>
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
done()
TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
throw error if error?
done()
after (done) ->
MockWebApi.getUserInfo.restore()

6
services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee
View file

@ -6,12 +6,16 @@ mongojs = require "../../../app/js/mongojs"
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
request = require "request"
rclient = require("redis").createClient() # Only works locally for now
rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockWebApi = require "./helpers/MockWebApi"
describe "Flushing updates", ->
before (done)->
TrackChangesApp.ensureRunning done
describe "flushing a doc's updates", ->
before (done) ->
@project_id = ObjectId().toString()

18
services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee
View file

@ -7,12 +7,14 @@ db = mongojs.db
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
MockWebApi = require "./helpers/MockWebApi"
describe "Getting a diff", ->
before (done) ->
beforeEach (done) ->
sinon.spy MockDocUpdaterApi, "getDoc"
@now = Date.now()
@ -58,15 +60,15 @@ describe "Getting a diff", ->
MockDocUpdaterApi.docs[@doc_id] =
lines: @lines
version: 7
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) =>
TrackChangesApp.ensureRunning =>
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
@diff = diff.diff
done()
TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) =>
throw error if error?
@diff = diff.diff
done()
after () ->
afterEach () ->
MockDocUpdaterApi.getDoc.restore()
MockWebApi.getUserInfo.restore()

10
services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee
View file

@ -7,6 +7,7 @@ db = mongojs.db
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockWebApi = require "./helpers/MockWebApi"
@ -46,10 +47,11 @@ describe "Getting updates", ->
v: 2 * i + 2
}
@updates[0].meta.user_id = @deleted_user_id
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
done()
TrackChangesApp.ensureRunning =>
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
done()
after: () ->
MockWebApi.getUserInfo.restore()

7
services/track-changes/test/acceptance/coffee/LockManagerTests.coffee
View file

@ -6,9 +6,14 @@ mongojs = require "../../../app/js/mongojs"
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
LockManager = require "../../../app/js/LockManager"
rclient = require("redis").createClient() # Only works locally for now
rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
TrackChangesApp = require "./helpers/TrackChangesApp"
describe "Locking document", ->
before (done)->
TrackChangesApp.ensureRunning done
describe "when the lock has expired in redis", ->
before (done) ->
LockManager.LOCK_TTL = 1 # second

10
services/track-changes/test/acceptance/coffee/RestoringVersions.coffee
View file

@ -7,6 +7,7 @@ db = mongojs.db
ObjectId = mongojs.ObjectId
Settings = require "settings-sharelatex"
TrackChangesApp = require "./helpers/TrackChangesApp"
TrackChangesClient = require "./helpers/TrackChangesClient"
MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
MockWebApi = require "./helpers/MockWebApi"
@ -54,11 +55,12 @@ describe "Restoring a version", ->
lines: @lines
version: 7
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) =>
TrackChangesApp.ensureRunning =>
TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
throw error if error?
done()
TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) =>
throw error if error?
done()
after () ->
MockDocUpdaterApi.setDoc.restore()

24
services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee Normal file
View file

@ -0,0 +1,24 @@
app = require('../../../../app')
require("logger-sharelatex").logger.level("info")
logger = require("logger-sharelatex")
Settings = require("settings-sharelatex")
module.exports =
running: false
initing: false
callbacks: []
ensureRunning: (callback = (error) ->) ->
if @running
return callback()
else if @initing
@callbacks.push callback
else
@initing = true
@callbacks.push callback
app.listen Settings.internal?.trackchanges?.port, "localhost", (error) =>
throw error if error?
@running = true
logger.log("track changes running in dev mode")
for callback in @callbacks
callback()