diff --git a/services/track-changes/.dockerignore b/services/track-changes/.dockerignore new file mode 100644 index 0000000000..386f26df30 --- /dev/null +++ b/services/track-changes/.dockerignore @@ -0,0 +1,9 @@ +node_modules/* +gitrev +.git +.gitignore +.npm +.nvmrc +nodemon.json +app.js +**/js/* diff --git a/services/track-changes/Dockerfile b/services/track-changes/Dockerfile new file mode 100644 index 0000000000..4fe121ef4c --- /dev/null +++ b/services/track-changes/Dockerfile @@ -0,0 +1,22 @@ +FROM node:6.11.2 as app + +WORKDIR /app + +#wildcard as some files may not be in all repos +COPY package*.json npm-shrink*.json /app/ + +RUN npm install --quiet + +COPY . /app + + +RUN npm run compile:all + +FROM node:6.11.2 + +COPY --from=app /app /app + +WORKDIR /app +USER node + +CMD ["node","app.js"] diff --git a/services/track-changes/Jenkinsfile b/services/track-changes/Jenkinsfile index 72206c4485..e8eefc2e9d 100644 --- a/services/track-changes/Jenkinsfile +++ b/services/track-changes/Jenkinsfile @@ -1,57 +1,44 @@ +String cron_string = BRANCH_NAME == "master" ? "@daily" : "" + pipeline { - - agent { - docker { - image 'node:6.11.2' - args "-v /var/lib/jenkins/.npm:/tmp/.npm" - } - } - - environment { - HOME = "/tmp" - } + agent any triggers { pollSCM('* * * * *') - cron('@daily') + cron(cron_string) } stages { - stage('Set up') { + stage('Build') { steps { - // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id - // which does not exist in the container's /etc/passwd file, causing the clone to fail. - sh 'git config --global core.logallrefupdates false' + sh 'make build' } } - stage('Install') { + + stage('Unit Tests') { steps { - sh 'rm -fr node_modules' - sh 'npm install' - sh 'npm install --quiet grunt-cli' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' } } - stage('Compile') { + + stage('Acceptance Tests') { steps { - sh 'node_modules/.bin/grunt compile' + withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) { + sh 'AWS_BUCKET="sl-doc-archive-testing" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' + } } } - stage('Test') { + + stage('Package and publish build') { steps { - sh 'NODE_ENV=development node_modules/.bin/grunt test:unit' + sh 'make publish' } } - stage('Package') { - steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' - } - } - stage('Publish') { + + stage('Publish build number') { steps { + sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } @@ -60,6 +47,10 @@ pipeline { } post { + always { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + } + failure { mail(from: "${EMAIL_ALERT_FROM}", to: "${EMAIL_ALERT_TO}", diff --git a/services/track-changes/Makefile b/services/track-changes/Makefile new file mode 100644 index 0000000000..b26c530f57 --- /dev/null +++ b/services/track-changes/Makefile @@ -0,0 +1,45 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = track-changes +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + AWS_BUCKET=${AWS_BUCKET} \ + AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ + AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ + docker-compose ${DOCKER_COMPOSE_FLAGS} + + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + @[ ! -d test/unit ] && echo "track-changes has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit + +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run + @[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance + +test_clean: + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_pre_run: + @[ ! -f test/acceptance/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run +build: + docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + +.PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index a83d78c7f1..513cdb7f50 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -92,9 +92,13 @@ app.use (error, req, res, next) -> port = Settings.internal?.trackchanges?.port or 3015 host = Settings.internal?.trackchanges?.host or "localhost" -app.listen port, host, (error) -> - if error? - logger.error err: error, "could not start track-changes server" - else - logger.info "trackchanges starting up, listening on #{host}:#{port}" + +if !module.parent # Called directly + app.listen port, host, (error) -> + if error? + logger.error err: error, "could not start track-changes server" + else + logger.info "trackchanges starting up, listening on #{host}:#{port}" + +module.exports = app diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index 6cc3b51c35..4bc5962d77 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -35,11 +35,10 @@ module.exports = trackchanges: s3: - key: "" - secret: "" + key: process.env['AWS_ACCESS_KEY_ID'] + secret: process.env['AWS_SECRET_ACCESS_KEY'] stores: - doc_history: "" - - + doc_history: process.env['AWS_BUCKET'] + path: dumpFolder: Path.join(TMP_DIR, "dumpFolder") diff --git a/services/track-changes/docker-compose.ci.yml b/services/track-changes/docker-compose.ci.yml new file mode 100644 index 0000000000..18a454adfd --- /dev/null +++ b/services/track-changes/docker-compose.ci.yml @@ -0,0 +1,35 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run + + test_acceptance: + build: . + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_BUCKET: ${AWS_BUCKET} + depends_on: + - mongo + - redis + user: node + command: npm run test:acceptance:_run + + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/track-changes/docker-compose.yml b/services/track-changes/docker-compose.yml new file mode 100644 index 0000000000..31fe46d3ed --- /dev/null +++ b/services/track-changes/docker-compose.yml @@ -0,0 +1,42 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + build: . + volumes: + - .:/app + working_dir: /app + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit + user: node + + test_acceptance: + build: . + volumes: + - .:/app + working_dir: /app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_BUCKET: ${AWS_BUCKET} + MOCHA_GREP: ${MOCHA_GREP} + user: node + depends_on: + - mongo + - redis + command: npm run test:acceptance + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/track-changes/nodemon.json b/services/track-changes/nodemon.json new file mode 100644 index 0000000000..98db38d71b --- /dev/null +++ b/services/track-changes/nodemon.json @@ -0,0 +1,19 @@ +{ + "ignore": [ + ".git", + "node_modules/" + ], + "verbose": true, + "legacyWatch": true, + "execMap": { + "js": "npm run start" + }, + + "watch": [ + "app/coffee/", + "app.coffee", + "config/" + ], + "ext": "coffee" + +} diff --git a/services/track-changes/package.json b/services/track-changes/package.json index ee3ceddd13..b8781ed635 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -7,8 +7,16 @@ "url": "https://github.com/sharelatex/track-changes-sharelatex.git" }, "scripts": { - "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js" + "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "nodemon": "nodemon --config nodemon.json" }, "dependencies": { "JSONStream": "^1.0.4", @@ -44,6 +52,7 @@ "grunt-available-tasks": "~0.4.2", "grunt-contrib-coffee": "~0.10.1", "bunyan": "~2.0.2", + "mocha": "^4.0.1", "grunt-bunyan": "~0.5.0", "grunt-forever": "~0.4.2", "timekeeper": "0.0.4", diff --git a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee index 69d378fd9d..e018cd2be0 100644 --- a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee @@ -6,12 +6,18 @@ mongojs = require "../../../app/js/mongojs" ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" request = require "request" -rclient = require("redis").createClient() # Only works locally for now +console.log "hiiiiis" +console.log Settings.redis.history +rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockWebApi = require "./helpers/MockWebApi" describe "Appending doc ops to the history", -> + before (done)-> + TrackChangesApp.ensureRunning done + describe "when the history does not exist yet", -> before (done) -> @project_id = ObjectId().toString() @@ -81,6 +87,7 @@ describe "Appending doc ops to the history", -> describe "when the updates are recent and from the same user", -> beforeEach (done) -> + console.log 1 TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [{ i: "b", p: 6 }] meta: { ts: Date.now(), user_id: @user_id } @@ -94,9 +101,13 @@ describe "Appending doc ops to the history", -> meta: { ts: Date.now(), user_id: @user_id } v: 8 }], (error) => + console.log 2, error throw error if error? + console.log 3 TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) => + console.log 4, error throw error if error? + console.log 5 done() it "should combine all the updates into one pack", -> diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index 398856e4d3..74d5d4031d 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -7,8 +7,9 @@ db = mongojs.db ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" request = require "request" -rclient = require("redis").createClient() # Only works locally for now +rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockDocStoreApi = require "./helpers/MockDocStoreApi" MockWebApi = require "./helpers/MockWebApi" @@ -53,12 +54,12 @@ describe "Archiving updates", -> meta: { ts: @now + (i-2048) * @hours + 10*@minutes, user_id: @user_id } v: 2 * i + 2 } - - TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => - throw error if error? - TrackChangesClient.flushDoc @project_id, @doc_id, (error) -> + TrackChangesApp.ensureRunning => + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => throw error if error? - done() + TrackChangesClient.flushDoc @project_id, @doc_id, (error) -> + throw error if error? + done() after (done) -> MockWebApi.getUserInfo.restore() diff --git a/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee index 3f82304da1..b3fd843c6b 100644 --- a/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee @@ -6,12 +6,16 @@ mongojs = require "../../../app/js/mongojs" ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" request = require "request" -rclient = require("redis").createClient() # Only works locally for now +rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockWebApi = require "./helpers/MockWebApi" describe "Flushing updates", -> + before (done)-> + TrackChangesApp.ensureRunning done + describe "flushing a doc's updates", -> before (done) -> @project_id = ObjectId().toString() diff --git a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee index f64f42ff87..916702eecf 100644 --- a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee @@ -7,12 +7,14 @@ db = mongojs.db ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi" MockWebApi = require "./helpers/MockWebApi" describe "Getting a diff", -> - before (done) -> + + beforeEach (done) -> sinon.spy MockDocUpdaterApi, "getDoc" @now = Date.now() @@ -58,15 +60,15 @@ describe "Getting a diff", -> MockDocUpdaterApi.docs[@doc_id] = lines: @lines version: 7 - - TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => - throw error if error? - TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) => + TrackChangesApp.ensureRunning => + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => throw error if error? - @diff = diff.diff - done() + TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) => + throw error if error? + @diff = diff.diff + done() - after () -> + afterEach () -> MockDocUpdaterApi.getDoc.restore() MockWebApi.getUserInfo.restore() diff --git a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee index d01797e25f..e8619be06e 100644 --- a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee @@ -7,6 +7,7 @@ db = mongojs.db ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockWebApi = require "./helpers/MockWebApi" @@ -46,10 +47,11 @@ describe "Getting updates", -> v: 2 * i + 2 } @updates[0].meta.user_id = @deleted_user_id - - TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => - throw error if error? - done() + + TrackChangesApp.ensureRunning => + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => + throw error if error? + done() after: () -> MockWebApi.getUserInfo.restore() diff --git a/services/track-changes/test/acceptance/coffee/LockManagerTests.coffee b/services/track-changes/test/acceptance/coffee/LockManagerTests.coffee index 6de4599a6b..0468a75109 100644 --- a/services/track-changes/test/acceptance/coffee/LockManagerTests.coffee +++ b/services/track-changes/test/acceptance/coffee/LockManagerTests.coffee @@ -6,9 +6,14 @@ mongojs = require "../../../app/js/mongojs" ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" LockManager = require "../../../app/js/LockManager" -rclient = require("redis").createClient() # Only works locally for now +rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now +TrackChangesApp = require "./helpers/TrackChangesApp" describe "Locking document", -> + + before (done)-> + TrackChangesApp.ensureRunning done + describe "when the lock has expired in redis", -> before (done) -> LockManager.LOCK_TTL = 1 # second diff --git a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee index b09328d2ea..62d374c28e 100644 --- a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee +++ b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee @@ -7,6 +7,7 @@ db = mongojs.db ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" +TrackChangesApp = require "./helpers/TrackChangesApp" TrackChangesClient = require "./helpers/TrackChangesClient" MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi" MockWebApi = require "./helpers/MockWebApi" @@ -54,11 +55,12 @@ describe "Restoring a version", -> lines: @lines version: 7 - TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => - throw error if error? - TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) => + TrackChangesApp.ensureRunning => + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => throw error if error? - done() + TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) => + throw error if error? + done() after () -> MockDocUpdaterApi.setDoc.restore() diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee new file mode 100644 index 0000000000..187427f56d --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee @@ -0,0 +1,24 @@ +app = require('../../../../app') +require("logger-sharelatex").logger.level("info") +logger = require("logger-sharelatex") +Settings = require("settings-sharelatex") + +module.exports = + running: false + initing: false + callbacks: [] + ensureRunning: (callback = (error) ->) -> + if @running + return callback() + else if @initing + @callbacks.push callback + else + @initing = true + @callbacks.push callback + app.listen Settings.internal?.trackchanges?.port, "localhost", (error) => + throw error if error? + @running = true + logger.log("track changes running in dev mode") + + for callback in @callbacks + callback() \ No newline at end of file