diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index d87edbfc10..26ec038c18 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -4.2.1 \ No newline at end of file +6.9.5 \ No newline at end of file diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile new file mode 100644 index 0000000000..d908d42063 --- /dev/null +++ b/services/real-time/Jenkinsfile @@ -0,0 +1,88 @@ +pipeline { + + agent any + + environment { + HOME = "/tmp" + } + + triggers { + pollSCM('* * * * *') + cron('@daily') + } + + stages { + stage('Install') { + agent { + docker { + image 'node:6.9.5' + args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" + reuseNode true + } + } + steps { + // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id + // which does not exist in the container's /etc/passwd file, causing the clone to fail. + sh 'git config --global core.logallrefupdates false' + sh 'rm -fr node_modules' + sh 'npm install' + sh 'npm rebuild' + sh 'npm install --quiet grunt-cli' + } + } + stage('Compile and Test') { + agent { + docker { + image 'node:6.9.5' + args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" + reuseNode true + } + } + steps { + sh 'node_modules/.bin/grunt install' + sh 'node_modules/.bin/grunt compile:acceptance_tests' + sh 'node_modules/.bin/grunt test:unit' + } + } + stage('Acceptance Tests') { + steps { + sh 'docker pull sharelatex/acceptance-test-runner' + sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner' + } + } + stage('Package') { + steps { + sh 'echo ${BUILD_NUMBER} > build_number.txt' + sh 'touch build.tar.gz' // Avoid tar warning about files changing during read + sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' + } + } + stage('Publish') { + steps { + withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { + s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + // The deployment process uses this file to figure out the latest build + s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") + } + } + } + } + + post { + failure { + mail(from: "${EMAIL_ALERT_FROM}", + to: "${EMAIL_ALERT_TO}", + subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", + body: "Build: ${BUILD_URL}") + } + } + + // The options directive is for configuration that applies to the whole job. + options { + // we'd like to make sure remove old builds, so we don't fill up our storage! + buildDiscarder(logRotator(numToKeepStr:'50')) + + // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: + timeout(time: 30, unit: 'MINUTES') + } +} diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 0365f773c6..824cdce5db 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -5,6 +5,8 @@ express = require("express") session = require("express-session") redis = require("redis-sharelatex") Settings = require "settings-sharelatex" +if Settings.sentry?.dsn? + logger.initializeErrorReporting(Settings.sentry.dsn) sessionRedisClient = redis.createClient(Settings.redis.websessions) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index ab893aa47f..e183d11cec 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -54,6 +54,10 @@ module.exports = DocumentUpdaterManager = queueChange: (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change + if jsonChange.indexOf("\u0000") != -1 + error = new Error("null bytes found in op") + logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message + return callback(error) doc_key = "#{project_id}:#{doc_id}" # Push onto pendingUpdates for doc_id first, because once the doc updater # gets an entry on pending-updates-list, it starts processing. diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 57a0c4cc07..d59853e831 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -81,14 +81,30 @@ module.exports = Router = if err? Router._handleError null, err, client, "leaveProject" - - client.on "joinDoc", (doc_id, fromVersion, callback) -> - # fromVersion is optional - if typeof fromVersion == "function" + # Variadic. The possible arguments: + # doc_id, callback + # doc_id, fromVersion, callback + # doc_id, options, callback + # doc_id, fromVersion, options, callback + client.on "joinDoc", (doc_id, fromVersion, options, callback) -> + if typeof fromVersion == "function" and !options callback = fromVersion fromVersion = -1 + options = {} + else if typeof fromVersion == "number" and typeof options == "function" + callback = options + options = {} + else if typeof fromVersion == "object" and typeof options == "function" + callback = options + options = fromVersion + fromVersion = -1 + else if typeof fromVersion == "number" and typeof options == "object" + # Called with 4 args, things are as expected + else + logger.error { arguments: arguments }, "unexpected arguments" + return callback?(new Error("unexpected arguments")) - WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> + WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> if err? Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} else diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 0da81b49a3..77b797d560 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -80,7 +80,7 @@ module.exports = WebsocketController = callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) -> + joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? @@ -91,16 +91,29 @@ module.exports = WebsocketController = return callback(error) if error? DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? + # Encode any binary bits of data so it can go via WebSockets # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) escapedLines = [] for line in lines try - line = unescape(encodeURIComponent(line)) + line = encodeForWebsockets(line) catch err logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" return callback(err) escapedLines.push line + if options.encodeRanges + try + for comment in ranges?.comments or [] + comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c? + for change in ranges?.changes or [] + change.op.i = encodeForWebsockets(change.op.i) if change.op.i? + change.op.d = encodeForWebsockets(change.op.d) if change.op.d? + catch err + logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" + return callback(err) + AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) callback null, escapedLines, version, ops, ranges diff --git a/services/real-time/package.json b/services/real-time/package.json index 4b3de3d5f3..8cfb35291a 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,10 +15,9 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "ioredis": "^2.4.0", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index a9d5406345..6c204b6079 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -166,4 +166,81 @@ describe "joinDoc", -> it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@doc_id in client.rooms).to.equal true - done() \ No newline at end of file + done() + + describe "with options", -> + before (done) -> + @options = { encodeRanges: true } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, @options, (error, @returnedArgs...) => cb(error) + ], done + + it "should get the doc from the doc updater with the default fromVersion", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() + + describe "with fromVersion and options", -> + before (done) -> + @fromVersion = 36 + @options = { encodeRanges: true } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, @fromVersion, @options, (error, @returnedArgs...) => cb(error) + ], done + + it "should get the doc from the doc updater with the fromVersion", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee index 0f196371b3..ac5bfc7093 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee @@ -38,5 +38,9 @@ module.exports = MockDocUpdaterServer = app.listen 3003, (error) -> MockDocUpdaterServer.running = true callback(error) + .on "error", (error) -> + console.error "error starting MockDocUpdaterServer:", error.message + process.exit(1) + -sinon.spy MockDocUpdaterServer, "getDocument" \ No newline at end of file +sinon.spy MockDocUpdaterServer, "getDocument" diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee index 2fff23e252..06f52a6b19 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee @@ -35,5 +35,9 @@ module.exports = MockWebServer = app.listen 3000, (error) -> MockWebServer.running = true callback(error) + .on "error", (error) -> + console.error "error starting MockWebServer:", error.message + process.exit(1) + -sinon.spy MockWebServer, "joinProject" \ No newline at end of file +sinon.spy MockWebServer, "joinProject" diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index e6f9b2098b..2d3e0ad3af 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -17,14 +17,17 @@ describe 'DocumentUpdaterManager', -> pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" @rclient = {auth:->} - @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: - 'settings-sharelatex':@settings - 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} - 'request': @request = {} - 'redis-sharelatex' : createClient: () => @rclient - 'metrics-sharelatex': @Metrics = - Timer: class Timer - done: () -> + @DocumentUpdaterManager = SandboxedModule.require modulePath, + requires: + 'settings-sharelatex':@settings + 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} + 'request': @request = {} + 'redis-sharelatex' : createClient: () => @rclient + 'metrics-sharelatex': @Metrics = + Timer: class Timer + done: () -> + globals: + JSON: @JSON = Object.create(JSON) # avoid modifying JSON object directly describe "getDocument", -> beforeEach -> @@ -147,3 +150,14 @@ describe 'DocumentUpdaterManager', -> it "should return an error", -> @callback.calledWithExactly(sinon.match(Error)).should.equal true + + describe "with null byte corruption", -> + beforeEach -> + @JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]' + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should return an error", -> + @callback.calledWithExactly(sinon.match(Error)).should.equal true + + it "should not push the change onto the pending-updates-list queue", -> + @rclient.rpush.called.should.equal false diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index cddbc1a4c5..dd220d254a 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -221,49 +221,59 @@ describe 'WebsocketController', -> @version = 42 @ops = ["mock", "ops"] @ranges = { "mock": "ranges" } + @options = {} @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) - - describe "with a fromVersion", -> + + describe "works", -> beforeEach -> - @fromVersion = 40 - @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @callback - + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) .should.equal true - - it "should get the document from the DocumentUpdaterManager", -> + + it "should get the document from the DocumentUpdaterManager with fromVersion", -> @DocumentUpdaterManager.getDocument - .calledWith(@project_id, @doc_id, @fromVersion) + .calledWith(@project_id, @doc_id, -1) .should.equal true it "should add permissions for the client to access the doc", -> @AuthorizationManager.addAccessToDoc .calledWith(@client, @doc_id) .should.equal true - + it "should join the client to room for the doc_id", -> @client.join .calledWith(@doc_id) .should.equal true - + it "should call the callback with the lines, version, ranges and ops", -> @callback .calledWith(null, @doc_lines, @version, @ops, @ranges) .should.equal true - + it "should increment the join-doc metric", -> @metrics.inc.calledWith("editor.join-doc").should.equal true + + describe "with a fromVersion", -> + beforeEach -> + @fromVersion = 40 + @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback + it "should get the document from the DocumentUpdaterManager with fromVersion", -> + @DocumentUpdaterManager.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + describe "with doclines that need escaping", -> beforeEach -> @doc_lines.push ["räksmörgås"] - @WebsocketController.joinDoc @client, @doc_id, -1, @callback + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should call the callback with the escaped lines", -> escaped_lines = @callback.args[0][1] @@ -271,11 +281,41 @@ describe 'WebsocketController', -> escaped_word.should.equal 'räksmörgÃ¥s' # Check that unescaping works decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås" - + + describe "with comments that need encoding", -> + beforeEach -> + @ranges.comments = [{ op: { c: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + it "should call the callback with the encoded comment", -> + encoded_comments = @callback.args[0][4] + encoded_comment = encoded_comments.comments.pop() + encoded_comment_text = encoded_comment.op.c + encoded_comment_text.should.equal 'räksmörgÃ¥s' + + describe "with changes that need encoding", -> + it "should call the callback with the encoded insert change", -> + @ranges.changes = [{ op: { i: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + encoded_changes = @callback.args[0][4] + encoded_change = encoded_changes.changes.pop() + encoded_change_text = encoded_change.op.i + encoded_change_text.should.equal 'räksmörgÃ¥s' + + it "should call the callback with the encoded delete change", -> + @ranges.changes = [{ op: { d: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + encoded_changes = @callback.args[0][4] + encoded_change = encoded_changes.changes.pop() + encoded_change_text = encoded_change.op.d + encoded_change_text.should.equal 'räksmörgÃ¥s' + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) - @WebsocketController.joinDoc @client, @doc_id, -1, @callback + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should call the callback with an error", -> @callback.calledWith(@err).should.equal true